



"""simple docstring"""




import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


lowerCAmelCase__		:					Dict							=get_tests_dir('''fixtures/test_sentencepiece.model''')

@require_sentencepiece
class        UpperCAmelCase_  (       UpperCamelCase_   ,   unittest.TestCase	):

						'''simple docstring'''




						UpperCamelCase__      :			Union[str, Any]      =  XLMProphetNetTokenizer
						UpperCamelCase__      :			Any      =  False
						UpperCamelCase__      :			Any      =  True






						def 	_A     (   self		):







													'''simple docstring'''

													super().setUp()

													# We have a SentencePiece fixture for testing
													__SCREAMING_SNAKE_CASE									= XLMProphetNetTokenizer(_lowercase				,							keep_accents=_lowercase		)
													tokenizer.save_pretrained(self.tmpdirname		)






						def 	_A     (   self		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= '[PAD]'
													__SCREAMING_SNAKE_CASE									= 0

													self.assertEqual(self.get_tokenizer()._convert_token_to_id(_lowercase		)				,							_lowercase		)
													self.assertEqual(self.get_tokenizer()._convert_id_to_token(_lowercase		)				,							_lowercase		)






						def 	_A     (   self		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= list(self.get_tokenizer().get_vocab().keys()		)

													self.assertEqual(vocab_keys[0]				,							'[PAD]'		)
													self.assertEqual(vocab_keys[1]				,							'[CLS]'		)
													self.assertEqual(vocab_keys[-1]				,							'j'		)
													self.assertEqual(len(_lowercase		)				,							1_012		)






						def 	_A     (   self		):







													'''simple docstring'''

													self.assertEqual(self.get_tokenizer().vocab_size				,							1_012		)






						def 	_A     (   self		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= XLMProphetNetTokenizer(_lowercase				,							keep_accents=_lowercase		)

													__SCREAMING_SNAKE_CASE									= tokenizer.tokenize('This is a test'		)
													self.assertListEqual(_lowercase				,							['▁This', '▁is', '▁a', '▁t', 'est']		)

													self.assertListEqual(
													    tokenizer.convert_tokens_to_ids(_lowercase		)				,							[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]				,							)

													__SCREAMING_SNAKE_CASE									= tokenizer.tokenize('I was born in 92000, and this is falsé.'		)
													self.assertListEqual(
													    _lowercase				,							[
													        SPIECE_UNDERLINE + 'I',
													        SPIECE_UNDERLINE + 'was',
													        SPIECE_UNDERLINE + 'b',
													        'or',
													        'n',
													        SPIECE_UNDERLINE + 'in',
													        SPIECE_UNDERLINE + '',
													        '9',
													        '2',
													        '0',
													        '0',
													        '0',
													        ',',
													        SPIECE_UNDERLINE + 'and',
													        SPIECE_UNDERLINE + 'this',
													        SPIECE_UNDERLINE + 'is',
													        SPIECE_UNDERLINE + 'f',
													        'al',
													        's',
													        'é',
													        '.',
													    ]				,							)
													__SCREAMING_SNAKE_CASE									= tokenizer.convert_tokens_to_ids(_lowercase		)
													self.assertListEqual(
													    _lowercase				,							[
													        value + tokenizer.fairseq_offset
													        for value in [8, 21, 84, 55, 24, 19, 7, -9, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, -9, 4]
													    ]				,							)

													__SCREAMING_SNAKE_CASE									= tokenizer.convert_ids_to_tokens(_lowercase		)
													self.assertListEqual(
													    _lowercase				,							[
													        SPIECE_UNDERLINE + 'I',
													        SPIECE_UNDERLINE + 'was',
													        SPIECE_UNDERLINE + 'b',
													        'or',
													        'n',
													        SPIECE_UNDERLINE + 'in',
													        SPIECE_UNDERLINE + '',
													        '[UNK]',
													        '2',
													        '0',
													        '0',
													        '0',
													        ',',
													        SPIECE_UNDERLINE + 'and',
													        SPIECE_UNDERLINE + 'this',
													        SPIECE_UNDERLINE + 'is',
													        SPIECE_UNDERLINE + 'f',
													        'al',
													        's',
													        '[UNK]',
													        '.',
													    ]				,							)






						@cached_property
						def 	_A     (   self		):







													'''simple docstring'''

													return XLMProphetNetTokenizer.from_pretrained('microsoft/xprophetnet-large-wiki100-cased'		)






						@slow
						def 	_A     (   self		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= 'Hello World!'
													__SCREAMING_SNAKE_CASE									= [35_389, 6_672, 49, 2]
													self.assertListEqual(_lowercase				,							self.big_tokenizer.encode(_lowercase		)		)






						@slow
						def 	_A     (   self		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= {'input_ids': [[11_073, 82_783, 18, 26, 82_783, 549, 51_540, 248, 17_209, 1_301, 217, 20, 215_186, 1_325, 147, 17_209, 1_301, 217, 20, 56_370, 53, 122_020, 20, 16_477, 27, 87_355, 4_548, 20, 4_728, 78_392, 17, 159_969, 18, 26, 24_491, 629, 15, 538, 22_704, 5_439, 15, 2_788, 24_491, 9_885, 15, 43_534, 605, 15, 814, 18_403, 33_200, 29, 15, 43_534, 24_458, 12_410, 111, 24_966, 83_669, 9_637, 144_068, 26, 850, 22_346, 27, 147, 24_966, 83_669, 83_490, 26, 39_113, 735, 27, 689, 656, 2_800, 1_339, 4_600, 53, 122_020, 115_785, 34, 816, 1_339, 46_887, 18, 147, 53_905, 1_951, 42_238, 41_170, 17_732, 834, 436, 15, 27_523, 98_733, 217, 147, 5_542, 4_981, 930, 17_347, 16, 2], [20_091, 629, 94, 82_786, 58, 490, 20, 1_528, 84, 53_905, 344, 80_592, 110_128, 18_822, 5_267, 1_306, 62, 152_537, 308, 7_997, 401, 124_427, 549, 35_442, 225, 109, 15_055, 25_748, 147, 7_119, 43_712, 34, 767, 135_366, 18, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [592, 63_784, 119_466, 17, 147_808, 88_214, 18, 656, 81, 32, 3_296, 10_280, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
													# fmt: on

													self.tokenizer_integration_test_util(
													    expected_encoding=_lowercase				,							model_name='microsoft/xprophetnet-large-wiki100-cased'				,							revision='1acad1643ddd54a44df6a1b797ada8373685d90e'				,							)











from collections import deque

from .hash_table import HashTable

class        UpperCAmelCase_  (       UpperCamelCase_	):

						'''simple docstring'''




						def __init__(   self				,							*_A				,							**_A		):







													'''simple docstring'''

													super().__init__(*_A				,							**_A		)






						def 	_A     (   self				,							_A				,							_A		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= deque([]		) if self.values[key] is None else self.values[key]
													self.values[key].appendleft(_A		)
													__SCREAMING_SNAKE_CASE									= self.values[key]






						def 	_A     (   self		):







													'''simple docstring'''

													return (
													    sum(self.charge_factor - len(_A		) for slot in self.values		)
													    / self.size_table
													    * self.charge_factor
													)






						def 	_A     (   self				,							_A				,							_A=None		):







													'''simple docstring'''

													if not (
													    len(self.values[key]		) == self.charge_factor and self.values.count(_A		) == 0
													):
																				return key
													return super()._collision_resolution(_A				,							_A		)






