




'''simple docstring'''





import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


_SCREAMING_SNAKE_CASE							:       List[Any]						=			get_tests_dir("fixtures/test_sentencepiece.model")




@require_sentencepiece
class 			_snake_case		(     lowerCAmelCase_						,						unittest.TestCase							):
			lowerCAmelCase_					:				Dict							=   XLMProphetNetTokenizer
			lowerCAmelCase_					:				Optional[int]							=   False
			lowerCAmelCase_					:				Dict							=   True




			def        lowerCAmelCase__    ( self							)    ->    Any:

										'''simple docstring'''




										super().setUp()

										# We have a SentencePiece fixture for testing
										snake_case_      =				XLMProphetNetTokenizer(__SCREAMING_SNAKE_CASE    ,							keep_accents=__SCREAMING_SNAKE_CASE							)
										tokenizer.save_pretrained(self.tmpdirname							)




			def        lowerCAmelCase__    ( self							)    ->    Optional[Any]:

										'''simple docstring'''




										snake_case_      =				"[PAD]"
										snake_case_      =				0

										self.assertEqual(self.get_tokenizer()._convert_token_to_id(__SCREAMING_SNAKE_CASE							)    ,							__SCREAMING_SNAKE_CASE							)
										self.assertEqual(self.get_tokenizer()._convert_id_to_token(__SCREAMING_SNAKE_CASE							)    ,							__SCREAMING_SNAKE_CASE							)




			def        lowerCAmelCase__    ( self							)    ->    Optional[Any]:

										'''simple docstring'''




										snake_case_      =				list(self.get_tokenizer().get_vocab().keys()							)

										self.assertEqual(vocab_keys[0]    ,							"[PAD]"							)
										self.assertEqual(vocab_keys[1]    ,							"[CLS]"							)
										self.assertEqual(vocab_keys[-1]    ,							"j"							)
										self.assertEqual(len(__SCREAMING_SNAKE_CASE							)    ,							1_012							)




			def        lowerCAmelCase__    ( self							)    ->    Optional[Any]:

										'''simple docstring'''




										self.assertEqual(self.get_tokenizer().vocab_size    ,							1_012							)




			def        lowerCAmelCase__    ( self							)    ->    Dict:

										'''simple docstring'''




										snake_case_      =				XLMProphetNetTokenizer(__SCREAMING_SNAKE_CASE    ,							keep_accents=__SCREAMING_SNAKE_CASE							)

										snake_case_      =				tokenizer.tokenize("This is a test"							)
										self.assertListEqual(__SCREAMING_SNAKE_CASE    ,							["▁This", "▁is", "▁a", "▁t", "est"]							)

										self.assertListEqual(
										    tokenizer.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE							)    ,							[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]    ,							)

										snake_case_      =				tokenizer.tokenize("I was born in 92000, and this is falsé."							)
										self.assertListEqual(
										    __SCREAMING_SNAKE_CASE    ,							[
										        SPIECE_UNDERLINE + "I",
										        SPIECE_UNDERLINE + "was",
										        SPIECE_UNDERLINE + "b",
										        "or",
										        "n",
										        SPIECE_UNDERLINE + "in",
										        SPIECE_UNDERLINE + "",
										        "9",
										        "2",
										        "0",
										        "0",
										        "0",
										        ",",
										        SPIECE_UNDERLINE + "and",
										        SPIECE_UNDERLINE + "this",
										        SPIECE_UNDERLINE + "is",
										        SPIECE_UNDERLINE + "f",
										        "al",
										        "s",
										        "é",
										        ".",
										    ]    ,							)
										snake_case_      =				tokenizer.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE							)
										self.assertListEqual(
										    __SCREAMING_SNAKE_CASE    ,							[
										        value + tokenizer.fairseq_offset
										        for value in [8, 21, 84, 55, 24, 19, 7, -9, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, -9, 4]
										    ]    ,							)

										snake_case_      =				tokenizer.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE							)
										self.assertListEqual(
										    __SCREAMING_SNAKE_CASE    ,							[
										        SPIECE_UNDERLINE + "I",
										        SPIECE_UNDERLINE + "was",
										        SPIECE_UNDERLINE + "b",
										        "or",
										        "n",
										        SPIECE_UNDERLINE + "in",
										        SPIECE_UNDERLINE + "",
										        "[UNK]",
										        "2",
										        "0",
										        "0",
										        "0",
										        ",",
										        SPIECE_UNDERLINE + "and",
										        SPIECE_UNDERLINE + "this",
										        SPIECE_UNDERLINE + "is",
										        SPIECE_UNDERLINE + "f",
										        "al",
										        "s",
										        "[UNK]",
										        ".",
										    ]    ,							)




			@cached_property
			def        lowerCAmelCase__    ( self							)    ->    int:

										'''simple docstring'''




										return XLMProphetNetTokenizer.from_pretrained("microsoft/xprophetnet-large-wiki100-cased"							)




			@slow
			def        lowerCAmelCase__    ( self							)    ->    int:

										'''simple docstring'''




										snake_case_      =				"Hello World!"
										snake_case_      =				[35_389, 6_672, 49, 2]
										self.assertListEqual(__SCREAMING_SNAKE_CASE    ,							self.big_tokenizer.encode(__SCREAMING_SNAKE_CASE							)							)




			@slow
			def        lowerCAmelCase__    ( self							)    ->    Optional[Any]:

										'''simple docstring'''




										snake_case_      =				{"input_ids": [[11_073, 82_783, 18, 26, 82_783, 549, 51_540, 248, 17_209, 1_301, 217, 20, 215_186, 1_325, 147, 17_209, 1_301, 217, 20, 56_370, 53, 122_020, 20, 16_477, 27, 87_355, 4_548, 20, 4_728, 78_392, 17, 159_969, 18, 26, 24_491, 629, 15, 538, 22_704, 5_439, 15, 2_788, 24_491, 9_885, 15, 43_534, 605, 15, 814, 18_403, 33_200, 29, 15, 43_534, 24_458, 12_410, 111, 24_966, 83_669, 9_637, 144_068, 26, 850, 22_346, 27, 147, 24_966, 83_669, 83_490, 26, 39_113, 735, 27, 689, 656, 2_800, 1_339, 4_600, 53, 122_020, 115_785, 34, 816, 1_339, 46_887, 18, 147, 53_905, 1_951, 42_238, 41_170, 17_732, 834, 436, 15, 27_523, 98_733, 217, 147, 5_542, 4_981, 930, 17_347, 16, 2], [20_091, 629, 94, 82_786, 58, 490, 20, 1_528, 84, 53_905, 344, 80_592, 110_128, 18_822, 5_267, 1_306, 62, 152_537, 308, 7_997, 401, 124_427, 549, 35_442, 225, 109, 15_055, 25_748, 147, 7_119, 43_712, 34, 767, 135_366, 18, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [592, 63_784, 119_466, 17, 147_808, 88_214, 18, 656, 81, 32, 3_296, 10_280, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
										# fmt: on

										self.tokenizer_integration_test_util(
										    expected_encoding=__SCREAMING_SNAKE_CASE    ,							model_name="microsoft/xprophetnet-large-wiki100-cased"    ,							revision="1acad1643ddd54a44df6a1b797ada8373685d90e"    ,							)










'''simple docstring'''





from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


_SCREAMING_SNAKE_CASE							:       Optional[int]						=			{
    "configuration_canine": ["CANINE_PRETRAINED_CONFIG_ARCHIVE_MAP", "CanineConfig"],
    "tokenization_canine": ["CanineTokenizer"],
}

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		_SCREAMING_SNAKE_CASE							:       Dict						=			[
		    "CANINE_PRETRAINED_MODEL_ARCHIVE_LIST",
		    "CanineForMultipleChoice",
		    "CanineForQuestionAnswering",
		    "CanineForSequenceClassification",
		    "CanineForTokenClassification",
		    "CanineLayer",
		    "CanineModel",
		    "CaninePreTrainedModel",
		    "load_tf_weights_in_canine",
		]


if TYPE_CHECKING:
		from .configuration_canine import CANINE_PRETRAINED_CONFIG_ARCHIVE_MAP, CanineConfig
		from .tokenization_canine import CanineTokenizer

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_canine import (
				    CANINE_PRETRAINED_MODEL_ARCHIVE_LIST,
				    CanineForMultipleChoice,
				    CanineForQuestionAnswering,
				    CanineForSequenceClassification,
				    CanineForTokenClassification,
				    CanineLayer,
				    CanineModel,
				    CaninePreTrainedModel,
				    load_tf_weights_in_canine,
				)


else:
		import sys

		_SCREAMING_SNAKE_CASE							:       int						=			_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)




