



import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__snake_case    :		Any  =get_tests_dir('fixtures/test_sentencepiece.model')



@require_sentencepiece
@require_tokenizers
class 	lowerCamelCase__      (							lowerCamelCase__				,    unittest.TestCase):




		'''simple docstring'''



		snake_case_     =XLNetTokenizer
		snake_case_     =XLNetTokenizerFast
		snake_case_     =True
		snake_case_     =True



		def 		lowerCAmelCase__				(self			)					->      Optional[Any]:

						"""simple docstring"""






						super().setUp()

						# We have a SentencePiece fixture for testing
						lowerCAmelCase__					:    int												=					XLNetTokenizer(__lowerCamelCase		,keep_accents=__lowerCamelCase			)
						tokenizer.sanitize_special_tokens()
						tokenizer.save_pretrained(self.tmpdirname			)



		def 		lowerCAmelCase__				(self			)					->      List[Any]:

						"""simple docstring"""






						lowerCAmelCase__					:    str												=					'''<s>'''
						lowerCAmelCase__					:    Tuple												=					1

						self.assertEqual(self.get_tokenizer()._convert_token_to_id(__lowerCamelCase			)		,__lowerCamelCase			)
						self.assertEqual(self.get_tokenizer()._convert_id_to_token(__lowerCamelCase			)		,__lowerCamelCase			)



		def 		lowerCAmelCase__				(self			)					->      Dict:

						"""simple docstring"""






						lowerCAmelCase__					:    Dict												=					list(self.get_tokenizer().get_vocab().keys()			)

						self.assertEqual(vocab_keys[0]		,'''<unk>'''			)
						self.assertEqual(vocab_keys[1]		,'''<s>'''			)
						self.assertEqual(vocab_keys[-1]		,'''<eod>'''			)
						self.assertEqual(len(__lowerCamelCase			)		,10_06			)



		def 		lowerCAmelCase__				(self			)					->      Optional[Any]:

						"""simple docstring"""






						self.assertEqual(self.get_tokenizer().vocab_size		,10_00			)



		def 		lowerCAmelCase__				(self			)					->      Any:

						"""simple docstring"""






						lowerCAmelCase__					:    Dict												=					XLNetTokenizer(__lowerCamelCase		,keep_accents=__lowerCamelCase			)

						lowerCAmelCase__					:    Tuple												=					tokenizer.tokenize('''This is a test'''			)
						self.assertListEqual(__lowerCamelCase		,['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']			)

						self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase			)		,[2_85, 46, 10, 1_70, 3_82]			)

						lowerCAmelCase__					:    int												=					tokenizer.tokenize('''I was born in 92000, and this is falsé.'''			)
						self.assertListEqual(
						    __lowerCamelCase		,[
						        SPIECE_UNDERLINE + '''I''',
						        SPIECE_UNDERLINE + '''was''',
						        SPIECE_UNDERLINE + '''b''',
						        '''or''',
						        '''n''',
						        SPIECE_UNDERLINE + '''in''',
						        SPIECE_UNDERLINE + '''''',
						        '''9''',
						        '''2''',
						        '''0''',
						        '''0''',
						        '''0''',
						        ''',''',
						        SPIECE_UNDERLINE + '''and''',
						        SPIECE_UNDERLINE + '''this''',
						        SPIECE_UNDERLINE + '''is''',
						        SPIECE_UNDERLINE + '''f''',
						        '''al''',
						        '''s''',
						        '''é''',
						        '''.''',
						    ]		,)
						lowerCAmelCase__					:    str												=					tokenizer.convert_tokens_to_ids(__lowerCamelCase			)
						self.assertListEqual(__lowerCamelCase		,[8, 21, 84, 55, 24, 19, 7, 0, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 0, 4]			)

						lowerCAmelCase__					:    Tuple												=					tokenizer.convert_ids_to_tokens(__lowerCamelCase			)
						self.assertListEqual(
						    __lowerCamelCase		,[
						        SPIECE_UNDERLINE + '''I''',
						        SPIECE_UNDERLINE + '''was''',
						        SPIECE_UNDERLINE + '''b''',
						        '''or''',
						        '''n''',
						        SPIECE_UNDERLINE + '''in''',
						        SPIECE_UNDERLINE + '''''',
						        '''<unk>''',
						        '''2''',
						        '''0''',
						        '''0''',
						        '''0''',
						        ''',''',
						        SPIECE_UNDERLINE + '''and''',
						        SPIECE_UNDERLINE + '''this''',
						        SPIECE_UNDERLINE + '''is''',
						        SPIECE_UNDERLINE + '''f''',
						        '''al''',
						        '''s''',
						        '''<unk>''',
						        '''.''',
						    ]		,)



		def 		lowerCAmelCase__				(self			)					->      Optional[int]:

						"""simple docstring"""






						lowerCAmelCase__					:    Union[str, Any]												=					XLNetTokenizer(__lowerCamelCase		,do_lower_case=__lowerCamelCase			)
						lowerCAmelCase__					:    Dict												=					tokenizer.tokenize('''I was born in 92000, and this is falsé.'''			)
						self.assertListEqual(
						    __lowerCamelCase		,[
						        SPIECE_UNDERLINE + '''''',
						        '''i''',
						        SPIECE_UNDERLINE + '''was''',
						        SPIECE_UNDERLINE + '''b''',
						        '''or''',
						        '''n''',
						        SPIECE_UNDERLINE + '''in''',
						        SPIECE_UNDERLINE + '''''',
						        '''9''',
						        '''2''',
						        '''0''',
						        '''0''',
						        '''0''',
						        ''',''',
						        SPIECE_UNDERLINE + '''and''',
						        SPIECE_UNDERLINE + '''this''',
						        SPIECE_UNDERLINE + '''is''',
						        SPIECE_UNDERLINE + '''f''',
						        '''al''',
						        '''se''',
						        '''.''',
						    ]		,)
						self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''			)		,['''▁he''', '''ll''', '''o''']			)



		def 		lowerCAmelCase__				(self			)					->      Dict:

						"""simple docstring"""






						lowerCAmelCase__					:    Optional[Any]												=					XLNetTokenizer(__lowerCamelCase		,do_lower_case=__lowerCamelCase			)
						lowerCAmelCase__					:    int												=					tokenizer.tokenize('''I was born in 92000, and this is falsé.'''			)
						self.assertListEqual(
						    __lowerCamelCase		,[
						        SPIECE_UNDERLINE + '''I''',
						        SPIECE_UNDERLINE + '''was''',
						        SPIECE_UNDERLINE + '''b''',
						        '''or''',
						        '''n''',
						        SPIECE_UNDERLINE + '''in''',
						        SPIECE_UNDERLINE + '''''',
						        '''9''',
						        '''2''',
						        '''0''',
						        '''0''',
						        '''0''',
						        ''',''',
						        SPIECE_UNDERLINE + '''and''',
						        SPIECE_UNDERLINE + '''this''',
						        SPIECE_UNDERLINE + '''is''',
						        SPIECE_UNDERLINE + '''f''',
						        '''al''',
						        '''se''',
						        '''.''',
						    ]		,)



		@slow
		def 		lowerCAmelCase__				(self			)					->      Any:

						"""simple docstring"""






						lowerCAmelCase__					:    Optional[Any]												=					XLNetTokenizer.from_pretrained('''xlnet-base-cased'''			)

						lowerCAmelCase__					:    Any												=					tokenizer.encode('''sequence builders'''		,add_special_tokens=__lowerCamelCase			)
						lowerCAmelCase__					:    Any												=					tokenizer.encode('''multi-sequence build'''		,add_special_tokens=__lowerCamelCase			)

						lowerCAmelCase__					:    Optional[int]												=					tokenizer.build_inputs_with_special_tokens(__lowerCamelCase			)
						lowerCAmelCase__					:    List[Any]												=					tokenizer.build_inputs_with_special_tokens(__lowerCamelCase		,__lowerCamelCase			)

						assert encoded_sentence == text + [4, 3]
						assert encoded_pair == text + [4] + text_a + [4, 3]



		@slow
		def 		lowerCAmelCase__				(self			)					->      Union[str, Any]:

						"""simple docstring"""






						lowerCAmelCase__					:    Dict												=					{'''input_ids''': [[17, 2_14_42, 2_70, 17, 10, 1_46_45, 3_18, 34, 17, 45_46, 31_45, 7_87, 13, 77_52, 2_20_18, 23, 21, 17, 45_46, 31_45, 7_87, 13, 33_52, 1_44_31, 13, 55_00, 11, 11_76, 5_80, 13, 1_68_19, 47_97, 23, 17, 10, 1_71_35, 6_58, 19, 4_57, 79_32, 13, 1_84, 19, 31_54, 1_71_35, 64_68, 19, 14_04, 1_22_69, 19, 42_29, 53_56, 1_62_64, 46, 19, 17, 2_05_45, 1_03_95, 9, 9, 9, 11, 28, 64_21, 95_31, 2_07_29, 17, 10, 3_53, 1_70_22, 11, 21, 64_21, 95_31, 1_69_49, 17, 10, 1_15_09, 7_53, 11, 33, 95, 24_21, 73_85, 9_56, 1_44_31, 26_26, 25, 8_42, 73_85, 48_36, 21, 14_29, 22_72, 98_55, 31_20, 1_61, 2_47_38, 19, 1_32_03, 6_58, 2_18, 7_87, 21, 4_30, 1_84_82, 8_47, 26_37, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 3_22, 2_21_78, 27, 10_64, 22, 9_56, 13, 1_11_01, 14_29, 58_54, 2_43_13, 1_89_53, 40, 4_22, 2_43_66, 68, 17_58, 37, 1_04_83, 1_42_57, 31, 2_07, 2_63, 21, 2_03, 37_73, 25, 71, 97_35, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 20_49, 34_42, 17, 1_38_94, 33_80, 23, 95, 18, 1_76_34, 22_88, 9, 4, 3]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
						# fmt: on

						self.tokenizer_integration_test_util(
						    expected_encoding=__lowerCamelCase		,model_name='''xlnet-base-cased'''		,revision='''c841166438c31ec7ca9a106dee7bb312b73ae511'''		,)






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_torch_available,
)


__snake_case    :		Optional[int]  ={
    'configuration_vision_text_dual_encoder': ['VisionTextDualEncoderConfig'],
    'processing_vision_text_dual_encoder': ['VisionTextDualEncoderProcessor'],
}


try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__snake_case    :		int  =['VisionTextDualEncoderModel']


try:
							if not is_flax_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__snake_case    :		List[str]  =['FlaxVisionTextDualEncoderModel']

try:
							if not is_tf_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__snake_case    :		Tuple  =['TFVisionTextDualEncoderModel']


if TYPE_CHECKING:
							from .configuration_vision_text_dual_encoder import VisionTextDualEncoderConfig
							from .processing_vision_text_dual_encoder import VisionTextDualEncoderProcessor

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_vision_text_dual_encoder import VisionTextDualEncoderModel

							try:
														if not is_flax_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_flax_vision_text_dual_encoder import FlaxVisionTextDualEncoderModel

							try:
														if not is_tf_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_tf_vision_text_dual_encoder import TFVisionTextDualEncoderModel


else:
							import sys

							__snake_case    :		str  =_LazyModule(__name__, globals()['__file__'], _import_structure)

