






import unittest
from pathlib import Path
from shutil import copyfile

from transformers import SPIECE_UNDERLINE, is_sentencepiece_available
from transformers.models.speech_to_text import SpeechaTextTokenizer
from transformers.models.speech_to_text.tokenization_speech_to_text import VOCAB_FILES_NAMES, save_json
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__magic_name__:					Dict							=     get_tests_dir("fixtures/test_sentencepiece.model")

if is_sentencepiece_available():
		import sentencepiece as sp


__magic_name__:					Optional[int]							=     5
__magic_name__:					List[str]							=     10





@require_sentencepiece
@require_tokenizers
class 	snake_case__ (							_lowerCAmelCase						,  unittest.TestCase				):
	lowercase__						:					Tuple											=   SpeechaTextTokenizer
	lowercase__						:					Optional[int]											=   False
	lowercase__						:					List[str]											=   True
	def       __magic_name__    (    self			)							->				List[str]:
							super().setUp()

							__magic_name__      :       Optional[Any]       						= sp.SentencePieceProcessor()
							spm_model.Load(lowerCAmelCase__			)
							__magic_name__      :       List[str]       						= ["""<s>""", """<pad>""", """</s>""", """<unk>"""]

							vocab += [spm_model.IdToPiece(id_			) for id_ in range(len(lowerCAmelCase__			)			)]
							__magic_name__      :       Optional[Any]       						= dict(zip(lowerCAmelCase__      ,       range(len(lowerCAmelCase__			)			)			)			)

							__magic_name__      :       int       						= Path(self.tmpdirname			)
							save_json(lowerCAmelCase__      ,       save_dir / VOCAB_FILES_NAMES["""vocab_file"""]			)
							if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
													copyfile(lowerCAmelCase__      ,       save_dir / VOCAB_FILES_NAMES["""spm_file"""]			)

							__magic_name__      :       List[Any]       						= SpeechaTextTokenizer.from_pretrained(self.tmpdirname			)
							tokenizer.save_pretrained(self.tmpdirname			)
	def       __magic_name__    (    self			)							->				List[str]:
							__magic_name__      :       Union[str, Any]       						= """<pad>"""
							__magic_name__      :       Any       						= 1

							self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCAmelCase__			)      ,       lowerCAmelCase__			)
							self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCAmelCase__			)      ,       lowerCAmelCase__			)
	def       __magic_name__    (    self			)							->				Optional[Any]:
							__magic_name__      :       Dict       						= list(self.get_tokenizer().get_vocab().keys()			)

							self.assertEqual(vocab_keys[0]      ,       """<s>"""			)
							self.assertEqual(vocab_keys[1]      ,       """<pad>"""			)
							self.assertEqual(vocab_keys[-1]      ,       """j"""			)
							self.assertEqual(len(lowerCAmelCase__			)      ,       10_01			)
	def       __magic_name__    (    self			)							->				Union[str, Any]:
							self.assertEqual(self.get_tokenizer().vocab_size      ,       10_01			)
	def       __magic_name__    (    self			)							->				Optional[int]:
							__magic_name__      :       Any       						= SpeechaTextTokenizer.from_pretrained(self.tmpdirname			)

							__magic_name__      :       str       						= tokenizer.tokenize("""This is a test"""			)
							self.assertListEqual(lowerCAmelCase__      ,       ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]			)

							self.assertListEqual(
							    tokenizer.convert_tokens_to_ids(lowerCAmelCase__			)      ,       [2_89, 50, 14, 1_74, 3_86]      ,       )

							__magic_name__      :       Tuple       						= tokenizer.tokenize("""I was born in 92000, and this is falsé."""			)
							self.assertListEqual(
							    lowerCAmelCase__      ,       [SPIECE_UNDERLINE + """I""", SPIECE_UNDERLINE + """was""", SPIECE_UNDERLINE + """b""", """or""", """n""", SPIECE_UNDERLINE + """in""", SPIECE_UNDERLINE + """""", """9""", """2""", """0""", """0""", """0""", """,""", SPIECE_UNDERLINE + """and""", SPIECE_UNDERLINE + """this""", SPIECE_UNDERLINE + """is""", SPIECE_UNDERLINE + """f""", """al""", """s""", """é""", """."""]      ,       )
							__magic_name__      :       List[Any]       						= tokenizer.convert_tokens_to_ids(lowerCAmelCase__			)
							self.assertListEqual(lowerCAmelCase__      ,       [12, 25, 88, 59, 28, 23, 11, 4, 6_06, 3_51, 3_51, 3_51, 7, 16, 70, 50, 76, 84, 10, 4, 8]			)

							__magic_name__      :       Any       						= tokenizer.convert_ids_to_tokens(lowerCAmelCase__			)
							self.assertListEqual(
							    lowerCAmelCase__      ,       [SPIECE_UNDERLINE + """I""", SPIECE_UNDERLINE + """was""", SPIECE_UNDERLINE + """b""", """or""", """n""", SPIECE_UNDERLINE + """in""", SPIECE_UNDERLINE + """""", """<unk>""", """2""", """0""", """0""", """0""", """,""", SPIECE_UNDERLINE + """and""", SPIECE_UNDERLINE + """this""", SPIECE_UNDERLINE + """is""", SPIECE_UNDERLINE + """f""", """al""", """s""", """<unk>""", """."""]      ,       )







	@slow
	def       __magic_name__    (    self			)							->				Optional[int]:
							# fmt: off
							__magic_name__      :       Any       						= {"""input_ids""": [[37_91, 7_97, 31, 11, 64, 7_97, 31, 24_29, 4_33, 12, 11_76, 12, 20, 7_86, 9_15, 1_42, 24_13, 2_40, 37, 32_38, 7_97, 31, 11, 35, 93, 9_15, 1_42, 24_13, 2_40, 37, 55_40, 5_67, 12_76, 93, 37, 6_10, 40, 62, 4_55, 6_57, 10_42, 1_23, 7_80, 1_77, 37, 3_09, 2_41, 12_98, 5_14, 20, 2_92, 27_37, 1_14, 24_69, 2_41, 85, 64, 3_02, 5_48, 5_28, 4_23, 4, 5_09, 4_06, 4_23, 37, 6_01, 4, 7_77, 3_02, 5_48, 5_28, 4_23, 2_84, 4, 33_88, 5_11, 4_59, 4, 35_55, 40, 3_21, 3_02, 7_05, 4, 33_88, 5_11, 5_83, 3_26, 5, 5, 5, 62, 33_10, 5_60, 1_77, 26_80, 2_17, 15_08, 32, 31, 8_53, 4_18, 64, 5_83, 5_11, 16_05, 62, 35, 93, 5_60, 1_77, 26_80, 2_17, 15_08, 15_21, 64, 5_83, 5_11, 5_19, 62, 20, 15_15, 7_64, 20, 1_49, 2_61, 56_25, 79_72, 20, 55_40, 5_67, 12_76, 93, 39_25, 16_75, 11, 15, 8_02, 79_72, 5_76, 2_17, 15_08, 11, 35, 93, 12_53, 24_41, 15, 2_89, 6_52, 31, 4_16, 3_21, 38_42, 1_15, 40, 9_11, 8, 4_76, 6_19, 4, 3_80, 1_42, 4_23, 3_35, 2_40, 35, 93, 2_64, 8, 11, 3_35, 5_69, 4_20, 1_63, 5, 2], [2_60, 5_48, 5_28, 4_23, 20, 4_51, 20, 26_81, 11_53, 34_34, 20, 55_40, 37, 5_67, 1_26, 12_53, 24_41, 33_76, 4_49, 2_10, 4_31, 15_63, 1_77, 7_67, 55_40, 11, 12_03, 4_72, 11, 29_53, 6_85, 2_85, 3_64, 7_06, 11_53, 20, 67_99, 20, 28_69, 20, 44_64, 1_26, 40, 24_29, 20, 10_40, 8_66, 26_64, 4_18, 20, 3_18, 20, 17_26, 1_86, 20, 2_65, 5_22, 35, 93, 21_91, 46_34, 20, 10_40, 12, 67_99, 15, 2_28, 23_56, 1_42, 31, 11, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [25_75, 26_66, 6_84, 15_82, 11_76, 12, 6_27, 1_49, 6_19, 20, 49_02, 5_63, 11, 20, 1_49, 2_61, 34_20, 23_56, 1_74, 1_42, 47_14, 1_31, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
							# fmt: on

							self.tokenizer_integration_test_util(
							    expected_encoding=lowerCAmelCase__      ,       model_name="""facebook/s2t-small-mustc-en-de-st"""      ,       revision="""a14f04cf0776c02f62a8cb800cf7909e15ea23ad"""      ,       )





@require_sentencepiece
class 	snake_case__ (							unittest.TestCase				):
	lowercase__						:					str											=   '''valhalla/s2t_mustc_multilinguial_medium'''

	lowercase__						:					Any											=   '''C\'est trop cool'''
	lowercase__						:					Dict											=   '''Esto es genial'''
	@classmethod
	def       __magic_name__    (    cls			)							->				Tuple:
							__magic_name__      :       SpeechaTextTokenizer       						= SpeechaTextTokenizer.from_pretrained(cls.checkpoint_name			)
							return cls
	def       __magic_name__    (    self			)							->				Optional[int]:
							self.assertEqual(self.tokenizer.lang_code_to_id["""pt"""]      ,       4			)
							self.assertEqual(self.tokenizer.lang_code_to_id["""ru"""]      ,       6			)
							self.assertEqual(self.tokenizer.lang_code_to_id["""it"""]      ,       9			)
							self.assertEqual(self.tokenizer.lang_code_to_id["""de"""]      ,       11			)
	def       __magic_name__    (    self			)							->				int:
							self.assertEqual(self.tokenizer.vocab_size      ,       1_00_00			)
	def       __magic_name__    (    self			)							->				Optional[int]:
							self.assertIn(lowerCAmelCase__      ,       self.tokenizer.all_special_ids			)
							__magic_name__      :       Optional[Any]       						= [ES_CODE, 4, 16_01, 47, 76_47, 2]
							__magic_name__      :       Optional[Any]       						= self.tokenizer.decode(lowerCAmelCase__      ,       skip_special_tokens=lowerCAmelCase__			)
							__magic_name__      :       Any       						= self.tokenizer.decode(generated_ids[1:]      ,       skip_special_tokens=lowerCAmelCase__			)
							self.assertEqual(lowerCAmelCase__      ,       lowerCAmelCase__			)
							self.assertNotIn(self.tokenizer.eos_token      ,       lowerCAmelCase__			)
	def       __magic_name__    (    self			)							->				List[Any]:
							__magic_name__      :       Union[str, Any]       						= """fr"""
							__magic_name__      :       List[str]       						= self.tokenizer(self.french_text			).input_ids
							self.assertEqual(encoded[0]      ,       lowerCAmelCase__			)
							self.assertEqual(encoded[-1]      ,       self.tokenizer.eos_token_id			)







	def       __magic_name__    (    self			)							->				Dict:
							__magic_name__      :       Union[str, Any]       						= """fr"""
							self.assertListEqual(self.tokenizer.prefix_tokens      ,       [FR_CODE]			)

							__magic_name__      :       Tuple       						= """es"""
							self.assertListEqual(self.tokenizer.prefix_tokens      ,       [ES_CODE]			)








__magic_name__:					List[Any]							=     "\n# Transformers 설치 방법\n! pip install transformers datasets\n# 마지막 릴리스 대신 소스에서 설치하려면, 위 명령을 주석으로 바꾸고 아래 명령을 해제하세요.\n# ! pip install git+https://github.com/huggingface/transformers.git\n"

__magic_name__:					Dict							=     [{"type": "code", "content": INSTALL_CONTENT}]
__magic_name__:					Optional[Any]							=     {
    "{processor_class}": "FakeProcessorClass",
    "{model_class}": "FakeModelClass",
    "{object_class}": "FakeObjectClass",
}
