




import unittest
from pathlib import Path
from shutil import copyfile

from transformers import SPIECE_UNDERLINE, is_sentencepiece_available
from transformers.models.speech_to_text import SpeechaTextTokenizer
from transformers.models.speech_to_text.tokenization_speech_to_text import VOCAB_FILES_NAMES, save_json
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


UpperCamelCase__     =	get_tests_dir("fixtures/test_sentencepiece.model")

if is_sentencepiece_available():
					import sentencepiece as sp


UpperCamelCase__     =	5
UpperCamelCase__     =	10

@require_sentencepiece
@require_tokenizers
class     __SCREAMING_SNAKE_CASE   (       _a				,						unittest.TestCase    ):
							snake_case       : str         =    SpeechaTextTokenizer
							snake_case       : Any         =    False
							snake_case       : Optional[int]         =    True






							def 			_lowerCamelCase						(  self					):
														super().setUp()

														UpperCamelCase__			       =     sp.SentencePieceProcessor()
														spm_model.Load(__lowerCAmelCase					)
														UpperCamelCase__			       =     ["""<s>""", """<pad>""", """</s>""", """<unk>"""]

														vocab += [spm_model.IdToPiece(id_					) for id_ in range(len(__lowerCAmelCase					)					)]
														UpperCamelCase__			       =     dict(zip(__lowerCAmelCase					,	range(len(__lowerCAmelCase					)					)					)					)

														UpperCamelCase__			       =     Path(self.tmpdirname					)
														save_json(__lowerCAmelCase					,	save_dir / VOCAB_FILES_NAMES["""vocab_file"""]					)
														if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
																					copyfile(__lowerCAmelCase					,	save_dir / VOCAB_FILES_NAMES["""spm_file"""]					)

														UpperCamelCase__			       =     SpeechaTextTokenizer.from_pretrained(self.tmpdirname					)
														tokenizer.save_pretrained(self.tmpdirname					)






							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     """<pad>"""
														UpperCamelCase__			       =     1

														self.assertEqual(self.get_tokenizer()._convert_token_to_id(__lowerCAmelCase					)					,	__lowerCAmelCase					)
														self.assertEqual(self.get_tokenizer()._convert_id_to_token(__lowerCAmelCase					)					,	__lowerCAmelCase					)






							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     list(self.get_tokenizer().get_vocab().keys()					)

														self.assertEqual(vocab_keys[0]					,	"""<s>"""					)
														self.assertEqual(vocab_keys[1]					,	"""<pad>"""					)
														self.assertEqual(vocab_keys[-1]					,	"""j"""					)
														self.assertEqual(len(__lowerCAmelCase					)					,	1001					)






							def 			_lowerCamelCase						(  self					):
														self.assertEqual(self.get_tokenizer().vocab_size					,	1001					)






							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     SpeechaTextTokenizer.from_pretrained(self.tmpdirname					)

														UpperCamelCase__			       =     tokenizer.tokenize("""This is a test"""					)
														self.assertListEqual(__lowerCAmelCase					,	["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]					)

														self.assertListEqual(
														    tokenizer.convert_tokens_to_ids(__lowerCAmelCase					)					,	[289, 50, 14, 174, 386]					,	)

														UpperCamelCase__			       =     tokenizer.tokenize("""I was born in 92000, and this is falsé."""					)
														self.assertListEqual(
														    __lowerCAmelCase					,	[SPIECE_UNDERLINE + """I""", SPIECE_UNDERLINE + """was""", SPIECE_UNDERLINE + """b""", """or""", """n""", SPIECE_UNDERLINE + """in""", SPIECE_UNDERLINE + """""", """9""", """2""", """0""", """0""", """0""", """,""", SPIECE_UNDERLINE + """and""", SPIECE_UNDERLINE + """this""", SPIECE_UNDERLINE + """is""", SPIECE_UNDERLINE + """f""", """al""", """s""", """é""", """."""]					,	)
														UpperCamelCase__			       =     tokenizer.convert_tokens_to_ids(__lowerCAmelCase					)
														self.assertListEqual(__lowerCAmelCase					,	[12, 25, 88, 59, 28, 23, 11, 4, 606, 351, 351, 351, 7, 16, 70, 50, 76, 84, 10, 4, 8]					)

														UpperCamelCase__			       =     tokenizer.convert_ids_to_tokens(__lowerCAmelCase					)
														self.assertListEqual(
														    __lowerCAmelCase					,	[SPIECE_UNDERLINE + """I""", SPIECE_UNDERLINE + """was""", SPIECE_UNDERLINE + """b""", """or""", """n""", SPIECE_UNDERLINE + """in""", SPIECE_UNDERLINE + """""", """<unk>""", """2""", """0""", """0""", """0""", """,""", SPIECE_UNDERLINE + """and""", SPIECE_UNDERLINE + """this""", SPIECE_UNDERLINE + """is""", SPIECE_UNDERLINE + """f""", """al""", """s""", """<unk>""", """."""]					,	)







							@slow
							def 			_lowerCamelCase						(  self					):
														# fmt: off
														UpperCamelCase__			       =     {"""input_ids""": [[3791, 797, 31, 11, 64, 797, 31, 2429, 433, 12, 1176, 12, 20, 786, 915, 142, 2413, 240, 37, 3238, 797, 31, 11, 35, 93, 915, 142, 2413, 240, 37, 5540, 567, 1276, 93, 37, 610, 40, 62, 455, 657, 1042, 123, 780, 177, 37, 309, 241, 1298, 514, 20, 292, 2737, 114, 2469, 241, 85, 64, 302, 548, 528, 423, 4, 509, 406, 423, 37, 601, 4, 777, 302, 548, 528, 423, 284, 4, 3388, 511, 459, 4, 3555, 40, 321, 302, 705, 4, 3388, 511, 583, 326, 5, 5, 5, 62, 3310, 560, 177, 2680, 217, 1508, 32, 31, 853, 418, 64, 583, 511, 1605, 62, 35, 93, 560, 177, 2680, 217, 1508, 1521, 64, 583, 511, 519, 62, 20, 1515, 764, 20, 149, 261, 5625, 7972, 20, 5540, 567, 1276, 93, 3925, 1675, 11, 15, 802, 7972, 576, 217, 1508, 11, 35, 93, 1253, 2441, 15, 289, 652, 31, 416, 321, 3842, 115, 40, 911, 8, 476, 619, 4, 380, 142, 423, 335, 240, 35, 93, 264, 8, 11, 335, 569, 420, 163, 5, 2], [260, 548, 528, 423, 20, 451, 20, 2681, 1153, 3434, 20, 5540, 37, 567, 126, 1253, 2441, 3376, 449, 210, 431, 1563, 177, 767, 5540, 11, 1203, 472, 11, 2953, 685, 285, 364, 706, 1153, 20, 6799, 20, 2869, 20, 4464, 126, 40, 2429, 20, 1040, 866, 2664, 418, 20, 318, 20, 1726, 186, 20, 265, 522, 35, 93, 2191, 4634, 20, 1040, 12, 6799, 15, 228, 2356, 142, 31, 11, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [2575, 2666, 684, 1582, 1176, 12, 627, 149, 619, 20, 4902, 563, 11, 20, 149, 261, 3420, 2356, 174, 142, 4714, 131, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
														# fmt: on

														self.tokenizer_integration_test_util(
														    expected_encoding=__lowerCAmelCase					,	model_name="""facebook/s2t-small-mustc-en-de-st"""					,	revision="""a14f04cf0776c02f62a8cb800cf7909e15ea23ad"""					,	)






@require_sentencepiece
class     __SCREAMING_SNAKE_CASE   (       unittest.TestCase    ):
							snake_case       : Any         =    """valhalla/s2t_mustc_multilinguial_medium"""

							snake_case       : List[Any]         =    """C'est trop cool"""
							snake_case       : Optional[Any]         =    """Esto es genial"""






							@classmethod
							def 			_lowerCamelCase						(  cls					):
														UpperCamelCase__			       =     SpeechaTextTokenizer.from_pretrained(cls.checkpoint_name					)
														return cls






							def 			_lowerCamelCase						(  self					):
														self.assertEqual(self.tokenizer.lang_code_to_id["""pt"""]					,	4					)
														self.assertEqual(self.tokenizer.lang_code_to_id["""ru"""]					,	6					)
														self.assertEqual(self.tokenizer.lang_code_to_id["""it"""]					,	9					)
														self.assertEqual(self.tokenizer.lang_code_to_id["""de"""]					,	11					)






							def 			_lowerCamelCase						(  self					):
														self.assertEqual(self.tokenizer.vocab_size					,	10000					)






							def 			_lowerCamelCase						(  self					):
														self.assertIn(__lowerCAmelCase					,	self.tokenizer.all_special_ids					)
														UpperCamelCase__			       =     [ES_CODE, 4, 1601, 47, 7647, 2]
														UpperCamelCase__			       =     self.tokenizer.decode(__lowerCAmelCase					,	skip_special_tokens=__lowerCAmelCase					)
														UpperCamelCase__			       =     self.tokenizer.decode(generated_ids[1:]					,	skip_special_tokens=__lowerCAmelCase					)
														self.assertEqual(__lowerCAmelCase					,	__lowerCAmelCase					)
														self.assertNotIn(self.tokenizer.eos_token					,	__lowerCAmelCase					)






							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     """fr"""
														UpperCamelCase__			       =     self.tokenizer(self.french_text					).input_ids
														self.assertEqual(encoded[0]					,	__lowerCAmelCase					)
														self.assertEqual(encoded[-1]					,	self.tokenizer.eos_token_id					)







							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     """fr"""
														self.assertListEqual(self.tokenizer.prefix_tokens					,	[FR_CODE]					)

														UpperCamelCase__			       =     """es"""
														self.assertListEqual(self.tokenizer.prefix_tokens					,	[ES_CODE]					)











UpperCamelCase__     =	{
    "meter": "m",
    "kilometer": "km",
    "megametre": "Mm",
    "gigametre": "Gm",
    "terametre": "Tm",
    "petametre": "Pm",
    "exametre": "Em",
    "zettametre": "Zm",
    "yottametre": "Ym",
}
# Exponent of the factor(meter)
UpperCamelCase__     =	{
    "m": 0,
    "km": 3,
    "Mm": 6,
    "Gm": 9,
    "Tm": 12,
    "Pm": 15,
    "Em": 18,
    "Zm": 21,
    "Ym": 24,
}





def       _UpperCamelCase       (a__				:float				,     a__				:str				,     a__				:str    ):






							"""simple docstring"""





							UpperCamelCase__			       =     from_type.lower().strip("""s"""    )
							UpperCamelCase__			       =     to_type.lower().strip("""s"""    )

							UpperCamelCase__			       =     UNIT_SYMBOL.get(a__				,     a__    )
							UpperCamelCase__			       =     UNIT_SYMBOL.get(a__				,     a__    )

							if from_sanitized not in METRIC_CONVERSION:
														UpperCamelCase__			       =     (
														    f"""Invalid 'from_type' value: {from_type!r}.\n"""
														    f"""Conversion abbreviations are: {", ".join(a__    )}"""
														)
														raise ValueError(a__    )
							if to_sanitized not in METRIC_CONVERSION:
														UpperCamelCase__			       =     (
														    f"""Invalid 'to_type' value: {to_type!r}.\n"""
														    f"""Conversion abbreviations are: {", ".join(a__    )}"""
														)
														raise ValueError(a__    )
							UpperCamelCase__			       =     METRIC_CONVERSION[from_sanitized]
							UpperCamelCase__			       =     METRIC_CONVERSION[to_sanitized]
							UpperCamelCase__			       =     1

							if from_exponent > to_exponent:
														UpperCamelCase__			       =     from_exponent - to_exponent
							else:
														UpperCamelCase__			       =     -(to_exponent - from_exponent)

							return value * pow(10				,     a__    )


if __name__ == "__main__":
					from doctest import testmod

					testmod()




