





"""simple docstring"""




import tempfile

import torch

from diffusers import (
    DEISMultistepScheduler,
    DPMSolverMultistepScheduler,
    DPMSolverSinglestepScheduler,
    UniPCMultistepScheduler,
)

from .test_schedulers import SchedulerCommonTest




class 			_a						(				_lowerCAmelCase    ):
				UpperCamelCase								=   (DEISMultistepScheduler,)
				UpperCamelCase								=   (('''num_inference_steps''', 25),)






				def     snake_case     (			self		:   str,			**lowerCAmelCase__		:   Any							)     ->				int:




										'''simple docstring'''



										_UpperCamelCase      : List[Any]         =   {
										    '''num_train_timesteps''': 1_0_0_0,
										    '''beta_start''': 0.0_001,
										    '''beta_end''': 0.02,
										    '''beta_schedule''': '''linear''',
										    '''solver_order''': 2,
										}

										config.update(**lowerCAmelCase__							)
										return config






				def     snake_case     (			self		:   str,			lowerCAmelCase__		:   int=0,			**lowerCAmelCase__		:   List[str]							)     ->				Tuple:




										'''simple docstring'''



										_UpperCamelCase      : Dict         =   dict(self.forward_default_kwargs							)
										_UpperCamelCase      : Tuple         =   kwargs.pop('''num_inference_steps''',			lowerCAmelCase__							)
										_UpperCamelCase      : Optional[Any]         =   self.dummy_sample
										_UpperCamelCase      : str         =   0.1 * sample
										_UpperCamelCase      : str         =   [residual + 0.2, residual + 0.15, residual + 0.10]

										for scheduler_class in self.scheduler_classes:
																_UpperCamelCase      : Dict         =   self.get_scheduler_config(**lowerCAmelCase__							)
																_UpperCamelCase      : Dict         =   scheduler_class(**lowerCAmelCase__							)
																scheduler.set_timesteps(lowerCAmelCase__							)
																# copy over dummy past residuals
																_UpperCamelCase      : int         =   dummy_past_residuals[: scheduler.config.solver_order]

																with tempfile.TemporaryDirectory() as tmpdirname:
																						scheduler.save_config(lowerCAmelCase__							)
																						_UpperCamelCase      : Tuple         =   scheduler_class.from_pretrained(lowerCAmelCase__							)
																						new_scheduler.set_timesteps(lowerCAmelCase__							)
																						# copy over dummy past residuals
																						_UpperCamelCase      : int         =   dummy_past_residuals[: new_scheduler.config.solver_order]

																_UpperCamelCase       ,							_UpperCamelCase      : Optional[int]         =   sample, sample
																for t in range(lowerCAmelCase__,			time_step + scheduler.config.solver_order + 1							):
																						_UpperCamelCase      : Any         =   scheduler.step(lowerCAmelCase__,			lowerCAmelCase__,			lowerCAmelCase__,			**lowerCAmelCase__							).prev_sample
																						_UpperCamelCase      : Tuple         =   new_scheduler.step(lowerCAmelCase__,			lowerCAmelCase__,			lowerCAmelCase__,			**lowerCAmelCase__							).prev_sample

																						assert torch.sum(torch.abs(output - new_output							)							) < 1e-5, "Scheduler outputs are not identical"






				def     snake_case     (			self		:   List[Any]							)     ->				Any:




										'''simple docstring'''



										pass






				def     snake_case     (			self		:   List[Any],			lowerCAmelCase__		:   Union[str, Any]=0,			**lowerCAmelCase__		:   Optional[Any]							)     ->				Optional[int]:




										'''simple docstring'''



										_UpperCamelCase      : Optional[Any]         =   dict(self.forward_default_kwargs							)
										_UpperCamelCase      : Union[str, Any]         =   kwargs.pop('''num_inference_steps''',			lowerCAmelCase__							)
										_UpperCamelCase      : Optional[int]         =   self.dummy_sample
										_UpperCamelCase      : Optional[Any]         =   0.1 * sample
										_UpperCamelCase      : Optional[Any]         =   [residual + 0.2, residual + 0.15, residual + 0.10]

										for scheduler_class in self.scheduler_classes:
																_UpperCamelCase      : int         =   self.get_scheduler_config()
																_UpperCamelCase      : Optional[int]         =   scheduler_class(**lowerCAmelCase__							)
																scheduler.set_timesteps(lowerCAmelCase__							)

																# copy over dummy past residuals (must be after setting timesteps)
																_UpperCamelCase      : List[Any]         =   dummy_past_residuals[: scheduler.config.solver_order]

																with tempfile.TemporaryDirectory() as tmpdirname:
																						scheduler.save_config(lowerCAmelCase__							)
																						_UpperCamelCase      : List[str]         =   scheduler_class.from_pretrained(lowerCAmelCase__							)
																						# copy over dummy past residuals
																						new_scheduler.set_timesteps(lowerCAmelCase__							)

																						# copy over dummy past residual (must be after setting timesteps)
																						_UpperCamelCase      : Optional[Any]         =   dummy_past_residuals[: new_scheduler.config.solver_order]

																_UpperCamelCase      : Any         =   scheduler.step(lowerCAmelCase__,			lowerCAmelCase__,			lowerCAmelCase__,			**lowerCAmelCase__							).prev_sample
																_UpperCamelCase      : Tuple         =   new_scheduler.step(lowerCAmelCase__,			lowerCAmelCase__,			lowerCAmelCase__,			**lowerCAmelCase__							).prev_sample

																assert torch.sum(torch.abs(output - new_output							)							) < 1e-5, "Scheduler outputs are not identical"






				def     snake_case     (			self		:   Optional[Any],			lowerCAmelCase__		:   Optional[Any]=None,			**lowerCAmelCase__		:   Union[str, Any]							)     ->				Optional[Any]:




										'''simple docstring'''



										if scheduler is None:
																_UpperCamelCase      : Dict         =   self.scheduler_classes[0]
																_UpperCamelCase      : Optional[int]         =   self.get_scheduler_config(**lowerCAmelCase__							)
																_UpperCamelCase      : Dict         =   scheduler_class(**lowerCAmelCase__							)

										_UpperCamelCase      : int         =   self.scheduler_classes[0]
										_UpperCamelCase      : Tuple         =   self.get_scheduler_config(**lowerCAmelCase__							)
										_UpperCamelCase      : Optional[Any]         =   scheduler_class(**lowerCAmelCase__							)

										_UpperCamelCase      : List[str]         =   1_0
										_UpperCamelCase      : Union[str, Any]         =   self.dummy_model()
										_UpperCamelCase      : List[str]         =   self.dummy_sample_deter
										scheduler.set_timesteps(lowerCAmelCase__							)

										for i, t in enumerate(scheduler.timesteps							):
																_UpperCamelCase      : List[Any]         =   model(lowerCAmelCase__,			lowerCAmelCase__							)
																_UpperCamelCase      : Dict         =   scheduler.step(lowerCAmelCase__,			lowerCAmelCase__,			lowerCAmelCase__							).prev_sample

										return sample






				def     snake_case     (			self		:   List[str]							)     ->				List[str]:




										'''simple docstring'''



										_UpperCamelCase      : Union[str, Any]         =   dict(self.forward_default_kwargs							)

										_UpperCamelCase      : Dict         =   kwargs.pop('''num_inference_steps''',			lowerCAmelCase__							)

										for scheduler_class in self.scheduler_classes:
																_UpperCamelCase      : Optional[int]         =   self.get_scheduler_config()
																_UpperCamelCase      : Any         =   scheduler_class(**lowerCAmelCase__							)

																_UpperCamelCase      : List[str]         =   self.dummy_sample
																_UpperCamelCase      : List[str]         =   0.1 * sample

																if num_inference_steps is not None and hasattr(lowerCAmelCase__,			'''set_timesteps'''							):
																						scheduler.set_timesteps(lowerCAmelCase__							)
																elif num_inference_steps is not None and not hasattr(lowerCAmelCase__,			'''set_timesteps'''							):
																						_UpperCamelCase      : Optional[int]         =   num_inference_steps

																# copy over dummy past residuals (must be done after set_timesteps)
																_UpperCamelCase      : Union[str, Any]         =   [residual + 0.2, residual + 0.15, residual + 0.10]
																_UpperCamelCase      : Tuple         =   dummy_past_residuals[: scheduler.config.solver_order]

																_UpperCamelCase      : Optional[Any]         =   scheduler.timesteps[5]
																_UpperCamelCase      : Union[str, Any]         =   scheduler.timesteps[6]

																_UpperCamelCase      : int         =   scheduler.step(lowerCAmelCase__,			lowerCAmelCase__,			lowerCAmelCase__,			**lowerCAmelCase__							).prev_sample
																_UpperCamelCase      : List[Any]         =   scheduler.step(lowerCAmelCase__,			lowerCAmelCase__,			lowerCAmelCase__,			**lowerCAmelCase__							).prev_sample

																self.assertEqual(output_a.shape,			sample.shape							)
																self.assertEqual(output_a.shape,			output_a.shape							)






				def     snake_case     (			self		:   Optional[int]							)     ->				List[Any]:




										'''simple docstring'''



										_UpperCamelCase      : Dict         =   DEISMultistepScheduler(**self.get_scheduler_config()							)
										_UpperCamelCase      : Dict         =   self.full_loop(scheduler=lowerCAmelCase__							)
										_UpperCamelCase      : Optional[Any]         =   torch.mean(torch.abs(lowerCAmelCase__							)							)

										assert abs(result_mean.item() - 0.23_916							) < 1e-3

										_UpperCamelCase      : int         =   DPMSolverSinglestepScheduler.from_config(scheduler.config							)
										_UpperCamelCase      : int         =   DPMSolverMultistepScheduler.from_config(scheduler.config							)
										_UpperCamelCase      : List[Any]         =   UniPCMultistepScheduler.from_config(scheduler.config							)
										_UpperCamelCase      : Union[str, Any]         =   DEISMultistepScheduler.from_config(scheduler.config							)

										_UpperCamelCase      : List[str]         =   self.full_loop(scheduler=lowerCAmelCase__							)
										_UpperCamelCase      : str         =   torch.mean(torch.abs(lowerCAmelCase__							)							)

										assert abs(result_mean.item() - 0.23_916							) < 1e-3






				def     snake_case     (			self		:   int							)     ->				List[Any]:




										'''simple docstring'''



										for timesteps in [2_5, 5_0, 1_0_0, 9_9_9, 1_0_0_0]:
																self.check_over_configs(num_train_timesteps=lowerCAmelCase__							)






				def     snake_case     (			self		:   List[Any]							)     ->				Optional[int]:




										'''simple docstring'''



										self.check_over_configs(thresholding=lowerCAmelCase__							)
										for order in [1, 2, 3]:
																for solver_type in ["logrho"]:
																						for threshold in [0.5, 1.0, 2.0]:
																												for prediction_type in ["epsilon", "sample"]:
																																		self.check_over_configs(
																																		    thresholding=lowerCAmelCase__,			prediction_type=lowerCAmelCase__,			sample_max_value=lowerCAmelCase__,			algorithm_type='''deis''',			solver_order=lowerCAmelCase__,			solver_type=lowerCAmelCase__,			)






				def     snake_case     (			self		:   int							)     ->				Dict:




										'''simple docstring'''



										for prediction_type in ["epsilon", "v_prediction"]:
																self.check_over_configs(prediction_type=lowerCAmelCase__							)






				def     snake_case     (			self		:   Optional[Any]							)     ->				Dict:




										'''simple docstring'''



										for algorithm_type in ["deis"]:
																for solver_type in ["logrho"]:
																						for order in [1, 2, 3]:
																												for prediction_type in ["epsilon", "sample"]:
																																		self.check_over_configs(
																																		    solver_order=lowerCAmelCase__,			solver_type=lowerCAmelCase__,			prediction_type=lowerCAmelCase__,			algorithm_type=lowerCAmelCase__,			)
																																		_UpperCamelCase      : List[Any]         =   self.full_loop(
																																		    solver_order=lowerCAmelCase__,			solver_type=lowerCAmelCase__,			prediction_type=lowerCAmelCase__,			algorithm_type=lowerCAmelCase__,			)
																																		assert not torch.isnan(lowerCAmelCase__							).any(), "Samples have nan numbers"






				def     snake_case     (			self		:   List[Any]							)     ->				Optional[Any]:




										'''simple docstring'''



										self.check_over_configs(lower_order_final=lowerCAmelCase__							)
										self.check_over_configs(lower_order_final=lowerCAmelCase__							)






				def     snake_case     (			self		:   str							)     ->				Any:




										'''simple docstring'''



										for num_inference_steps in [1, 2, 3, 5, 1_0, 5_0, 1_0_0, 9_9_9, 1_0_0_0]:
																self.check_over_forward(num_inference_steps=lowerCAmelCase__,			time_step=0							)






				def     snake_case     (			self		:   Dict							)     ->				Union[str, Any]:




										'''simple docstring'''



										_UpperCamelCase      : int         =   self.full_loop()
										_UpperCamelCase      : int         =   torch.mean(torch.abs(lowerCAmelCase__							)							)

										assert abs(result_mean.item() - 0.23_916							) < 1e-3






				def     snake_case     (			self		:   Dict							)     ->				int:




										'''simple docstring'''



										_UpperCamelCase      : Any         =   self.full_loop(prediction_type='''v_prediction'''							)
										_UpperCamelCase      : List[str]         =   torch.mean(torch.abs(lowerCAmelCase__							)							)

										assert abs(result_mean.item() - 0.091							) < 1e-3






				def     snake_case     (			self		:   Any							)     ->				Dict:




										'''simple docstring'''



										_UpperCamelCase      : Any         =   self.scheduler_classes[0]
										_UpperCamelCase      : Any         =   self.get_scheduler_config(thresholding=lowerCAmelCase__,			dynamic_thresholding_ratio=0							)
										_UpperCamelCase      : List[Any]         =   scheduler_class(**lowerCAmelCase__							)

										_UpperCamelCase      : Optional[Any]         =   1_0
										_UpperCamelCase      : Optional[Any]         =   self.dummy_model()
										_UpperCamelCase      : int         =   self.dummy_sample_deter.half()
										scheduler.set_timesteps(lowerCAmelCase__							)

										for i, t in enumerate(scheduler.timesteps							):
																_UpperCamelCase      : Dict         =   model(lowerCAmelCase__,			lowerCAmelCase__							)
																_UpperCamelCase      : int         =   scheduler.step(lowerCAmelCase__,			lowerCAmelCase__,			lowerCAmelCase__							).prev_sample

										assert sample.dtype == torch.floataa










"""simple docstring"""




import unittest
from pathlib import Path
from shutil import copyfile

from transformers import SPIECE_UNDERLINE, is_sentencepiece_available
from transformers.models.speech_to_text import SpeechaTextTokenizer
from transformers.models.speech_to_text.tokenization_speech_to_text import VOCAB_FILES_NAMES, save_json
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


UpperCamelCase_									=get_tests_dir("""fixtures/test_sentencepiece.model""")

if is_sentencepiece_available():
						import sentencepiece as sp


UpperCamelCase_									=5
UpperCamelCase_									=10




@require_sentencepiece
@require_tokenizers
class 			_a						(				_lowerCAmelCase     ,    unittest.TestCase    ):
				UpperCamelCase								=   SpeechaTextTokenizer
				UpperCamelCase								=   False
				UpperCamelCase								=   True






				def     snake_case     (			self		:   Tuple							)     ->				Optional[Any]:




										'''simple docstring'''



										super().setUp()

										_UpperCamelCase      : Dict         =   sp.SentencePieceProcessor()
										spm_model.Load(lowerCAmelCase__							)
										_UpperCamelCase      : Union[str, Any]         =   ['''<s>''', '''<pad>''', '''</s>''', '''<unk>''']

										vocab += [spm_model.IdToPiece(id_							) for id_ in range(len(lowerCAmelCase__							)							)]
										_UpperCamelCase      : Tuple         =   dict(zip(lowerCAmelCase__,			range(len(lowerCAmelCase__							)							)							)							)

										_UpperCamelCase      : Dict         =   Path(self.tmpdirname							)
										save_json(lowerCAmelCase__,			save_dir / VOCAB_FILES_NAMES['''vocab_file''']							)
										if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
																copyfile(lowerCAmelCase__,			save_dir / VOCAB_FILES_NAMES['''spm_file''']							)

										_UpperCamelCase      : List[str]         =   SpeechaTextTokenizer.from_pretrained(self.tmpdirname							)
										tokenizer.save_pretrained(self.tmpdirname							)






				def     snake_case     (			self		:   str							)     ->				str:




										'''simple docstring'''



										_UpperCamelCase      : List[str]         =   '''<pad>'''
										_UpperCamelCase      : List[Any]         =   1

										self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCAmelCase__							),			lowerCAmelCase__							)
										self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCAmelCase__							),			lowerCAmelCase__							)






				def     snake_case     (			self		:   Any							)     ->				Optional[int]:




										'''simple docstring'''



										_UpperCamelCase      : int         =   list(self.get_tokenizer().get_vocab().keys()							)

										self.assertEqual(vocab_keys[0],			'''<s>'''							)
										self.assertEqual(vocab_keys[1],			'''<pad>'''							)
										self.assertEqual(vocab_keys[-1],			'''j'''							)
										self.assertEqual(len(lowerCAmelCase__							),			1_0_0_1							)






				def     snake_case     (			self		:   Any							)     ->				Dict:




										'''simple docstring'''



										self.assertEqual(self.get_tokenizer().vocab_size,			1_0_0_1							)






				def     snake_case     (			self		:   List[Any]							)     ->				Tuple:




										'''simple docstring'''



										_UpperCamelCase      : Optional[int]         =   SpeechaTextTokenizer.from_pretrained(self.tmpdirname							)

										_UpperCamelCase      : List[str]         =   tokenizer.tokenize('''This is a test'''							)
										self.assertListEqual(lowerCAmelCase__,			['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']							)

										self.assertListEqual(
										    tokenizer.convert_tokens_to_ids(lowerCAmelCase__							),			[2_8_9, 5_0, 1_4, 1_7_4, 3_8_6],			)

										_UpperCamelCase      : List[str]         =   tokenizer.tokenize('''I was born in 92000, and this is falsé.'''							)
										self.assertListEqual(
										    lowerCAmelCase__,			[SPIECE_UNDERLINE + '''I''', SPIECE_UNDERLINE + '''was''', SPIECE_UNDERLINE + '''b''', '''or''', '''n''', SPIECE_UNDERLINE + '''in''', SPIECE_UNDERLINE + '''''', '''9''', '''2''', '''0''', '''0''', '''0''', ''',''', SPIECE_UNDERLINE + '''and''', SPIECE_UNDERLINE + '''this''', SPIECE_UNDERLINE + '''is''', SPIECE_UNDERLINE + '''f''', '''al''', '''s''', '''é''', '''.'''],			)
										_UpperCamelCase      : Optional[Any]         =   tokenizer.convert_tokens_to_ids(lowerCAmelCase__							)
										self.assertListEqual(lowerCAmelCase__,			[1_2, 2_5, 8_8, 5_9, 2_8, 2_3, 1_1, 4, 6_0_6, 3_5_1, 3_5_1, 3_5_1, 7, 1_6, 7_0, 5_0, 7_6, 8_4, 1_0, 4, 8]							)

										_UpperCamelCase      : Union[str, Any]         =   tokenizer.convert_ids_to_tokens(lowerCAmelCase__							)
										self.assertListEqual(
										    lowerCAmelCase__,			[SPIECE_UNDERLINE + '''I''', SPIECE_UNDERLINE + '''was''', SPIECE_UNDERLINE + '''b''', '''or''', '''n''', SPIECE_UNDERLINE + '''in''', SPIECE_UNDERLINE + '''''', '''<unk>''', '''2''', '''0''', '''0''', '''0''', ''',''', SPIECE_UNDERLINE + '''and''', SPIECE_UNDERLINE + '''this''', SPIECE_UNDERLINE + '''is''', SPIECE_UNDERLINE + '''f''', '''al''', '''s''', '''<unk>''', '''.'''],			)






				@slow
				def     snake_case     (			self		:   List[str]							)     ->				Union[str, Any]:




										'''simple docstring'''



										_UpperCamelCase      : Any         =   {'''input_ids''': [[3_7_9_1, 7_9_7, 3_1, 1_1, 6_4, 7_9_7, 3_1, 2_4_2_9, 4_3_3, 1_2, 1_1_7_6, 1_2, 2_0, 7_8_6, 9_1_5, 1_4_2, 2_4_1_3, 2_4_0, 3_7, 3_2_3_8, 7_9_7, 3_1, 1_1, 3_5, 9_3, 9_1_5, 1_4_2, 2_4_1_3, 2_4_0, 3_7, 5_5_4_0, 5_6_7, 1_2_7_6, 9_3, 3_7, 6_1_0, 4_0, 6_2, 4_5_5, 6_5_7, 1_0_4_2, 1_2_3, 7_8_0, 1_7_7, 3_7, 3_0_9, 2_4_1, 1_2_9_8, 5_1_4, 2_0, 2_9_2, 2_7_3_7, 1_1_4, 2_4_6_9, 2_4_1, 8_5, 6_4, 3_0_2, 5_4_8, 5_2_8, 4_2_3, 4, 5_0_9, 4_0_6, 4_2_3, 3_7, 6_0_1, 4, 7_7_7, 3_0_2, 5_4_8, 5_2_8, 4_2_3, 2_8_4, 4, 3_3_8_8, 5_1_1, 4_5_9, 4, 3_5_5_5, 4_0, 3_2_1, 3_0_2, 7_0_5, 4, 3_3_8_8, 5_1_1, 5_8_3, 3_2_6, 5, 5, 5, 6_2, 3_3_1_0, 5_6_0, 1_7_7, 2_6_8_0, 2_1_7, 1_5_0_8, 3_2, 3_1, 8_5_3, 4_1_8, 6_4, 5_8_3, 5_1_1, 1_6_0_5, 6_2, 3_5, 9_3, 5_6_0, 1_7_7, 2_6_8_0, 2_1_7, 1_5_0_8, 1_5_2_1, 6_4, 5_8_3, 5_1_1, 5_1_9, 6_2, 2_0, 1_5_1_5, 7_6_4, 2_0, 1_4_9, 2_6_1, 5_6_2_5, 7_9_7_2, 2_0, 5_5_4_0, 5_6_7, 1_2_7_6, 9_3, 3_9_2_5, 1_6_7_5, 1_1, 1_5, 8_0_2, 7_9_7_2, 5_7_6, 2_1_7, 1_5_0_8, 1_1, 3_5, 9_3, 1_2_5_3, 2_4_4_1, 1_5, 2_8_9, 6_5_2, 3_1, 4_1_6, 3_2_1, 3_8_4_2, 1_1_5, 4_0, 9_1_1, 8, 4_7_6, 6_1_9, 4, 3_8_0, 1_4_2, 4_2_3, 3_3_5, 2_4_0, 3_5, 9_3, 2_6_4, 8, 1_1, 3_3_5, 5_6_9, 4_2_0, 1_6_3, 5, 2], [2_6_0, 5_4_8, 5_2_8, 4_2_3, 2_0, 4_5_1, 2_0, 2_6_8_1, 1_1_5_3, 3_4_3_4, 2_0, 5_5_4_0, 3_7, 5_6_7, 1_2_6, 1_2_5_3, 2_4_4_1, 3_3_7_6, 4_4_9, 2_1_0, 4_3_1, 1_5_6_3, 1_7_7, 7_6_7, 5_5_4_0, 1_1, 1_2_0_3, 4_7_2, 1_1, 2_9_5_3, 6_8_5, 2_8_5, 3_6_4, 7_0_6, 1_1_5_3, 2_0, 6_7_9_9, 2_0, 2_8_6_9, 2_0, 4_4_6_4, 1_2_6, 4_0, 2_4_2_9, 2_0, 1_0_4_0, 8_6_6, 2_6_6_4, 4_1_8, 2_0, 3_1_8, 2_0, 1_7_2_6, 1_8_6, 2_0, 2_6_5, 5_2_2, 3_5, 9_3, 2_1_9_1, 4_6_3_4, 2_0, 1_0_4_0, 1_2, 6_7_9_9, 1_5, 2_2_8, 2_3_5_6, 1_4_2, 3_1, 1_1, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [2_5_7_5, 2_6_6_6, 6_8_4, 1_5_8_2, 1_1_7_6, 1_2, 6_2_7, 1_4_9, 6_1_9, 2_0, 4_9_0_2, 5_6_3, 1_1, 2_0, 1_4_9, 2_6_1, 3_4_2_0, 2_3_5_6, 1_7_4, 1_4_2, 4_7_1_4, 1_3_1, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
										# fmt: on

										self.tokenizer_integration_test_util(
										    expected_encoding=lowerCAmelCase__,			model_name='''facebook/s2t-small-mustc-en-de-st''',			revision='''a14f04cf0776c02f62a8cb800cf7909e15ea23ad''',			)




@require_sentencepiece
class 			_a						(				unittest.TestCase    ):
				UpperCamelCase								=   '''valhalla/s2t_mustc_multilinguial_medium'''

				UpperCamelCase								=   '''C\'est trop cool'''
				UpperCamelCase								=   '''Esto es genial'''






				@classmethod
				def     snake_case     (			cls		:   Dict							)     ->				str:




										'''simple docstring'''



										_UpperCamelCase      : SpeechaTextTokenizer         =   SpeechaTextTokenizer.from_pretrained(cls.checkpoint_name							)
										return cls






				def     snake_case     (			self		:   Tuple							)     ->				List[str]:




										'''simple docstring'''



										self.assertEqual(self.tokenizer.lang_code_to_id['''pt'''],			4							)
										self.assertEqual(self.tokenizer.lang_code_to_id['''ru'''],			6							)
										self.assertEqual(self.tokenizer.lang_code_to_id['''it'''],			9							)
										self.assertEqual(self.tokenizer.lang_code_to_id['''de'''],			1_1							)






				def     snake_case     (			self		:   List[str]							)     ->				Union[str, Any]:




										'''simple docstring'''



										self.assertEqual(self.tokenizer.vocab_size,			1_0_0_0_0							)






				def     snake_case     (			self		:   Any							)     ->				Union[str, Any]:




										'''simple docstring'''



										self.assertIn(lowerCAmelCase__,			self.tokenizer.all_special_ids							)
										_UpperCamelCase      : List[Any]         =   [ES_CODE, 4, 1_6_0_1, 4_7, 7_6_4_7, 2]
										_UpperCamelCase      : Optional[Any]         =   self.tokenizer.decode(lowerCAmelCase__,			skip_special_tokens=lowerCAmelCase__							)
										_UpperCamelCase      : Optional[int]         =   self.tokenizer.decode(generated_ids[1:],			skip_special_tokens=lowerCAmelCase__							)
										self.assertEqual(lowerCAmelCase__,			lowerCAmelCase__							)
										self.assertNotIn(self.tokenizer.eos_token,			lowerCAmelCase__							)






				def     snake_case     (			self		:   List[str]							)     ->				List[str]:




										'''simple docstring'''



										_UpperCamelCase      : Dict         =   '''fr'''
										_UpperCamelCase      : int         =   self.tokenizer(self.french_text							).input_ids
										self.assertEqual(encoded[0],			lowerCAmelCase__							)
										self.assertEqual(encoded[-1],			self.tokenizer.eos_token_id							)






				def     snake_case     (			self		:   Union[str, Any]							)     ->				Union[str, Any]:




										'''simple docstring'''



										_UpperCamelCase      : Dict         =   '''fr'''
										self.assertListEqual(self.tokenizer.prefix_tokens,			[FR_CODE]							)

										_UpperCamelCase      : List[str]         =   '''es'''
										self.assertListEqual(self.tokenizer.prefix_tokens,			[ES_CODE]							)



