





"""simple docstring"""




import warnings
from typing import Any, Dict, List, Optional, Union

import numpy as np

from ...audio_utils import mel_filter_bank, optimal_fft_length, spectrogram, window_function
from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
from ...feature_extraction_utils import BatchFeature
from ...utils import PaddingStrategy, TensorType, logging


UpperCamelCase_									=logging.get_logger(__name__)




class 			_a						(				_lowerCAmelCase    ):
				UpperCamelCase								=   ['''input_values''', '''attention_mask''']






				def __init__(			self		:   Dict,			lowerCAmelCase__		:   int = 1,			lowerCAmelCase__		:   int = 1_6_0_0_0,			lowerCAmelCase__		:   float = 0.0,			lowerCAmelCase__		:   bool = False,			lowerCAmelCase__		:   int = 8_0,			lowerCAmelCase__		:   int = 1_6,			lowerCAmelCase__		:   int = 6_4,			lowerCAmelCase__		:   str = "hann_window",			lowerCAmelCase__		:   float = 1.0,			lowerCAmelCase__		:   float = 8_0,			lowerCAmelCase__		:   float = 7_6_0_0,			lowerCAmelCase__		:   float = 1e-1_0,			lowerCAmelCase__		:   int = 2,			lowerCAmelCase__		:   bool = True,			**lowerCAmelCase__		:   str,			)     ->				Any:




										'''simple docstring'''



										super().__init__(feature_size=lowerCAmelCase__,			sampling_rate=lowerCAmelCase__,			padding_value=lowerCAmelCase__,			**lowerCAmelCase__							)
										_UpperCamelCase      : List[str]         =   do_normalize
										_UpperCamelCase      : List[Any]         =   return_attention_mask

										_UpperCamelCase      : List[Any]         =   num_mel_bins
										_UpperCamelCase      : Any         =   hop_length
										_UpperCamelCase      : int         =   win_length
										_UpperCamelCase      : Optional[Any]         =   win_function
										_UpperCamelCase      : Optional[int]         =   frame_signal_scale
										_UpperCamelCase      : List[str]         =   fmin
										_UpperCamelCase      : Tuple         =   fmax
										_UpperCamelCase      : int         =   mel_floor
										_UpperCamelCase      : List[str]         =   reduction_factor

										_UpperCamelCase      : Any         =   win_length * sampling_rate // 1_0_0_0
										_UpperCamelCase      : int         =   hop_length * sampling_rate // 1_0_0_0
										_UpperCamelCase      : Optional[Any]         =   optimal_fft_length(self.sample_size							)
										_UpperCamelCase      : List[str]         =   (self.n_fft // 2) + 1

										_UpperCamelCase      : int         =   window_function(window_length=self.sample_size,			name=self.win_function,			periodic=lowerCAmelCase__							)

										_UpperCamelCase      : int         =   mel_filter_bank(
										    num_frequency_bins=self.n_freqs,			num_mel_filters=self.num_mel_bins,			min_frequency=self.fmin,			max_frequency=self.fmax,			sampling_rate=self.sampling_rate,			norm='''slaney''',			mel_scale='''slaney''',			)

										if frame_signal_scale != 1.0:
																warnings.warn(
																    '''The argument `frame_signal_scale` is deprecated and will be removed in version 4.30.0 of Transformers''',			lowerCAmelCase__,			)
										if reduction_factor != 2.0:
																warnings.warn(
																    '''The argument `reduction_factor` is deprecated and will be removed in version 4.30.0 of Transformers''',			lowerCAmelCase__,			)






				@staticmethod
				# Copied from transformers.models.wav2vec2.feature_extraction_wav2vec2.Wav2Vec2FeatureExtractor.zero_mean_unit_var_norm
				def     snake_case     (			lowerCAmelCase__		:   List[np.ndarray],			lowerCAmelCase__		:   List[np.ndarray],			lowerCAmelCase__		:   float = 0.0							)     ->				List[np.ndarray]:




										'''simple docstring'''



										if attention_mask is not None:
																_UpperCamelCase      : Union[str, Any]         =   np.array(lowerCAmelCase__,			np.intaa							)
																_UpperCamelCase      : Any         =   []

																for vector, length in zip(lowerCAmelCase__,			attention_mask.sum(-1							)							):
																						_UpperCamelCase      : int         =   (vector - vector[:length].mean()) / np.sqrt(vector[:length].var() + 1e-7							)
																						if length < normed_slice.shape[0]:
																												_UpperCamelCase      : Dict         =   padding_value

																						normed_input_values.append(lowerCAmelCase__							)
										else:
																_UpperCamelCase      : str         =   [(x - x.mean()) / np.sqrt(x.var() + 1e-7							) for x in input_values]

										return normed_input_values






				def     snake_case     (			self		:   Union[str, Any],			lowerCAmelCase__		:   np.ndarray,			)     ->				np.ndarray:




										'''simple docstring'''



										_UpperCamelCase      : List[Any]         =   spectrogram(
										    lowerCAmelCase__,			window=self.window,			frame_length=self.sample_size,			hop_length=self.sample_stride,			fft_length=self.n_fft,			mel_filters=self.mel_filters,			mel_floor=self.mel_floor,			log_mel='''log10''',			)
										return log_mel_spec.T






				def __call__(			self		:   List[str],			lowerCAmelCase__		:   Optional[Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]]] = None,			lowerCAmelCase__		:   Optional[Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]]] = None,			lowerCAmelCase__		:   Union[bool, str, PaddingStrategy] = False,			lowerCAmelCase__		:   Optional[int] = None,			lowerCAmelCase__		:   bool = False,			lowerCAmelCase__		:   Optional[int] = None,			lowerCAmelCase__		:   Optional[bool] = None,			lowerCAmelCase__		:   Optional[Union[str, TensorType]] = None,			lowerCAmelCase__		:   Optional[int] = None,			**lowerCAmelCase__		:   Tuple,			)     ->				BatchFeature:




										'''simple docstring'''



										if audio is None and audio_target is None:
																raise ValueError('''You must provide either `audio` or `audio_target` values.'''							)

										if sampling_rate is not None:
																if sampling_rate != self.sampling_rate:
																						raise ValueError(
																						    f"""The model corresponding to this feature extractor: {self} was trained using a sampling rate of"""
																						    f""" {self.sampling_rate}. Please make sure that the provided audio input was sampled with"""
																						    f""" {self.sampling_rate} and not {sampling_rate}."""							)
										else:
																logger.warning(
																    '''It is strongly recommended to pass the ``sampling_rate`` argument to this function. '''
																    '''Failing to do so can result in silent errors that might be hard to debug.'''							)

										if audio is not None:
																_UpperCamelCase      : List[Any]         =   self._process_audio(
																    lowerCAmelCase__,			lowerCAmelCase__,			lowerCAmelCase__,			lowerCAmelCase__,			lowerCAmelCase__,			lowerCAmelCase__,			lowerCAmelCase__,			lowerCAmelCase__,			**lowerCAmelCase__,			)
										else:
																_UpperCamelCase      : int         =   None

										if audio_target is not None:
																_UpperCamelCase      : Union[str, Any]         =   self._process_audio(
																    lowerCAmelCase__,			lowerCAmelCase__,			lowerCAmelCase__,			lowerCAmelCase__,			lowerCAmelCase__,			lowerCAmelCase__,			lowerCAmelCase__,			lowerCAmelCase__,			**lowerCAmelCase__,			)

																if inputs is None:
																						return inputs_target
																else:
																						_UpperCamelCase      : Optional[Any]         =   inputs_target['''input_values''']
																						_UpperCamelCase      : str         =   inputs_target.get('''attention_mask'''							)
																						if decoder_attention_mask is not None:
																												_UpperCamelCase      : Optional[int]         =   decoder_attention_mask

										return inputs






				def     snake_case     (			self		:   str,			lowerCAmelCase__		:   Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]],			lowerCAmelCase__		:   bool = False,			lowerCAmelCase__		:   Union[bool, str, PaddingStrategy] = False,			lowerCAmelCase__		:   Optional[int] = None,			lowerCAmelCase__		:   bool = False,			lowerCAmelCase__		:   Optional[int] = None,			lowerCAmelCase__		:   Optional[bool] = None,			lowerCAmelCase__		:   Optional[Union[str, TensorType]] = None,			**lowerCAmelCase__		:   str,			)     ->				BatchFeature:




										'''simple docstring'''



										_UpperCamelCase      : Union[str, Any]         =   isinstance(lowerCAmelCase__,			np.ndarray							) and len(speech.shape							) > 1
										if is_batched_numpy and len(speech.shape							) > 2:
																raise ValueError(f"""Only mono-channel audio is supported for input to {self}"""							)
										_UpperCamelCase      : str         =   is_batched_numpy or (
										    isinstance(lowerCAmelCase__,			(list, tuple)							) and (isinstance(speech[0],			(np.ndarray, tuple, list)							))
										)

										if is_batched:
																_UpperCamelCase      : Union[str, Any]         =   [np.asarray(lowerCAmelCase__,			dtype=np.floataa							) for speech in speech]
										elif not is_batched and not isinstance(lowerCAmelCase__,			np.ndarray							):
																_UpperCamelCase      : Dict         =   np.asarray(lowerCAmelCase__,			dtype=np.floataa							)
										elif isinstance(lowerCAmelCase__,			np.ndarray							) and speech.dtype is np.dtype(np.floataa							):
																_UpperCamelCase      : Dict         =   speech.astype(np.floataa							)

										# always return batch
										if not is_batched:
																_UpperCamelCase      : Union[str, Any]         =   [speech]

										# needed to make pad() work on spectrogram inputs
										_UpperCamelCase      : Any         =   self.feature_size

										# convert into correct format for padding
										if is_target:
																_UpperCamelCase      : List[Any]         =   [self._extract_mel_features(lowerCAmelCase__							) for waveform in speech]
																_UpperCamelCase      : str         =   BatchFeature({'''input_values''': features}							)
																_UpperCamelCase      : str         =   self.num_mel_bins
										else:
																_UpperCamelCase      : int         =   BatchFeature({'''input_values''': speech}							)

										_UpperCamelCase      : Dict         =   self.pad(
										    lowerCAmelCase__,			padding=lowerCAmelCase__,			max_length=lowerCAmelCase__,			truncation=lowerCAmelCase__,			pad_to_multiple_of=lowerCAmelCase__,			return_attention_mask=lowerCAmelCase__,			**lowerCAmelCase__,			)

										_UpperCamelCase      : str         =   feature_size_hack

										# convert input values to correct format
										_UpperCamelCase      : Tuple         =   padded_inputs['''input_values''']
										if not isinstance(input_values[0],			np.ndarray							):
																_UpperCamelCase      : str         =   [np.asarray(lowerCAmelCase__,			dtype=np.floataa							) for array in input_values]
										elif (
										    not isinstance(lowerCAmelCase__,			np.ndarray							)
										    and isinstance(input_values[0],			np.ndarray							)
										    and input_values[0].dtype is np.dtype(np.floataa							)
										):
																_UpperCamelCase      : List[str]         =   [array.astype(np.floataa							) for array in input_values]
										elif isinstance(lowerCAmelCase__,			np.ndarray							) and input_values.dtype is np.dtype(np.floataa							):
																_UpperCamelCase      : List[str]         =   input_values.astype(np.floataa							)

										# convert attention_mask to correct format
										_UpperCamelCase      : Any         =   padded_inputs.get('''attention_mask'''							)
										if attention_mask is not None:
																_UpperCamelCase      : int         =   [np.asarray(lowerCAmelCase__,			dtype=np.intaa							) for array in attention_mask]

										# zero-mean and unit-variance normalization
										if not is_target and self.do_normalize:
																_UpperCamelCase      : List[Any]         =   (
																    attention_mask
																    if self._get_padding_strategies(lowerCAmelCase__,			max_length=lowerCAmelCase__							) is not PaddingStrategy.DO_NOT_PAD
																    else None
																)
																_UpperCamelCase      : Dict         =   self.zero_mean_unit_var_norm(
																    padded_inputs['''input_values'''],			attention_mask=lowerCAmelCase__,			padding_value=self.padding_value							)

										if return_tensors is not None:
																_UpperCamelCase      : List[str]         =   padded_inputs.convert_to_tensors(lowerCAmelCase__							)

										return padded_inputs






				def     snake_case     (			self		:   Optional[int]							)     ->				Dict[str, Any]:




										'''simple docstring'''



										_UpperCamelCase      : List[Any]         =   super().to_dict()

										# Don't serialize these as they are derived from the other properties.
										_UpperCamelCase      : str         =   ['''window''', '''mel_filters''', '''sample_size''', '''sample_stride''', '''n_fft''', '''n_freqs''']
										for name in names:
																if name in output:
																						del output[name]

										return output










"""simple docstring"""




import unittest
from pathlib import Path
from shutil import copyfile

from transformers import SPIECE_UNDERLINE, is_sentencepiece_available
from transformers.models.speech_to_text import SpeechaTextTokenizer
from transformers.models.speech_to_text.tokenization_speech_to_text import VOCAB_FILES_NAMES, save_json
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


UpperCamelCase_									=get_tests_dir("""fixtures/test_sentencepiece.model""")

if is_sentencepiece_available():
						import sentencepiece as sp


UpperCamelCase_									=5
UpperCamelCase_									=10




@require_sentencepiece
@require_tokenizers
class 			_a						(				_lowerCAmelCase     ,    unittest.TestCase    ):
				UpperCamelCase								=   SpeechaTextTokenizer
				UpperCamelCase								=   False
				UpperCamelCase								=   True






				def     snake_case     (			self		:   Tuple							)     ->				Optional[Any]:




										'''simple docstring'''



										super().setUp()

										_UpperCamelCase      : Dict         =   sp.SentencePieceProcessor()
										spm_model.Load(lowerCAmelCase__							)
										_UpperCamelCase      : Union[str, Any]         =   ['''<s>''', '''<pad>''', '''</s>''', '''<unk>''']

										vocab += [spm_model.IdToPiece(id_							) for id_ in range(len(lowerCAmelCase__							)							)]
										_UpperCamelCase      : Tuple         =   dict(zip(lowerCAmelCase__,			range(len(lowerCAmelCase__							)							)							)							)

										_UpperCamelCase      : Dict         =   Path(self.tmpdirname							)
										save_json(lowerCAmelCase__,			save_dir / VOCAB_FILES_NAMES['''vocab_file''']							)
										if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
																copyfile(lowerCAmelCase__,			save_dir / VOCAB_FILES_NAMES['''spm_file''']							)

										_UpperCamelCase      : List[str]         =   SpeechaTextTokenizer.from_pretrained(self.tmpdirname							)
										tokenizer.save_pretrained(self.tmpdirname							)






				def     snake_case     (			self		:   str							)     ->				str:




										'''simple docstring'''



										_UpperCamelCase      : List[str]         =   '''<pad>'''
										_UpperCamelCase      : List[Any]         =   1

										self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCAmelCase__							),			lowerCAmelCase__							)
										self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCAmelCase__							),			lowerCAmelCase__							)






				def     snake_case     (			self		:   Any							)     ->				Optional[int]:




										'''simple docstring'''



										_UpperCamelCase      : int         =   list(self.get_tokenizer().get_vocab().keys()							)

										self.assertEqual(vocab_keys[0],			'''<s>'''							)
										self.assertEqual(vocab_keys[1],			'''<pad>'''							)
										self.assertEqual(vocab_keys[-1],			'''j'''							)
										self.assertEqual(len(lowerCAmelCase__							),			1_0_0_1							)






				def     snake_case     (			self		:   Any							)     ->				Dict:




										'''simple docstring'''



										self.assertEqual(self.get_tokenizer().vocab_size,			1_0_0_1							)






				def     snake_case     (			self		:   List[Any]							)     ->				Tuple:




										'''simple docstring'''



										_UpperCamelCase      : Optional[int]         =   SpeechaTextTokenizer.from_pretrained(self.tmpdirname							)

										_UpperCamelCase      : List[str]         =   tokenizer.tokenize('''This is a test'''							)
										self.assertListEqual(lowerCAmelCase__,			['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']							)

										self.assertListEqual(
										    tokenizer.convert_tokens_to_ids(lowerCAmelCase__							),			[2_8_9, 5_0, 1_4, 1_7_4, 3_8_6],			)

										_UpperCamelCase      : List[str]         =   tokenizer.tokenize('''I was born in 92000, and this is falsé.'''							)
										self.assertListEqual(
										    lowerCAmelCase__,			[SPIECE_UNDERLINE + '''I''', SPIECE_UNDERLINE + '''was''', SPIECE_UNDERLINE + '''b''', '''or''', '''n''', SPIECE_UNDERLINE + '''in''', SPIECE_UNDERLINE + '''''', '''9''', '''2''', '''0''', '''0''', '''0''', ''',''', SPIECE_UNDERLINE + '''and''', SPIECE_UNDERLINE + '''this''', SPIECE_UNDERLINE + '''is''', SPIECE_UNDERLINE + '''f''', '''al''', '''s''', '''é''', '''.'''],			)
										_UpperCamelCase      : Optional[Any]         =   tokenizer.convert_tokens_to_ids(lowerCAmelCase__							)
										self.assertListEqual(lowerCAmelCase__,			[1_2, 2_5, 8_8, 5_9, 2_8, 2_3, 1_1, 4, 6_0_6, 3_5_1, 3_5_1, 3_5_1, 7, 1_6, 7_0, 5_0, 7_6, 8_4, 1_0, 4, 8]							)

										_UpperCamelCase      : Union[str, Any]         =   tokenizer.convert_ids_to_tokens(lowerCAmelCase__							)
										self.assertListEqual(
										    lowerCAmelCase__,			[SPIECE_UNDERLINE + '''I''', SPIECE_UNDERLINE + '''was''', SPIECE_UNDERLINE + '''b''', '''or''', '''n''', SPIECE_UNDERLINE + '''in''', SPIECE_UNDERLINE + '''''', '''<unk>''', '''2''', '''0''', '''0''', '''0''', ''',''', SPIECE_UNDERLINE + '''and''', SPIECE_UNDERLINE + '''this''', SPIECE_UNDERLINE + '''is''', SPIECE_UNDERLINE + '''f''', '''al''', '''s''', '''<unk>''', '''.'''],			)






				@slow
				def     snake_case     (			self		:   List[str]							)     ->				Union[str, Any]:




										'''simple docstring'''



										_UpperCamelCase      : Any         =   {'''input_ids''': [[3_7_9_1, 7_9_7, 3_1, 1_1, 6_4, 7_9_7, 3_1, 2_4_2_9, 4_3_3, 1_2, 1_1_7_6, 1_2, 2_0, 7_8_6, 9_1_5, 1_4_2, 2_4_1_3, 2_4_0, 3_7, 3_2_3_8, 7_9_7, 3_1, 1_1, 3_5, 9_3, 9_1_5, 1_4_2, 2_4_1_3, 2_4_0, 3_7, 5_5_4_0, 5_6_7, 1_2_7_6, 9_3, 3_7, 6_1_0, 4_0, 6_2, 4_5_5, 6_5_7, 1_0_4_2, 1_2_3, 7_8_0, 1_7_7, 3_7, 3_0_9, 2_4_1, 1_2_9_8, 5_1_4, 2_0, 2_9_2, 2_7_3_7, 1_1_4, 2_4_6_9, 2_4_1, 8_5, 6_4, 3_0_2, 5_4_8, 5_2_8, 4_2_3, 4, 5_0_9, 4_0_6, 4_2_3, 3_7, 6_0_1, 4, 7_7_7, 3_0_2, 5_4_8, 5_2_8, 4_2_3, 2_8_4, 4, 3_3_8_8, 5_1_1, 4_5_9, 4, 3_5_5_5, 4_0, 3_2_1, 3_0_2, 7_0_5, 4, 3_3_8_8, 5_1_1, 5_8_3, 3_2_6, 5, 5, 5, 6_2, 3_3_1_0, 5_6_0, 1_7_7, 2_6_8_0, 2_1_7, 1_5_0_8, 3_2, 3_1, 8_5_3, 4_1_8, 6_4, 5_8_3, 5_1_1, 1_6_0_5, 6_2, 3_5, 9_3, 5_6_0, 1_7_7, 2_6_8_0, 2_1_7, 1_5_0_8, 1_5_2_1, 6_4, 5_8_3, 5_1_1, 5_1_9, 6_2, 2_0, 1_5_1_5, 7_6_4, 2_0, 1_4_9, 2_6_1, 5_6_2_5, 7_9_7_2, 2_0, 5_5_4_0, 5_6_7, 1_2_7_6, 9_3, 3_9_2_5, 1_6_7_5, 1_1, 1_5, 8_0_2, 7_9_7_2, 5_7_6, 2_1_7, 1_5_0_8, 1_1, 3_5, 9_3, 1_2_5_3, 2_4_4_1, 1_5, 2_8_9, 6_5_2, 3_1, 4_1_6, 3_2_1, 3_8_4_2, 1_1_5, 4_0, 9_1_1, 8, 4_7_6, 6_1_9, 4, 3_8_0, 1_4_2, 4_2_3, 3_3_5, 2_4_0, 3_5, 9_3, 2_6_4, 8, 1_1, 3_3_5, 5_6_9, 4_2_0, 1_6_3, 5, 2], [2_6_0, 5_4_8, 5_2_8, 4_2_3, 2_0, 4_5_1, 2_0, 2_6_8_1, 1_1_5_3, 3_4_3_4, 2_0, 5_5_4_0, 3_7, 5_6_7, 1_2_6, 1_2_5_3, 2_4_4_1, 3_3_7_6, 4_4_9, 2_1_0, 4_3_1, 1_5_6_3, 1_7_7, 7_6_7, 5_5_4_0, 1_1, 1_2_0_3, 4_7_2, 1_1, 2_9_5_3, 6_8_5, 2_8_5, 3_6_4, 7_0_6, 1_1_5_3, 2_0, 6_7_9_9, 2_0, 2_8_6_9, 2_0, 4_4_6_4, 1_2_6, 4_0, 2_4_2_9, 2_0, 1_0_4_0, 8_6_6, 2_6_6_4, 4_1_8, 2_0, 3_1_8, 2_0, 1_7_2_6, 1_8_6, 2_0, 2_6_5, 5_2_2, 3_5, 9_3, 2_1_9_1, 4_6_3_4, 2_0, 1_0_4_0, 1_2, 6_7_9_9, 1_5, 2_2_8, 2_3_5_6, 1_4_2, 3_1, 1_1, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [2_5_7_5, 2_6_6_6, 6_8_4, 1_5_8_2, 1_1_7_6, 1_2, 6_2_7, 1_4_9, 6_1_9, 2_0, 4_9_0_2, 5_6_3, 1_1, 2_0, 1_4_9, 2_6_1, 3_4_2_0, 2_3_5_6, 1_7_4, 1_4_2, 4_7_1_4, 1_3_1, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
										# fmt: on

										self.tokenizer_integration_test_util(
										    expected_encoding=lowerCAmelCase__,			model_name='''facebook/s2t-small-mustc-en-de-st''',			revision='''a14f04cf0776c02f62a8cb800cf7909e15ea23ad''',			)




@require_sentencepiece
class 			_a						(				unittest.TestCase    ):
				UpperCamelCase								=   '''valhalla/s2t_mustc_multilinguial_medium'''

				UpperCamelCase								=   '''C\'est trop cool'''
				UpperCamelCase								=   '''Esto es genial'''






				@classmethod
				def     snake_case     (			cls		:   Dict							)     ->				str:




										'''simple docstring'''



										_UpperCamelCase      : SpeechaTextTokenizer         =   SpeechaTextTokenizer.from_pretrained(cls.checkpoint_name							)
										return cls






				def     snake_case     (			self		:   Tuple							)     ->				List[str]:




										'''simple docstring'''



										self.assertEqual(self.tokenizer.lang_code_to_id['''pt'''],			4							)
										self.assertEqual(self.tokenizer.lang_code_to_id['''ru'''],			6							)
										self.assertEqual(self.tokenizer.lang_code_to_id['''it'''],			9							)
										self.assertEqual(self.tokenizer.lang_code_to_id['''de'''],			1_1							)






				def     snake_case     (			self		:   List[str]							)     ->				Union[str, Any]:




										'''simple docstring'''



										self.assertEqual(self.tokenizer.vocab_size,			1_0_0_0_0							)






				def     snake_case     (			self		:   Any							)     ->				Union[str, Any]:




										'''simple docstring'''



										self.assertIn(lowerCAmelCase__,			self.tokenizer.all_special_ids							)
										_UpperCamelCase      : List[Any]         =   [ES_CODE, 4, 1_6_0_1, 4_7, 7_6_4_7, 2]
										_UpperCamelCase      : Optional[Any]         =   self.tokenizer.decode(lowerCAmelCase__,			skip_special_tokens=lowerCAmelCase__							)
										_UpperCamelCase      : Optional[int]         =   self.tokenizer.decode(generated_ids[1:],			skip_special_tokens=lowerCAmelCase__							)
										self.assertEqual(lowerCAmelCase__,			lowerCAmelCase__							)
										self.assertNotIn(self.tokenizer.eos_token,			lowerCAmelCase__							)






				def     snake_case     (			self		:   List[str]							)     ->				List[str]:




										'''simple docstring'''



										_UpperCamelCase      : Dict         =   '''fr'''
										_UpperCamelCase      : int         =   self.tokenizer(self.french_text							).input_ids
										self.assertEqual(encoded[0],			lowerCAmelCase__							)
										self.assertEqual(encoded[-1],			self.tokenizer.eos_token_id							)






				def     snake_case     (			self		:   Union[str, Any]							)     ->				Union[str, Any]:




										'''simple docstring'''



										_UpperCamelCase      : Dict         =   '''fr'''
										self.assertListEqual(self.tokenizer.prefix_tokens,			[FR_CODE]							)

										_UpperCamelCase      : List[str]         =   '''es'''
										self.assertListEqual(self.tokenizer.prefix_tokens,			[ES_CODE]							)



