





"""simple docstring"""




import argparse

import torch

from transformers import (
    EncodecConfig,
    EncodecFeatureExtractor,
    EncodecModel,
    logging,
)


# checkpoints downloaded from:
# https://dl.fbaipublicfiles.com/encodec/v0/encodec_24khz-d7cc33bc.th
# https://huggingface.co/facebook/musicgen-small/resolve/main/compression_state_dict.bin
# https://dl.fbaipublicfiles.com/encodec/v0/encodec_48khz-7e698e3e.th


logging.set_verbosity_info()
UpperCamelCase_									=logging.get_logger("""transformers.models.encodec""")

UpperCamelCase_									={
    """quantizer.vq.layers.*._codebook.inited""": """quantizer.layers.*.codebook.inited""",
    """quantizer.vq.layers.*._codebook.cluster_size""": """quantizer.layers.*.codebook.cluster_size""",
    """quantizer.vq.layers.*._codebook.embed""": """quantizer.layers.*.codebook.embed""",
    """quantizer.vq.layers.*._codebook.embed_avg""": """quantizer.layers.*.codebook.embed_avg""",
}
UpperCamelCase_									={
    """encoder.model.0.conv.conv""": """encoder.layers.0.conv""",
    """encoder.model.1.block.1.conv.conv""": """encoder.layers.1.block.1.conv""",
    """encoder.model.1.block.3.conv.conv""": """encoder.layers.1.block.3.conv""",
    """encoder.model.1.shortcut.conv.conv""": """encoder.layers.1.shortcut.conv""",
    """encoder.model.3.conv.conv""": """encoder.layers.3.conv""",
    """encoder.model.4.block.1.conv.conv""": """encoder.layers.4.block.1.conv""",
    """encoder.model.4.block.3.conv.conv""": """encoder.layers.4.block.3.conv""",
    """encoder.model.4.shortcut.conv.conv""": """encoder.layers.4.shortcut.conv""",
    """encoder.model.6.conv.conv""": """encoder.layers.6.conv""",
    """encoder.model.7.block.1.conv.conv""": """encoder.layers.7.block.1.conv""",
    """encoder.model.7.block.3.conv.conv""": """encoder.layers.7.block.3.conv""",
    """encoder.model.7.shortcut.conv.conv""": """encoder.layers.7.shortcut.conv""",
    """encoder.model.9.conv.conv""": """encoder.layers.9.conv""",
    """encoder.model.10.block.1.conv.conv""": """encoder.layers.10.block.1.conv""",
    """encoder.model.10.block.3.conv.conv""": """encoder.layers.10.block.3.conv""",
    """encoder.model.10.shortcut.conv.conv""": """encoder.layers.10.shortcut.conv""",
    """encoder.model.12.conv.conv""": """encoder.layers.12.conv""",
    """encoder.model.13.lstm""": """encoder.layers.13.lstm""",
    """encoder.model.15.conv.conv""": """encoder.layers.15.conv""",
}
UpperCamelCase_									={
    """encoder.model.0.conv.norm""": """encoder.layers.0.norm""",
    """encoder.model.1.block.1.conv.norm""": """encoder.layers.1.block.1.norm""",
    """encoder.model.1.block.3.conv.norm""": """encoder.layers.1.block.3.norm""",
    """encoder.model.1.shortcut.conv.norm""": """encoder.layers.1.shortcut.norm""",
    """encoder.model.3.conv.norm""": """encoder.layers.3.norm""",
    """encoder.model.4.block.1.conv.norm""": """encoder.layers.4.block.1.norm""",
    """encoder.model.4.block.3.conv.norm""": """encoder.layers.4.block.3.norm""",
    """encoder.model.4.shortcut.conv.norm""": """encoder.layers.4.shortcut.norm""",
    """encoder.model.6.conv.norm""": """encoder.layers.6.norm""",
    """encoder.model.7.block.1.conv.norm""": """encoder.layers.7.block.1.norm""",
    """encoder.model.7.block.3.conv.norm""": """encoder.layers.7.block.3.norm""",
    """encoder.model.7.shortcut.conv.norm""": """encoder.layers.7.shortcut.norm""",
    """encoder.model.9.conv.norm""": """encoder.layers.9.norm""",
    """encoder.model.10.block.1.conv.norm""": """encoder.layers.10.block.1.norm""",
    """encoder.model.10.block.3.conv.norm""": """encoder.layers.10.block.3.norm""",
    """encoder.model.10.shortcut.conv.norm""": """encoder.layers.10.shortcut.norm""",
    """encoder.model.12.conv.norm""": """encoder.layers.12.norm""",
    """encoder.model.15.conv.norm""": """encoder.layers.15.norm""",
}
UpperCamelCase_									={
    """decoder.model.0.conv.conv""": """decoder.layers.0.conv""",
    """decoder.model.1.lstm""": """decoder.layers.1.lstm""",
    """decoder.model.3.convtr.convtr""": """decoder.layers.3.conv""",
    """decoder.model.4.block.1.conv.conv""": """decoder.layers.4.block.1.conv""",
    """decoder.model.4.block.3.conv.conv""": """decoder.layers.4.block.3.conv""",
    """decoder.model.4.shortcut.conv.conv""": """decoder.layers.4.shortcut.conv""",
    """decoder.model.6.convtr.convtr""": """decoder.layers.6.conv""",
    """decoder.model.7.block.1.conv.conv""": """decoder.layers.7.block.1.conv""",
    """decoder.model.7.block.3.conv.conv""": """decoder.layers.7.block.3.conv""",
    """decoder.model.7.shortcut.conv.conv""": """decoder.layers.7.shortcut.conv""",
    """decoder.model.9.convtr.convtr""": """decoder.layers.9.conv""",
    """decoder.model.10.block.1.conv.conv""": """decoder.layers.10.block.1.conv""",
    """decoder.model.10.block.3.conv.conv""": """decoder.layers.10.block.3.conv""",
    """decoder.model.10.shortcut.conv.conv""": """decoder.layers.10.shortcut.conv""",
    """decoder.model.12.convtr.convtr""": """decoder.layers.12.conv""",
    """decoder.model.13.block.1.conv.conv""": """decoder.layers.13.block.1.conv""",
    """decoder.model.13.block.3.conv.conv""": """decoder.layers.13.block.3.conv""",
    """decoder.model.13.shortcut.conv.conv""": """decoder.layers.13.shortcut.conv""",
    """decoder.model.15.conv.conv""": """decoder.layers.15.conv""",
}
UpperCamelCase_									={
    """decoder.model.0.conv.norm""": """decoder.layers.0.norm""",
    """decoder.model.3.convtr.norm""": """decoder.layers.3.norm""",
    """decoder.model.4.block.1.conv.norm""": """decoder.layers.4.block.1.norm""",
    """decoder.model.4.block.3.conv.norm""": """decoder.layers.4.block.3.norm""",
    """decoder.model.4.shortcut.conv.norm""": """decoder.layers.4.shortcut.norm""",
    """decoder.model.6.convtr.norm""": """decoder.layers.6.norm""",
    """decoder.model.7.block.1.conv.norm""": """decoder.layers.7.block.1.norm""",
    """decoder.model.7.block.3.conv.norm""": """decoder.layers.7.block.3.norm""",
    """decoder.model.7.shortcut.conv.norm""": """decoder.layers.7.shortcut.norm""",
    """decoder.model.9.convtr.norm""": """decoder.layers.9.norm""",
    """decoder.model.10.block.1.conv.norm""": """decoder.layers.10.block.1.norm""",
    """decoder.model.10.block.3.conv.norm""": """decoder.layers.10.block.3.norm""",
    """decoder.model.10.shortcut.conv.norm""": """decoder.layers.10.shortcut.norm""",
    """decoder.model.12.convtr.norm""": """decoder.layers.12.norm""",
    """decoder.model.13.block.1.conv.norm""": """decoder.layers.13.block.1.norm""",
    """decoder.model.13.block.3.conv.norm""": """decoder.layers.13.block.3.norm""",
    """decoder.model.13.shortcut.conv.norm""": """decoder.layers.13.shortcut.norm""",
    """decoder.model.15.conv.norm""": """decoder.layers.15.norm""",
}
UpperCamelCase_									={
    **MAPPING_QUANTIZER,
    **MAPPING_ENCODER,
    **MAPPING_DECODER,
}
UpperCamelCase_									={
    **MAPPING_QUANTIZER,
    **MAPPING_ENCODER,
    **MAPPING_ENCODER_48K,
    **MAPPING_DECODER,
    **MAPPING_DECODER_48K,
}
UpperCamelCase_									=[]
UpperCamelCase_									=[]





def 	a_    (							_lowercase	,						_lowercase	,						_lowercase	,						_lowercase	,						_lowercase       ):





      for attribute in key.split('''.'''       ):
            _UpperCamelCase      : Optional[Any]         =   getattr(lowercase__	,						lowercase__       )

      if weight_type is not None:
            _UpperCamelCase      : Tuple         =   getattr(lowercase__	,						lowercase__       ).shape
      else:
            _UpperCamelCase      : List[Any]         =   hf_pointer.shape

      if hf_shape != value.shape:
            raise ValueError(
                F"""Shape of hf {key + '.' + weight_type if weight_type is not None else ''} is {hf_shape}, but should be"""
                F""" {value.shape} for {full_name}"""       )

      if weight_type == "weight":
            _UpperCamelCase      : Union[str, Any]         =   value
      elif weight_type == "weight_g":
            _UpperCamelCase      : Any         =   value
      elif weight_type == "weight_v":
            _UpperCamelCase      : str         =   value
      elif weight_type == "bias":
            _UpperCamelCase      : Any         =   value
      elif weight_type == "running_mean":
            _UpperCamelCase      : List[str]         =   value
      elif weight_type == "running_var":
            _UpperCamelCase      : Optional[int]         =   value
      elif weight_type == "num_batches_tracked":
            _UpperCamelCase      : List[Any]         =   value
      elif weight_type == "weight_ih_l0":
            _UpperCamelCase      : Optional[int]         =   value
      elif weight_type == "weight_hh_l0":
            _UpperCamelCase      : Tuple         =   value
      elif weight_type == "bias_ih_l0":
            _UpperCamelCase      : int         =   value
      elif weight_type == "bias_hh_l0":
            _UpperCamelCase      : Tuple         =   value
      elif weight_type == "weight_ih_l1":
            _UpperCamelCase      : List[str]         =   value
      elif weight_type == "weight_hh_l1":
            _UpperCamelCase      : int         =   value
      elif weight_type == "bias_ih_l1":
            _UpperCamelCase      : int         =   value
      elif weight_type == "bias_hh_l1":
            _UpperCamelCase      : Optional[int]         =   value
      else:
            _UpperCamelCase      : Any         =   value

      logger.info(F"""{key + ('.' + weight_type if weight_type is not None else '')} was initialized from {full_name}."""       )





def 	a_    (							_lowercase	,						_lowercase       ):





      for key in ignore_keys:
            if key.endswith('''.*'''       ):
                  if name.startswith(key[:-1]       ):
                        return True
            elif ".*." in key:
                  _UpperCamelCase      : List[Any]         =   key.split('''.*.'''       )
                  if prefix in name and suffix in name:
                        return True
            elif key in name:
                  return True
      return False





def 	a_    (							_lowercase	,						_lowercase	,						_lowercase       ):
      _UpperCamelCase      : int         =   []

      if model_name == "encodec_24khz" or "encodec_32khz":
            _UpperCamelCase      : Optional[int]         =   MAPPING_24K
      elif model_name == "encodec_48khz":
            _UpperCamelCase      : List[Any]         =   MAPPING_48K
      else:
            raise ValueError(F"""Unsupported model: {model_name}"""       )

      for name, value in orig_dict.items():
            if should_ignore(lowercase__	,						lowercase__       ):
                  logger.info(F"""{name} was ignored"""       )
                  continue

            _UpperCamelCase      : str         =   False
            for key, mapped_key in MAPPING.items():
                  if "*" in key:
                        _UpperCamelCase      : Union[str, Any]         =   key.split('''.*.'''       )
                        if prefix in name and suffix in name:
                              _UpperCamelCase      : Tuple         =   suffix

                  if key in name:
                        # HACK otherwise .embed gets initialized with .embed_avg too
                        if key.endswith('''embed'''       ) and name.endswith('''embed_avg'''       ):
                              continue

                        _UpperCamelCase      : str         =   True
                        if "*" in mapped_key:
                              _UpperCamelCase      : Optional[Any]         =   name.split(lowercase__       )[0].split('''.'''       )[-2]
                              _UpperCamelCase      : Optional[Any]         =   mapped_key.replace('''*'''	,						lowercase__       )
                        if "weight_g" in name:
                              _UpperCamelCase      : List[str]         =   """weight_g"""
                        elif "weight_v" in name:
                              _UpperCamelCase      : Optional[int]         =   """weight_v"""
                        elif "weight_ih_l0" in name:
                              _UpperCamelCase      : Dict         =   """weight_ih_l0"""
                        elif "weight_hh_l0" in name:
                              _UpperCamelCase      : List[Any]         =   """weight_hh_l0"""
                        elif "bias_ih_l0" in name:
                              _UpperCamelCase      : int         =   """bias_ih_l0"""
                        elif "bias_hh_l0" in name:
                              _UpperCamelCase      : Optional[int]         =   """bias_hh_l0"""
                        elif "weight_ih_l1" in name:
                              _UpperCamelCase      : Union[str, Any]         =   """weight_ih_l1"""
                        elif "weight_hh_l1" in name:
                              _UpperCamelCase      : Any         =   """weight_hh_l1"""
                        elif "bias_ih_l1" in name:
                              _UpperCamelCase      : Dict         =   """bias_ih_l1"""
                        elif "bias_hh_l1" in name:
                              _UpperCamelCase      : List[str]         =   """bias_hh_l1"""
                        elif "bias" in name:
                              _UpperCamelCase      : Tuple         =   """bias"""
                        elif "weight" in name:
                              _UpperCamelCase      : List[str]         =   """weight"""
                        elif "running_mean" in name:
                              _UpperCamelCase      : Optional[Any]         =   """running_mean"""
                        elif "running_var" in name:
                              _UpperCamelCase      : int         =   """running_var"""
                        elif "num_batches_tracked" in name:
                              _UpperCamelCase      : Tuple         =   """num_batches_tracked"""
                        else:
                              _UpperCamelCase      : Any         =   None
                        set_recursively(lowercase__	,						lowercase__	,						lowercase__	,						lowercase__	,						lowercase__       )
                  continue
            if not is_used:
                  unused_weights.append(lowercase__       )

      logger.warning(F"""Unused weights: {unused_weights}"""       )





@torch.no_grad()
def 	a_    (							_lowercase	,						_lowercase	,						_lowercase	,						_lowercase=None	,						_lowercase=None	,						):





      if config_path is not None:
            _UpperCamelCase      : List[str]         =   EncodecConfig.from_pretrained(lowercase__       )
      else:
            _UpperCamelCase      : Any         =   EncodecConfig()

      if model_name == "encodec_24khz":
            pass  # config is already correct
      elif model_name == "encodec_32khz":
            _UpperCamelCase      : List[str]         =   [8, 5, 4, 4]
            _UpperCamelCase      : int         =   [2.2]
            _UpperCamelCase      : List[str]         =   64
            _UpperCamelCase      : Optional[int]         =   3_2000
            _UpperCamelCase      : Tuple         =   2048
            _UpperCamelCase      : Optional[int]         =   False
            _UpperCamelCase      : Any         =   False
            _UpperCamelCase      : int         =   False
      elif model_name == "encodec_48khz":
            _UpperCamelCase      : Optional[int]         =   [8, 5, 4, 2]
            _UpperCamelCase      : Optional[Any]         =   [3.0, 6.0, 12.0, 24.0]
            _UpperCamelCase      : Optional[int]         =   4_8000
            _UpperCamelCase      : str         =   2
            _UpperCamelCase      : List[str]         =   False
            _UpperCamelCase      : str         =   """time_group_norm"""
            _UpperCamelCase      : int         =   True
            _UpperCamelCase      : Optional[Any]         =   1.0
            _UpperCamelCase      : Optional[Any]         =   0.01
      else:
            raise ValueError(F"""Unknown model name: {model_name}"""       )

      _UpperCamelCase      : Dict         =   EncodecModel(lowercase__       )

      _UpperCamelCase      : Tuple         =   EncodecFeatureExtractor(
          feature_size=config.audio_channels	,						sampling_rate=config.sampling_rate	,						chunk_length_s=config.chunk_length_s	,						overlap=config.overlap	,						)
      feature_extractor.save_pretrained(lowercase__       )

      _UpperCamelCase      : int         =   torch.load(lowercase__       )
      if "best_state" in original_checkpoint:
            # we might have a training state saved, in which case discard the yaml results and just retain the weights
            _UpperCamelCase      : Optional[Any]         =   original_checkpoint["""best_state"""]
      recursively_load_weights(lowercase__	,						lowercase__	,						lowercase__       )
      model.save_pretrained(lowercase__       )

      if repo_id:
            print('''Pushing to the hub...'''       )
            feature_extractor.push_to_hub(lowercase__       )
            model.push_to_hub(lowercase__       )


if __name__ == "__main__":
      UpperCamelCase_									=argparse.ArgumentParser()
      parser.add_argument(
          """--model""",
          default="""encodec_24khz""",
          type=str,
          help="""The model to convert. Should be one of \'encodec_24khz\', \'encodec_32khz\', \'encodec_48khz\'.""",
      )
      parser.add_argument("""--checkpoint_path""", required=True, default=None, type=str, help="""Path to original checkpoint""")
      parser.add_argument("""--config_path""", default=None, type=str, help="""Path to hf config.json of model to convert""")
      parser.add_argument(
          """--pytorch_dump_folder_path""", required=True, default=None, type=str, help="""Path to the output PyTorch model."""
      )
      parser.add_argument(
          """--push_to_hub""", default=None, type=str, help="""Where to upload the converted model on the 🤗 hub."""
      )

      UpperCamelCase_									=parser.parse_args()
      convert_checkpoint(
          args.model,
          args.checkpoint_path,
          args.pytorch_dump_folder_path,
          args.config_path,
          args.push_to_hub,
      )










"""simple docstring"""




import unittest
from pathlib import Path
from shutil import copyfile

from transformers import SPIECE_UNDERLINE, is_sentencepiece_available
from transformers.models.speech_to_text import SpeechaTextTokenizer
from transformers.models.speech_to_text.tokenization_speech_to_text import VOCAB_FILES_NAMES, save_json
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


UpperCamelCase_									=get_tests_dir("""fixtures/test_sentencepiece.model""")

if is_sentencepiece_available():
						import sentencepiece as sp


UpperCamelCase_									=5
UpperCamelCase_									=10




@require_sentencepiece
@require_tokenizers
class 			_a						(				_lowerCAmelCase     ,    unittest.TestCase    ):
				UpperCamelCase								=   SpeechaTextTokenizer
				UpperCamelCase								=   False
				UpperCamelCase								=   True






				def     snake_case     (			self		:   Tuple							)     ->				Optional[Any]:




										'''simple docstring'''



										super().setUp()

										_UpperCamelCase      : Dict         =   sp.SentencePieceProcessor()
										spm_model.Load(lowerCAmelCase__							)
										_UpperCamelCase      : Union[str, Any]         =   ['''<s>''', '''<pad>''', '''</s>''', '''<unk>''']

										vocab += [spm_model.IdToPiece(id_							) for id_ in range(len(lowerCAmelCase__							)							)]
										_UpperCamelCase      : Tuple         =   dict(zip(lowerCAmelCase__,			range(len(lowerCAmelCase__							)							)							)							)

										_UpperCamelCase      : Dict         =   Path(self.tmpdirname							)
										save_json(lowerCAmelCase__,			save_dir / VOCAB_FILES_NAMES['''vocab_file''']							)
										if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
																copyfile(lowerCAmelCase__,			save_dir / VOCAB_FILES_NAMES['''spm_file''']							)

										_UpperCamelCase      : List[str]         =   SpeechaTextTokenizer.from_pretrained(self.tmpdirname							)
										tokenizer.save_pretrained(self.tmpdirname							)






				def     snake_case     (			self		:   str							)     ->				str:




										'''simple docstring'''



										_UpperCamelCase      : List[str]         =   '''<pad>'''
										_UpperCamelCase      : List[Any]         =   1

										self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCAmelCase__							),			lowerCAmelCase__							)
										self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCAmelCase__							),			lowerCAmelCase__							)






				def     snake_case     (			self		:   Any							)     ->				Optional[int]:




										'''simple docstring'''



										_UpperCamelCase      : int         =   list(self.get_tokenizer().get_vocab().keys()							)

										self.assertEqual(vocab_keys[0],			'''<s>'''							)
										self.assertEqual(vocab_keys[1],			'''<pad>'''							)
										self.assertEqual(vocab_keys[-1],			'''j'''							)
										self.assertEqual(len(lowerCAmelCase__							),			1_0_0_1							)






				def     snake_case     (			self		:   Any							)     ->				Dict:




										'''simple docstring'''



										self.assertEqual(self.get_tokenizer().vocab_size,			1_0_0_1							)






				def     snake_case     (			self		:   List[Any]							)     ->				Tuple:




										'''simple docstring'''



										_UpperCamelCase      : Optional[int]         =   SpeechaTextTokenizer.from_pretrained(self.tmpdirname							)

										_UpperCamelCase      : List[str]         =   tokenizer.tokenize('''This is a test'''							)
										self.assertListEqual(lowerCAmelCase__,			['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']							)

										self.assertListEqual(
										    tokenizer.convert_tokens_to_ids(lowerCAmelCase__							),			[2_8_9, 5_0, 1_4, 1_7_4, 3_8_6],			)

										_UpperCamelCase      : List[str]         =   tokenizer.tokenize('''I was born in 92000, and this is falsé.'''							)
										self.assertListEqual(
										    lowerCAmelCase__,			[SPIECE_UNDERLINE + '''I''', SPIECE_UNDERLINE + '''was''', SPIECE_UNDERLINE + '''b''', '''or''', '''n''', SPIECE_UNDERLINE + '''in''', SPIECE_UNDERLINE + '''''', '''9''', '''2''', '''0''', '''0''', '''0''', ''',''', SPIECE_UNDERLINE + '''and''', SPIECE_UNDERLINE + '''this''', SPIECE_UNDERLINE + '''is''', SPIECE_UNDERLINE + '''f''', '''al''', '''s''', '''é''', '''.'''],			)
										_UpperCamelCase      : Optional[Any]         =   tokenizer.convert_tokens_to_ids(lowerCAmelCase__							)
										self.assertListEqual(lowerCAmelCase__,			[1_2, 2_5, 8_8, 5_9, 2_8, 2_3, 1_1, 4, 6_0_6, 3_5_1, 3_5_1, 3_5_1, 7, 1_6, 7_0, 5_0, 7_6, 8_4, 1_0, 4, 8]							)

										_UpperCamelCase      : Union[str, Any]         =   tokenizer.convert_ids_to_tokens(lowerCAmelCase__							)
										self.assertListEqual(
										    lowerCAmelCase__,			[SPIECE_UNDERLINE + '''I''', SPIECE_UNDERLINE + '''was''', SPIECE_UNDERLINE + '''b''', '''or''', '''n''', SPIECE_UNDERLINE + '''in''', SPIECE_UNDERLINE + '''''', '''<unk>''', '''2''', '''0''', '''0''', '''0''', ''',''', SPIECE_UNDERLINE + '''and''', SPIECE_UNDERLINE + '''this''', SPIECE_UNDERLINE + '''is''', SPIECE_UNDERLINE + '''f''', '''al''', '''s''', '''<unk>''', '''.'''],			)






				@slow
				def     snake_case     (			self		:   List[str]							)     ->				Union[str, Any]:




										'''simple docstring'''



										_UpperCamelCase      : Any         =   {'''input_ids''': [[3_7_9_1, 7_9_7, 3_1, 1_1, 6_4, 7_9_7, 3_1, 2_4_2_9, 4_3_3, 1_2, 1_1_7_6, 1_2, 2_0, 7_8_6, 9_1_5, 1_4_2, 2_4_1_3, 2_4_0, 3_7, 3_2_3_8, 7_9_7, 3_1, 1_1, 3_5, 9_3, 9_1_5, 1_4_2, 2_4_1_3, 2_4_0, 3_7, 5_5_4_0, 5_6_7, 1_2_7_6, 9_3, 3_7, 6_1_0, 4_0, 6_2, 4_5_5, 6_5_7, 1_0_4_2, 1_2_3, 7_8_0, 1_7_7, 3_7, 3_0_9, 2_4_1, 1_2_9_8, 5_1_4, 2_0, 2_9_2, 2_7_3_7, 1_1_4, 2_4_6_9, 2_4_1, 8_5, 6_4, 3_0_2, 5_4_8, 5_2_8, 4_2_3, 4, 5_0_9, 4_0_6, 4_2_3, 3_7, 6_0_1, 4, 7_7_7, 3_0_2, 5_4_8, 5_2_8, 4_2_3, 2_8_4, 4, 3_3_8_8, 5_1_1, 4_5_9, 4, 3_5_5_5, 4_0, 3_2_1, 3_0_2, 7_0_5, 4, 3_3_8_8, 5_1_1, 5_8_3, 3_2_6, 5, 5, 5, 6_2, 3_3_1_0, 5_6_0, 1_7_7, 2_6_8_0, 2_1_7, 1_5_0_8, 3_2, 3_1, 8_5_3, 4_1_8, 6_4, 5_8_3, 5_1_1, 1_6_0_5, 6_2, 3_5, 9_3, 5_6_0, 1_7_7, 2_6_8_0, 2_1_7, 1_5_0_8, 1_5_2_1, 6_4, 5_8_3, 5_1_1, 5_1_9, 6_2, 2_0, 1_5_1_5, 7_6_4, 2_0, 1_4_9, 2_6_1, 5_6_2_5, 7_9_7_2, 2_0, 5_5_4_0, 5_6_7, 1_2_7_6, 9_3, 3_9_2_5, 1_6_7_5, 1_1, 1_5, 8_0_2, 7_9_7_2, 5_7_6, 2_1_7, 1_5_0_8, 1_1, 3_5, 9_3, 1_2_5_3, 2_4_4_1, 1_5, 2_8_9, 6_5_2, 3_1, 4_1_6, 3_2_1, 3_8_4_2, 1_1_5, 4_0, 9_1_1, 8, 4_7_6, 6_1_9, 4, 3_8_0, 1_4_2, 4_2_3, 3_3_5, 2_4_0, 3_5, 9_3, 2_6_4, 8, 1_1, 3_3_5, 5_6_9, 4_2_0, 1_6_3, 5, 2], [2_6_0, 5_4_8, 5_2_8, 4_2_3, 2_0, 4_5_1, 2_0, 2_6_8_1, 1_1_5_3, 3_4_3_4, 2_0, 5_5_4_0, 3_7, 5_6_7, 1_2_6, 1_2_5_3, 2_4_4_1, 3_3_7_6, 4_4_9, 2_1_0, 4_3_1, 1_5_6_3, 1_7_7, 7_6_7, 5_5_4_0, 1_1, 1_2_0_3, 4_7_2, 1_1, 2_9_5_3, 6_8_5, 2_8_5, 3_6_4, 7_0_6, 1_1_5_3, 2_0, 6_7_9_9, 2_0, 2_8_6_9, 2_0, 4_4_6_4, 1_2_6, 4_0, 2_4_2_9, 2_0, 1_0_4_0, 8_6_6, 2_6_6_4, 4_1_8, 2_0, 3_1_8, 2_0, 1_7_2_6, 1_8_6, 2_0, 2_6_5, 5_2_2, 3_5, 9_3, 2_1_9_1, 4_6_3_4, 2_0, 1_0_4_0, 1_2, 6_7_9_9, 1_5, 2_2_8, 2_3_5_6, 1_4_2, 3_1, 1_1, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [2_5_7_5, 2_6_6_6, 6_8_4, 1_5_8_2, 1_1_7_6, 1_2, 6_2_7, 1_4_9, 6_1_9, 2_0, 4_9_0_2, 5_6_3, 1_1, 2_0, 1_4_9, 2_6_1, 3_4_2_0, 2_3_5_6, 1_7_4, 1_4_2, 4_7_1_4, 1_3_1, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
										# fmt: on

										self.tokenizer_integration_test_util(
										    expected_encoding=lowerCAmelCase__,			model_name='''facebook/s2t-small-mustc-en-de-st''',			revision='''a14f04cf0776c02f62a8cb800cf7909e15ea23ad''',			)




@require_sentencepiece
class 			_a						(				unittest.TestCase    ):
				UpperCamelCase								=   '''valhalla/s2t_mustc_multilinguial_medium'''

				UpperCamelCase								=   '''C\'est trop cool'''
				UpperCamelCase								=   '''Esto es genial'''






				@classmethod
				def     snake_case     (			cls		:   Dict							)     ->				str:




										'''simple docstring'''



										_UpperCamelCase      : SpeechaTextTokenizer         =   SpeechaTextTokenizer.from_pretrained(cls.checkpoint_name							)
										return cls






				def     snake_case     (			self		:   Tuple							)     ->				List[str]:




										'''simple docstring'''



										self.assertEqual(self.tokenizer.lang_code_to_id['''pt'''],			4							)
										self.assertEqual(self.tokenizer.lang_code_to_id['''ru'''],			6							)
										self.assertEqual(self.tokenizer.lang_code_to_id['''it'''],			9							)
										self.assertEqual(self.tokenizer.lang_code_to_id['''de'''],			1_1							)






				def     snake_case     (			self		:   List[str]							)     ->				Union[str, Any]:




										'''simple docstring'''



										self.assertEqual(self.tokenizer.vocab_size,			1_0_0_0_0							)






				def     snake_case     (			self		:   Any							)     ->				Union[str, Any]:




										'''simple docstring'''



										self.assertIn(lowerCAmelCase__,			self.tokenizer.all_special_ids							)
										_UpperCamelCase      : List[Any]         =   [ES_CODE, 4, 1_6_0_1, 4_7, 7_6_4_7, 2]
										_UpperCamelCase      : Optional[Any]         =   self.tokenizer.decode(lowerCAmelCase__,			skip_special_tokens=lowerCAmelCase__							)
										_UpperCamelCase      : Optional[int]         =   self.tokenizer.decode(generated_ids[1:],			skip_special_tokens=lowerCAmelCase__							)
										self.assertEqual(lowerCAmelCase__,			lowerCAmelCase__							)
										self.assertNotIn(self.tokenizer.eos_token,			lowerCAmelCase__							)






				def     snake_case     (			self		:   List[str]							)     ->				List[str]:




										'''simple docstring'''



										_UpperCamelCase      : Dict         =   '''fr'''
										_UpperCamelCase      : int         =   self.tokenizer(self.french_text							).input_ids
										self.assertEqual(encoded[0],			lowerCAmelCase__							)
										self.assertEqual(encoded[-1],			self.tokenizer.eos_token_id							)






				def     snake_case     (			self		:   Union[str, Any]							)     ->				Union[str, Any]:




										'''simple docstring'''



										_UpperCamelCase      : Dict         =   '''fr'''
										self.assertListEqual(self.tokenizer.prefix_tokens,			[FR_CODE]							)

										_UpperCamelCase      : List[str]         =   '''es'''
										self.assertListEqual(self.tokenizer.prefix_tokens,			[ES_CODE]							)



