





"""simple docstring"""




import logging
import re

import pytorch_quantization
import pytorch_quantization.nn as quant_nn
import torch
from pytorch_quantization import calib
from pytorch_quantization.tensor_quant import QuantDescriptor


UpperCamelCase_									=logging.getLogger(__name__)

UpperCamelCase_									=50  # max width of layer names
UpperCamelCase_									=70  # max width of quantizer names





def 	a_    (							_lowercase       ):
      _UpperCamelCase      : int         =   parser.add_argument_group('''quant_trainer arguments'''       )
      group.add_argument('''--wprec'''	,						type=_lowercase	,						default=8	,						help='''weight precision'''       )
      group.add_argument('''--aprec'''	,						type=_lowercase	,						default=8	,						help='''activation precision'''       )
      group.add_argument('''--quant-per-tensor'''	,						action='''store_true'''	,						help='''per tensor weight scaling'''       )
      group.add_argument('''--quant-disable'''	,						action='''store_true'''	,						help='''disable all quantizers'''       )
      group.add_argument('''--quant-disable-embeddings'''	,						action='''store_true'''	,						help='''disable all embeddings quantizers'''       )
      group.add_argument('''--quant-disable-keyword'''	,						type=_lowercase	,						nargs='''+'''	,						help='''disable quantizers by keyword'''       )
      group.add_argument('''--quant-disable-layer-module'''	,						type=_lowercase	,						help='''disable quantizers by keyword under layer.'''       )
      group.add_argument('''--quant-enable-layer-module'''	,						type=_lowercase	,						help='''enable quantizers by keyword under layer'''       )
      group.add_argument('''--calibrator'''	,						default='''max'''	,						help='''which quantization range calibrator to use'''       )
      group.add_argument('''--percentile'''	,						default=_lowercase	,						type=_lowercase	,						help='''percentile for PercentileCalibrator'''       )
      group.add_argument('''--fuse-qkv'''	,						action='''store_true'''	,						help='''use the same scale factor for qkv'''       )
      group.add_argument('''--clip-gelu'''	,						metavar='''N'''	,						type=_lowercase	,						help='''clip gelu output maximum value to N'''       )
      group.add_argument(
          '''--recalibrate-weights'''	,						action='''store_true'''	,						help=(
              '''recalibrate weight amaxes by taking the max of the weights.'''
              ''' amaxes will be computed with the current quantization granularity (axis).'''
          )	,						)





def 	a_    (							_lowercase       ):

      if args.calibrator == "max":
            _UpperCamelCase      : Dict         =   '''max'''
      elif args.calibrator == "percentile":
            if args.percentile is None:
                  raise ValueError('''Specify --percentile when using percentile calibrator'''       )
            _UpperCamelCase      : int         =   '''histogram'''
      elif args.calibrator == "mse":
            _UpperCamelCase      : Optional[int]         =   '''histogram'''
      else:
            raise ValueError(F"""Invalid calibrator {args.calibrator}"""       )

      _UpperCamelCase      : str         =   QuantDescriptor(num_bits=args.aprec	,						calib_method=_lowercase       )
      _UpperCamelCase      : int         =   QuantDescriptor(num_bits=args.wprec	,						axis=(None if args.quant_per_tensor else (0,))       )
      quant_nn.QuantLinear.set_default_quant_desc_input(_lowercase       )
      quant_nn.QuantLinear.set_default_quant_desc_weight(_lowercase       )





def 	a_    (							_lowercase	,						_lowercase	,						_lowercase=False	,						_lowercase=False       ):
      logger.info('''Configuring Model for Quantization'''       )
      logger.info(F"""using quantization package {pytorch_quantization.__file__}"""       )

      if not calib:
            if args.quant_disable_embeddings:
                  set_quantizer_by_name(_lowercase	,						['''embeddings''']	,						which='''weight'''	,						_disabled=_lowercase       )

            if args.quant_disable:
                  set_quantizer_by_name(_lowercase	,						['''''']	,						_disabled=_lowercase       )

            if args.quant_disable_keyword:
                  set_quantizer_by_name(_lowercase	,						args.quant_disable_keyword	,						_disabled=_lowercase       )

            if args.quant_disable_layer_module:
                  set_quantizer_by_name(_lowercase	,						[R'''layer.\d+.''' + args.quant_disable_layer_module]	,						_disabled=_lowercase       )

            if args.quant_enable_layer_module:
                  set_quantizer_by_name(_lowercase	,						[R'''layer.\d+.''' + args.quant_enable_layer_module]	,						_disabled=_lowercase       )

            if args.recalibrate_weights:
                  recalibrate_weights(_lowercase       )

            if args.fuse_qkv:
                  fuse_qkv(_lowercase	,						_lowercase       )

      if args.clip_gelu:
            clip_gelu(_lowercase	,						args.clip_gelu       )

      # if args.local_rank in [-1, 0] and not calib:
      print_quant_summary(_lowercase       )





def 	a_    (							_lowercase       ):
      logger.info('''Enabling Calibration'''       )
      for name, module in model.named_modules():
            if name.endswith('''_quantizer'''       ):
                  if module._calibrator is not None:
                        module.disable_quant()
                        module.enable_calib()
                  else:
                        module.disable()
                  logger.info(F"""{name:80}: {module}"""       )





def 	a_    (							_lowercase	,						_lowercase       ):
      logger.info('''Loading calibrated amax'''       )
      for name, module in model.named_modules():
            if name.endswith('''_quantizer'''       ):
                  if module._calibrator is not None:
                        if isinstance(module._calibrator	,						calib.MaxCalibrator       ):
                              module.load_calib_amax()
                        else:
                              module.load_calib_amax('''percentile'''	,						percentile=args.percentile       )
                        module.enable_quant()
                        module.disable_calib()
                  else:
                        module.enable()
      model.cuda()
      print_quant_summary(_lowercase       )





def 	a_    (							_lowercase	,						_lowercase       ):

      def fusea(_lowercase	,						_lowercase	,						_lowercase       ):
            for mod in [qq, qk, qv]:
                  if not hasattr(_lowercase	,						'''_amax'''       ):
                        print('''          WARNING: NO AMAX BUFFER'''       )
                        return
            _UpperCamelCase      : str         =   qq._amax.detach().item()
            _UpperCamelCase      : Tuple         =   qk._amax.detach().item()
            _UpperCamelCase      : Union[str, Any]         =   qv._amax.detach().item()

            _UpperCamelCase      : List[str]         =   max(_lowercase	,						_lowercase	,						_lowercase       )
            qq._amax.fill_(_lowercase       )
            qk._amax.fill_(_lowercase       )
            qv._amax.fill_(_lowercase       )
            logger.info(F"""          q={q:5.2f} k={k:5.2f} v={v:5.2f} -> {amax:5.2f}"""       )

      for name, mod in model.named_modules():
            if name.endswith('''.attention.self'''       ):
                  logger.info(F"""FUSE_QKV: {name:{name_width}}"""       )
                  fusea(mod.matmul_q_input_quantizer	,						mod.matmul_k_input_quantizer	,						mod.matmul_v_input_quantizer       )
                  if args.quant_per_tensor:
                        fusea(mod.query._weight_quantizer	,						mod.key._weight_quantizer	,						mod.value._weight_quantizer       )





def 	a_    (							_lowercase	,						_lowercase       ):

      for name, mod in model.named_modules():
            if name.endswith('''.output.dense'''       ) and not name.endswith('''attention.output.dense'''       ):
                  _UpperCamelCase      : List[str]         =   mod._input_quantizer._amax.data.detach().item()
                  mod._input_quantizer._amax.data.detach().clamp_(max=_lowercase       )
                  _UpperCamelCase      : List[Any]         =   mod._input_quantizer._amax.data.detach().item()
                  logger.info(F"""CLIP_GELU: {name:{name_width}} amax: {amax_init:5.2f} -> {amax:5.2f}"""       )





def 	a_    (							_lowercase       ):

      for name, mod in model.named_modules():
            if hasattr(_lowercase	,						'''_weight_quantizer'''       ) and mod._weight_quantizer.axis is not None:
                  _UpperCamelCase      : str         =   mod.weight.shape[0]
                  _UpperCamelCase      : Union[str, Any]         =   mod._weight_quantizer._amax.detach()
                  _UpperCamelCase      : Dict         =   torch.ones(_lowercase	,						dtype=amax.dtype	,						device=amax.device       ) * amax
                  print(F"""expanding {name} {amax} -> {mod._weight_quantizer._amax}"""       )





def 	a_    (							_lowercase       ):

      for name, mod in model.named_modules():
            if hasattr(_lowercase	,						'''_weight_quantizer'''       ):
                  if not hasattr(mod.weight_quantizer	,						'''_amax'''       ):
                        print('''RECALIB: {name:{name_width}} WARNING: NO AMAX BUFFER'''       )
                        continue

                  # determine which axes to reduce across
                  # e.g. a 4D tensor quantized per axis 0 should reduce over (1,2,3)
                  _UpperCamelCase      : Tuple         =   set() if mod._weight_quantizer.axis is None else set(mod._weight_quantizer.axis       )
                  _UpperCamelCase      : List[Any]         =   set(range(len(mod.weight.size()       )       )       ) - axis_set
                  _UpperCamelCase      : Any         =   pytorch_quantization.utils.reduce_amax(mod.weight	,						axis=_lowercase	,						keepdims=_lowercase       ).detach()
                  logger.info(F"""RECALIB: {name:{name_width}} {mod._weight_quantizer._amax.flatten()} -> {amax.flatten()}"""       )
                  _UpperCamelCase      : Tuple         =   amax





def 	a_    (							_lowercase	,						_lowercase=25	,						_lowercase=180	,						_lowercase=None       ):

      if ignore is None:
            _UpperCamelCase      : int         =   []
      elif not isinstance(_lowercase	,						_lowercase       ):
            _UpperCamelCase      : Optional[Any]         =   [ignore]

      _UpperCamelCase      : List[Any]         =   0
      for name, mod in model.named_modules():
            if not hasattr(_lowercase	,						'''weight'''       ):
                  continue
            _UpperCamelCase      : int         =   max(_lowercase	,						len(_lowercase       )       )

      for name, mod in model.named_modules():
            _UpperCamelCase      : str         =   getattr(_lowercase	,						'''_input_quantizer'''	,						_lowercase       )
            _UpperCamelCase      : List[Any]         =   getattr(_lowercase	,						'''_weight_quantizer'''	,						_lowercase       )
            if not hasattr(_lowercase	,						'''weight'''       ):
                  continue
            if type(_lowercase       ) in ignore:
                  continue
            if [True for s in ignore if type(_lowercase       ) is str and s in name]:
                  continue
            _UpperCamelCase      : int         =   F"""Act:{input_q.extra_repr()}"""
            _UpperCamelCase      : Dict         =   F"""Wgt:{weight_q.extra_repr()}"""
            _UpperCamelCase      : int         =   F"""{name:{name_width}} {act_str} {wgt_str}"""
            if len(_lowercase       ) <= line_width:
                  logger.info(_lowercase       )
            else:
                  logger.info(F"""{name:{name_width}} {act_str}"""       )
                  logger.info(F"""{'  ':{name_width}} {wgt_str}"""       )





def 	a_    (							_lowercase       ):
      _UpperCamelCase      : Optional[int]         =   0
      for name, mod in model.named_modules():
            if isinstance(_lowercase	,						pytorch_quantization.nn.TensorQuantizer       ):
                  print(F"""{name:80} {mod}"""       )
                  count += 1
      print(F"""{count} TensorQuantizers found in model"""       )





def 	a_    (							_lowercase	,						_lowercase	,						_lowercase	,						_lowercase	,						_lowercase       ):
      _UpperCamelCase      : Optional[int]         =   getattr(_lowercase	,						_lowercase	,						_lowercase       )
      if quantizer_mod is not None:
            assert hasattr(_lowercase	,						_lowercase       )
            setattr(_lowercase	,						_lowercase	,						_lowercase       )
      else:
            logger.warning(F"""{name} has no {quantizer}"""       )





def 	a_    (							_lowercase	,						_lowercase	,						_lowercase="both"	,						**_lowercase       ):
      _UpperCamelCase      : List[str]         =   F"""Warning: changing {which} quantizers of {name:{qname_width}}"""
      for k, v in kwargs.items():
            s += F""" {k}={v}"""
            if which in ["input", "both"]:
                  set_quantizer(_lowercase	,						_lowercase	,						'''_input_quantizer'''	,						_lowercase	,						_lowercase       )
            if which in ["weight", "both"]:
                  set_quantizer(_lowercase	,						_lowercase	,						'''_weight_quantizer'''	,						_lowercase	,						_lowercase       )
      logger.info(_lowercase       )






def 	a_    (							_lowercase	,						_lowercase	,						**_lowercase       ):

      for name, mod in model.named_modules():
            if hasattr(_lowercase	,						'''_input_quantizer'''       ) or hasattr(_lowercase	,						'''_weight_quantizer'''       ):
                  for n in names:
                        if re.search(_lowercase	,						_lowercase       ):
                              set_quantizers(_lowercase	,						_lowercase	,						**_lowercase       )
            elif name.endswith('''_quantizer'''       ):
                  for n in names:
                        if re.search(_lowercase	,						_lowercase       ):
                              _UpperCamelCase      : Dict         =   F"""Warning: changing {name:{name_width}}"""
                              for k, v in kwargs.items():
                                    s += F""" {k}={v}"""
                                    setattr(_lowercase	,						_lowercase	,						_lowercase       )
                              logger.info(_lowercase       )










"""simple docstring"""




import unittest
from pathlib import Path
from shutil import copyfile

from transformers import SPIECE_UNDERLINE, is_sentencepiece_available
from transformers.models.speech_to_text import SpeechaTextTokenizer
from transformers.models.speech_to_text.tokenization_speech_to_text import VOCAB_FILES_NAMES, save_json
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


UpperCamelCase_									=get_tests_dir("""fixtures/test_sentencepiece.model""")

if is_sentencepiece_available():
						import sentencepiece as sp


UpperCamelCase_									=5
UpperCamelCase_									=10




@require_sentencepiece
@require_tokenizers
class 			_a						(				_lowerCAmelCase     ,    unittest.TestCase    ):
				UpperCamelCase								=   SpeechaTextTokenizer
				UpperCamelCase								=   False
				UpperCamelCase								=   True






				def     snake_case     (			self		:   Tuple							)     ->				Optional[Any]:




										'''simple docstring'''



										super().setUp()

										_UpperCamelCase      : Dict         =   sp.SentencePieceProcessor()
										spm_model.Load(lowerCAmelCase__							)
										_UpperCamelCase      : Union[str, Any]         =   ['''<s>''', '''<pad>''', '''</s>''', '''<unk>''']

										vocab += [spm_model.IdToPiece(id_							) for id_ in range(len(lowerCAmelCase__							)							)]
										_UpperCamelCase      : Tuple         =   dict(zip(lowerCAmelCase__,			range(len(lowerCAmelCase__							)							)							)							)

										_UpperCamelCase      : Dict         =   Path(self.tmpdirname							)
										save_json(lowerCAmelCase__,			save_dir / VOCAB_FILES_NAMES['''vocab_file''']							)
										if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
																copyfile(lowerCAmelCase__,			save_dir / VOCAB_FILES_NAMES['''spm_file''']							)

										_UpperCamelCase      : List[str]         =   SpeechaTextTokenizer.from_pretrained(self.tmpdirname							)
										tokenizer.save_pretrained(self.tmpdirname							)






				def     snake_case     (			self		:   str							)     ->				str:




										'''simple docstring'''



										_UpperCamelCase      : List[str]         =   '''<pad>'''
										_UpperCamelCase      : List[Any]         =   1

										self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCAmelCase__							),			lowerCAmelCase__							)
										self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCAmelCase__							),			lowerCAmelCase__							)






				def     snake_case     (			self		:   Any							)     ->				Optional[int]:




										'''simple docstring'''



										_UpperCamelCase      : int         =   list(self.get_tokenizer().get_vocab().keys()							)

										self.assertEqual(vocab_keys[0],			'''<s>'''							)
										self.assertEqual(vocab_keys[1],			'''<pad>'''							)
										self.assertEqual(vocab_keys[-1],			'''j'''							)
										self.assertEqual(len(lowerCAmelCase__							),			1_0_0_1							)






				def     snake_case     (			self		:   Any							)     ->				Dict:




										'''simple docstring'''



										self.assertEqual(self.get_tokenizer().vocab_size,			1_0_0_1							)






				def     snake_case     (			self		:   List[Any]							)     ->				Tuple:




										'''simple docstring'''



										_UpperCamelCase      : Optional[int]         =   SpeechaTextTokenizer.from_pretrained(self.tmpdirname							)

										_UpperCamelCase      : List[str]         =   tokenizer.tokenize('''This is a test'''							)
										self.assertListEqual(lowerCAmelCase__,			['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']							)

										self.assertListEqual(
										    tokenizer.convert_tokens_to_ids(lowerCAmelCase__							),			[2_8_9, 5_0, 1_4, 1_7_4, 3_8_6],			)

										_UpperCamelCase      : List[str]         =   tokenizer.tokenize('''I was born in 92000, and this is falsé.'''							)
										self.assertListEqual(
										    lowerCAmelCase__,			[SPIECE_UNDERLINE + '''I''', SPIECE_UNDERLINE + '''was''', SPIECE_UNDERLINE + '''b''', '''or''', '''n''', SPIECE_UNDERLINE + '''in''', SPIECE_UNDERLINE + '''''', '''9''', '''2''', '''0''', '''0''', '''0''', ''',''', SPIECE_UNDERLINE + '''and''', SPIECE_UNDERLINE + '''this''', SPIECE_UNDERLINE + '''is''', SPIECE_UNDERLINE + '''f''', '''al''', '''s''', '''é''', '''.'''],			)
										_UpperCamelCase      : Optional[Any]         =   tokenizer.convert_tokens_to_ids(lowerCAmelCase__							)
										self.assertListEqual(lowerCAmelCase__,			[1_2, 2_5, 8_8, 5_9, 2_8, 2_3, 1_1, 4, 6_0_6, 3_5_1, 3_5_1, 3_5_1, 7, 1_6, 7_0, 5_0, 7_6, 8_4, 1_0, 4, 8]							)

										_UpperCamelCase      : Union[str, Any]         =   tokenizer.convert_ids_to_tokens(lowerCAmelCase__							)
										self.assertListEqual(
										    lowerCAmelCase__,			[SPIECE_UNDERLINE + '''I''', SPIECE_UNDERLINE + '''was''', SPIECE_UNDERLINE + '''b''', '''or''', '''n''', SPIECE_UNDERLINE + '''in''', SPIECE_UNDERLINE + '''''', '''<unk>''', '''2''', '''0''', '''0''', '''0''', ''',''', SPIECE_UNDERLINE + '''and''', SPIECE_UNDERLINE + '''this''', SPIECE_UNDERLINE + '''is''', SPIECE_UNDERLINE + '''f''', '''al''', '''s''', '''<unk>''', '''.'''],			)






				@slow
				def     snake_case     (			self		:   List[str]							)     ->				Union[str, Any]:




										'''simple docstring'''



										_UpperCamelCase      : Any         =   {'''input_ids''': [[3_7_9_1, 7_9_7, 3_1, 1_1, 6_4, 7_9_7, 3_1, 2_4_2_9, 4_3_3, 1_2, 1_1_7_6, 1_2, 2_0, 7_8_6, 9_1_5, 1_4_2, 2_4_1_3, 2_4_0, 3_7, 3_2_3_8, 7_9_7, 3_1, 1_1, 3_5, 9_3, 9_1_5, 1_4_2, 2_4_1_3, 2_4_0, 3_7, 5_5_4_0, 5_6_7, 1_2_7_6, 9_3, 3_7, 6_1_0, 4_0, 6_2, 4_5_5, 6_5_7, 1_0_4_2, 1_2_3, 7_8_0, 1_7_7, 3_7, 3_0_9, 2_4_1, 1_2_9_8, 5_1_4, 2_0, 2_9_2, 2_7_3_7, 1_1_4, 2_4_6_9, 2_4_1, 8_5, 6_4, 3_0_2, 5_4_8, 5_2_8, 4_2_3, 4, 5_0_9, 4_0_6, 4_2_3, 3_7, 6_0_1, 4, 7_7_7, 3_0_2, 5_4_8, 5_2_8, 4_2_3, 2_8_4, 4, 3_3_8_8, 5_1_1, 4_5_9, 4, 3_5_5_5, 4_0, 3_2_1, 3_0_2, 7_0_5, 4, 3_3_8_8, 5_1_1, 5_8_3, 3_2_6, 5, 5, 5, 6_2, 3_3_1_0, 5_6_0, 1_7_7, 2_6_8_0, 2_1_7, 1_5_0_8, 3_2, 3_1, 8_5_3, 4_1_8, 6_4, 5_8_3, 5_1_1, 1_6_0_5, 6_2, 3_5, 9_3, 5_6_0, 1_7_7, 2_6_8_0, 2_1_7, 1_5_0_8, 1_5_2_1, 6_4, 5_8_3, 5_1_1, 5_1_9, 6_2, 2_0, 1_5_1_5, 7_6_4, 2_0, 1_4_9, 2_6_1, 5_6_2_5, 7_9_7_2, 2_0, 5_5_4_0, 5_6_7, 1_2_7_6, 9_3, 3_9_2_5, 1_6_7_5, 1_1, 1_5, 8_0_2, 7_9_7_2, 5_7_6, 2_1_7, 1_5_0_8, 1_1, 3_5, 9_3, 1_2_5_3, 2_4_4_1, 1_5, 2_8_9, 6_5_2, 3_1, 4_1_6, 3_2_1, 3_8_4_2, 1_1_5, 4_0, 9_1_1, 8, 4_7_6, 6_1_9, 4, 3_8_0, 1_4_2, 4_2_3, 3_3_5, 2_4_0, 3_5, 9_3, 2_6_4, 8, 1_1, 3_3_5, 5_6_9, 4_2_0, 1_6_3, 5, 2], [2_6_0, 5_4_8, 5_2_8, 4_2_3, 2_0, 4_5_1, 2_0, 2_6_8_1, 1_1_5_3, 3_4_3_4, 2_0, 5_5_4_0, 3_7, 5_6_7, 1_2_6, 1_2_5_3, 2_4_4_1, 3_3_7_6, 4_4_9, 2_1_0, 4_3_1, 1_5_6_3, 1_7_7, 7_6_7, 5_5_4_0, 1_1, 1_2_0_3, 4_7_2, 1_1, 2_9_5_3, 6_8_5, 2_8_5, 3_6_4, 7_0_6, 1_1_5_3, 2_0, 6_7_9_9, 2_0, 2_8_6_9, 2_0, 4_4_6_4, 1_2_6, 4_0, 2_4_2_9, 2_0, 1_0_4_0, 8_6_6, 2_6_6_4, 4_1_8, 2_0, 3_1_8, 2_0, 1_7_2_6, 1_8_6, 2_0, 2_6_5, 5_2_2, 3_5, 9_3, 2_1_9_1, 4_6_3_4, 2_0, 1_0_4_0, 1_2, 6_7_9_9, 1_5, 2_2_8, 2_3_5_6, 1_4_2, 3_1, 1_1, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [2_5_7_5, 2_6_6_6, 6_8_4, 1_5_8_2, 1_1_7_6, 1_2, 6_2_7, 1_4_9, 6_1_9, 2_0, 4_9_0_2, 5_6_3, 1_1, 2_0, 1_4_9, 2_6_1, 3_4_2_0, 2_3_5_6, 1_7_4, 1_4_2, 4_7_1_4, 1_3_1, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
										# fmt: on

										self.tokenizer_integration_test_util(
										    expected_encoding=lowerCAmelCase__,			model_name='''facebook/s2t-small-mustc-en-de-st''',			revision='''a14f04cf0776c02f62a8cb800cf7909e15ea23ad''',			)




@require_sentencepiece
class 			_a						(				unittest.TestCase    ):
				UpperCamelCase								=   '''valhalla/s2t_mustc_multilinguial_medium'''

				UpperCamelCase								=   '''C\'est trop cool'''
				UpperCamelCase								=   '''Esto es genial'''






				@classmethod
				def     snake_case     (			cls		:   Dict							)     ->				str:




										'''simple docstring'''



										_UpperCamelCase      : SpeechaTextTokenizer         =   SpeechaTextTokenizer.from_pretrained(cls.checkpoint_name							)
										return cls






				def     snake_case     (			self		:   Tuple							)     ->				List[str]:




										'''simple docstring'''



										self.assertEqual(self.tokenizer.lang_code_to_id['''pt'''],			4							)
										self.assertEqual(self.tokenizer.lang_code_to_id['''ru'''],			6							)
										self.assertEqual(self.tokenizer.lang_code_to_id['''it'''],			9							)
										self.assertEqual(self.tokenizer.lang_code_to_id['''de'''],			1_1							)






				def     snake_case     (			self		:   List[str]							)     ->				Union[str, Any]:




										'''simple docstring'''



										self.assertEqual(self.tokenizer.vocab_size,			1_0_0_0_0							)






				def     snake_case     (			self		:   Any							)     ->				Union[str, Any]:




										'''simple docstring'''



										self.assertIn(lowerCAmelCase__,			self.tokenizer.all_special_ids							)
										_UpperCamelCase      : List[Any]         =   [ES_CODE, 4, 1_6_0_1, 4_7, 7_6_4_7, 2]
										_UpperCamelCase      : Optional[Any]         =   self.tokenizer.decode(lowerCAmelCase__,			skip_special_tokens=lowerCAmelCase__							)
										_UpperCamelCase      : Optional[int]         =   self.tokenizer.decode(generated_ids[1:],			skip_special_tokens=lowerCAmelCase__							)
										self.assertEqual(lowerCAmelCase__,			lowerCAmelCase__							)
										self.assertNotIn(self.tokenizer.eos_token,			lowerCAmelCase__							)






				def     snake_case     (			self		:   List[str]							)     ->				List[str]:




										'''simple docstring'''



										_UpperCamelCase      : Dict         =   '''fr'''
										_UpperCamelCase      : int         =   self.tokenizer(self.french_text							).input_ids
										self.assertEqual(encoded[0],			lowerCAmelCase__							)
										self.assertEqual(encoded[-1],			self.tokenizer.eos_token_id							)






				def     snake_case     (			self		:   Union[str, Any]							)     ->				Union[str, Any]:




										'''simple docstring'''



										_UpperCamelCase      : Dict         =   '''fr'''
										self.assertListEqual(self.tokenizer.prefix_tokens,			[FR_CODE]							)

										_UpperCamelCase      : List[str]         =   '''es'''
										self.assertListEqual(self.tokenizer.prefix_tokens,			[ES_CODE]							)



