

'''simple docstring'''




import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, MBartTokenizer, MBartTokenizerFast, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


_lowerCAmelCase         =						get_tests_dir("fixtures/test_sentencepiece.model")


if is_torch_available():
						from transformers.models.mbart.modeling_mbart import shift_tokens_right

_lowerCAmelCase         =						250004
_lowerCAmelCase         =						250020






@require_sentencepiece
@require_tokenizers
class   _SCREAMING_SNAKE_CASE   (   __a		,unittest.TestCase     ):
	__SCREAMING_SNAKE_CASE					:List[Any]      	=					MBartTokenizer
	__SCREAMING_SNAKE_CASE					:Tuple      	=					MBartTokenizerFast
	__SCREAMING_SNAKE_CASE					:Dict      	=					True
	__SCREAMING_SNAKE_CASE					:Optional[Any]      	=					True



	def 			snake_case__  (   self		:   int      ):
						super().setUp()

						# We have a SentencePiece fixture for testing
						__magic_name__						       =    MBartTokenizer(a__			,				keep_accents=a__      )
						tokenizer.save_pretrained(self.tmpdirname      )



	def 			snake_case__  (   self		:   Optional[Any]      ):
						__magic_name__						       =    MBartTokenizer(a__			,				keep_accents=a__      )

						__magic_name__						       =    tokenizer.tokenize('''This is a test'''      )
						self.assertListEqual(a__			,				['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']      )

						self.assertListEqual(
						    tokenizer.convert_tokens_to_ids(a__      )			,				[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]			,				)

						__magic_name__						       =    tokenizer.tokenize('''I was born in 92000, and this is falsé.'''      )
						self.assertListEqual(
						    a__			,				[
						        SPIECE_UNDERLINE + '''I''',
						        SPIECE_UNDERLINE + '''was''',
						        SPIECE_UNDERLINE + '''b''',
						        '''or''',
						        '''n''',
						        SPIECE_UNDERLINE + '''in''',
						        SPIECE_UNDERLINE + '''''',
						        '''9''',
						        '''2''',
						        '''0''',
						        '''0''',
						        '''0''',
						        ''',''',
						        SPIECE_UNDERLINE + '''and''',
						        SPIECE_UNDERLINE + '''this''',
						        SPIECE_UNDERLINE + '''is''',
						        SPIECE_UNDERLINE + '''f''',
						        '''al''',
						        '''s''',
						        '''é''',
						        '''.''',
						    ]			,				)
						__magic_name__						       =    tokenizer.convert_tokens_to_ids(a__      )
						self.assertListEqual(
						    a__			,				[
						        value + tokenizer.fairseq_offset
						        for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
						        #                                       ^ unk: 2 + 1 = 3                  unk: 2 + 1 = 3 ^
						    ]			,				)

						__magic_name__						       =    tokenizer.convert_ids_to_tokens(a__      )
						self.assertListEqual(
						    a__			,				[
						        SPIECE_UNDERLINE + '''I''',
						        SPIECE_UNDERLINE + '''was''',
						        SPIECE_UNDERLINE + '''b''',
						        '''or''',
						        '''n''',
						        SPIECE_UNDERLINE + '''in''',
						        SPIECE_UNDERLINE + '''''',
						        '''<unk>''',
						        '''2''',
						        '''0''',
						        '''0''',
						        '''0''',
						        ''',''',
						        SPIECE_UNDERLINE + '''and''',
						        SPIECE_UNDERLINE + '''this''',
						        SPIECE_UNDERLINE + '''is''',
						        SPIECE_UNDERLINE + '''f''',
						        '''al''',
						        '''s''',
						        '''<unk>''',
						        '''.''',
						    ]			,				)



	def 			snake_case__  (   self		:   Any      ):
						if not self.test_slow_tokenizer:
											# as we don't have a slow version, we can't compare the outputs between slow and fast versions
											return

						__magic_name__						       =    (self.rust_tokenizer_class, '''hf-internal-testing/tiny-random-mbart''', {})
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''      ):
																__magic_name__						       =    self.rust_tokenizer_class.from_pretrained(a__			,				**a__      )
																__magic_name__						       =    self.tokenizer_class.from_pretrained(a__			,				**a__      )

																__magic_name__						       =    tempfile.mkdtemp()

																__magic_name__						       =    tokenizer_r.save_pretrained(a__      )
																__magic_name__						       =    tokenizer_p.save_pretrained(a__      )

																# Checks it save with the same files + the tokenizer.json file for the fast one
																self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files      )      )
																__magic_name__						       =    tuple(f for f in tokenizer_r_files if '''tokenizer.json''' not in f      )
																self.assertSequenceEqual(a__			,				a__      )

																# Checks everything loads correctly in the same way
																__magic_name__						       =    tokenizer_r.from_pretrained(a__      )
																__magic_name__						       =    tokenizer_p.from_pretrained(a__      )

																# Check special tokens are set accordingly on Rust and Python
																for key in tokenizer_pp.special_tokens_map:
																					self.assertTrue(hasattr(a__			,				a__      )      )
																					# self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
																					# self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

																shutil.rmtree(a__      )

																# Save tokenizer rust, legacy_format=True
																__magic_name__						       =    tempfile.mkdtemp()

																__magic_name__						       =    tokenizer_r.save_pretrained(a__			,				legacy_format=a__      )
																__magic_name__						       =    tokenizer_p.save_pretrained(a__      )

																# Checks it save with the same files
																self.assertSequenceEqual(a__			,				a__      )

																# Checks everything loads correctly in the same way
																__magic_name__						       =    tokenizer_r.from_pretrained(a__      )
																__magic_name__						       =    tokenizer_p.from_pretrained(a__      )

																# Check special tokens are set accordingly on Rust and Python
																for key in tokenizer_pp.special_tokens_map:
																					self.assertTrue(hasattr(a__			,				a__      )      )

																shutil.rmtree(a__      )

																# Save tokenizer rust, legacy_format=False
																__magic_name__						       =    tempfile.mkdtemp()

																__magic_name__						       =    tokenizer_r.save_pretrained(a__			,				legacy_format=a__      )
																__magic_name__						       =    tokenizer_p.save_pretrained(a__      )

																# Checks it saved the tokenizer.json file
																self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files      )      )

																# Checks everything loads correctly in the same way
																__magic_name__						       =    tokenizer_r.from_pretrained(a__      )
																__magic_name__						       =    tokenizer_p.from_pretrained(a__      )

																# Check special tokens are set accordingly on Rust and Python
																for key in tokenizer_pp.special_tokens_map:
																					self.assertTrue(hasattr(a__			,				a__      )      )

																shutil.rmtree(a__      )






@require_torch
@require_sentencepiece
@require_tokenizers
class   _SCREAMING_SNAKE_CASE   (   unittest.TestCase     ):
	__SCREAMING_SNAKE_CASE					:int      	=					"""facebook/mbart-large-en-ro"""
	__SCREAMING_SNAKE_CASE					:int      	=					[
	    """ UN Chief Says There Is No Military Solution in Syria""",
	    """ Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.""",
	]
	__SCREAMING_SNAKE_CASE					:Dict      	=					[
	    """Şeful ONU declară că nu există o soluţie militară în Siria""",
	    """Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"""
	    """ pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"""
	    """ face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.""",
	]
	__SCREAMING_SNAKE_CASE					:List[Any]      	=					[8274, 127873, 25916, 7, 8622, 2071, 438, 67485, 53, 187895, 23, 51712, 2, EN_CODE]



	@classmethod
	def 			snake_case__  (   cls		:   Tuple      ):
						__magic_name__						       =    MBartTokenizer.from_pretrained(
						    cls.checkpoint_name			,				src_lang='''en_XX'''			,				tgt_lang='''ro_RO'''      )
						__magic_name__						       =    1
						return cls



	def 			snake_case__  (   self		:   int      ):
						self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ar_AR''']			,				25_0001      )
						self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''en_EN''']			,				25_0004      )
						self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ro_RO''']			,				25_0020      )



	def 			snake_case__  (   self		:   Union[str, Any]      ):
						__magic_name__						       =    self.tokenizer.batch_encode_plus(self.src_text      ).input_ids[0]
						self.assertListEqual(self.expected_src_tokens			,				a__      )



	def 			snake_case__  (   self		:   Dict      ):
						self.assertIn(a__			,				self.tokenizer.all_special_ids      )
						__magic_name__						       =    [RO_CODE, 884, 9019, 96, 9, 916, 8_6792, 36, 1_8743, 1_5596, 5, 2]
						__magic_name__						       =    self.tokenizer.decode(a__			,				skip_special_tokens=a__      )
						__magic_name__						       =    self.tokenizer.decode(generated_ids[1:]			,				skip_special_tokens=a__      )
						self.assertEqual(a__			,				a__      )
						self.assertNotIn(self.tokenizer.eos_token			,				a__      )



	def 			snake_case__  (   self		:   List[str]      ):
						__magic_name__						       =    ['''this is gunna be a long sentence ''' * 20]
						assert isinstance(src_text[0]			,				a__      )
						__magic_name__						       =    10
						__magic_name__						       =    self.tokenizer(a__			,				max_length=a__			,				truncation=a__      ).input_ids[0]
						self.assertEqual(ids[-2]			,				2      )
						self.assertEqual(ids[-1]			,				a__      )
						self.assertEqual(len(a__      )			,				a__      )



	def 			snake_case__  (   self		:   Dict      ):
						self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['''<mask>''', '''ar_AR''']      )			,				[25_0026, 25_0001]      )



	def 			snake_case__  (   self		:   Any      ):
						__magic_name__						       =    tempfile.mkdtemp()
						__magic_name__						       =    self.tokenizer.fairseq_tokens_to_ids
						self.tokenizer.save_pretrained(a__      )
						__magic_name__						       =    MBartTokenizer.from_pretrained(a__      )
						self.assertDictEqual(new_tok.fairseq_tokens_to_ids			,				a__      )



	@require_torch
	def 			snake_case__  (   self		:   int      ):
						__magic_name__						       =    self.tokenizer(self.src_text			,				text_target=self.tgt_text			,				padding=a__			,				return_tensors='''pt'''      )
						__magic_name__						       =    shift_tokens_right(batch['''labels''']			,				self.tokenizer.pad_token_id      )

						# fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
						assert batch.input_ids[1][-2:].tolist() == [2, EN_CODE]
						assert batch.decoder_input_ids[1][0].tolist() == RO_CODE
						assert batch.decoder_input_ids[1][-1] == 2
						assert batch.labels[1][-2:].tolist() == [2, RO_CODE]



	@require_torch
	def 			snake_case__  (   self		:   Any      ):
						__magic_name__						       =    self.tokenizer(
						    self.src_text			,				text_target=self.tgt_text			,				padding=a__			,				truncation=a__			,				max_length=len(self.expected_src_tokens      )			,				return_tensors='''pt'''			,				)

						__magic_name__						       =    shift_tokens_right(batch['''labels''']			,				self.tokenizer.pad_token_id      )

						self.assertIsInstance(a__			,				a__      )

						self.assertEqual((2, 14)			,				batch.input_ids.shape      )
						self.assertEqual((2, 14)			,				batch.attention_mask.shape      )
						__magic_name__						       =    batch.input_ids.tolist()[0]
						self.assertListEqual(self.expected_src_tokens			,				a__      )
						self.assertEqual(2			,				batch.decoder_input_ids[0, -1]      )  # EOS
						# Test that special tokens are reset
						self.assertEqual(self.tokenizer.prefix_tokens			,				[]      )
						self.assertEqual(self.tokenizer.suffix_tokens			,				[self.tokenizer.eos_token_id, EN_CODE]      )



	def 			snake_case__  (   self		:   Optional[Any]      ):
						__magic_name__						       =    self.tokenizer(self.src_text			,				padding=a__			,				truncation=a__			,				max_length=3			,				return_tensors='''pt'''      )
						__magic_name__						       =    self.tokenizer(
						    text_target=self.tgt_text			,				padding=a__			,				truncation=a__			,				max_length=10			,				return_tensors='''pt'''      )
						__magic_name__						       =    targets['''input_ids''']
						__magic_name__						       =    shift_tokens_right(a__			,				self.tokenizer.pad_token_id      )

						self.assertEqual(batch.input_ids.shape[1]			,				3      )
						self.assertEqual(batch.decoder_input_ids.shape[1]			,				10      )



	@require_torch
	def 			snake_case__  (   self		:   Any      ):
						__magic_name__						       =    self.tokenizer._build_translation_inputs(
						    '''A test'''			,				return_tensors='''pt'''			,				src_lang='''en_XX'''			,				tgt_lang='''ar_AR'''      )

						self.assertEqual(
						    nested_simplify(a__      )			,				{
						        # A, test, EOS, en_XX
						        '''input_ids''': [[62, 3034, 2, 25_0004]],
						        '''attention_mask''': [[1, 1, 1, 1]],
						        # ar_AR
						        '''forced_bos_token_id''': 25_0001,
						    }			,				)









'''simple docstring'''




from collections import OrderedDict
from typing import TYPE_CHECKING, Any, Mapping, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxSeqaSeqConfigWithPast
from ...utils import logging


if TYPE_CHECKING:
      from ...feature_extraction_utils import FeatureExtractionMixin
      from ...tokenization_utils_base import PreTrainedTokenizerBase
      from ...utils import TensorType

_lowerCAmelCase         =						logging.get_logger(__name__)

_lowerCAmelCase         =						{
    "openai/whisper-base": "https://huggingface.co/openai/whisper-base/resolve/main/config.json",
}

# fmt: off
_lowerCAmelCase         =						[
    1, 2, 7, 8, 9, 10, 14, 25,
    26, 27, 28, 29, 31, 58, 59, 60, 61, 62,
    63, 90, 91, 92, 93, 357, 366, 438, 532, 685,
    705, 796, 930, 1058, 1220, 1267, 1279, 1303, 1343, 1377,
    1391, 1635, 1782, 1875, 2162, 2361, 2488, 3467, 4008, 4211,
    4600, 4808, 5299, 5855, 6329, 7203, 9609, 9959, 10563, 10786,
    11420, 11709, 11907, 13163, 13697, 13700, 14808, 15306, 16410, 16791,
    17992, 19203, 19510, 20724, 22305, 22935, 27007, 30109, 30420, 33409,
    34949, 40283, 40493, 40549, 47282, 49146, 50257, 50359, 50360, 50361
]
_lowerCAmelCase         =						[
    1, 2, 7, 8, 9, 10, 14, 25,
    26, 27, 28, 29, 31, 58, 59, 60, 61, 62,
    63, 90, 91, 92, 93, 359, 503, 522, 542, 873,
    893, 902, 918, 922, 931, 1350, 1853, 1982, 2460, 2627,
    3246, 3253, 3268, 3536, 3846, 3961, 4183, 4667, 6585, 6647,
    7273, 9061, 9383, 10428, 10929, 11938, 12033, 12331, 12562, 13793,
    14157, 14635, 15265, 15618, 16553, 16604, 18362, 18956, 20075, 21675,
    22520, 26130, 26161, 26435, 28279, 29464, 31650, 32302, 32470, 36865,
    42863, 47425, 49870, 50254, 50258, 50360, 50361, 50362
]






class   _SCREAMING_SNAKE_CASE   (   __a     ):
 __SCREAMING_SNAKE_CASE					:str      	=					"""whisper"""
 __SCREAMING_SNAKE_CASE					:str      	=					["""past_key_values"""]
 __SCREAMING_SNAKE_CASE					:Tuple      	=					{"""num_attention_heads""": """encoder_attention_heads""", """hidden_size""": """d_model"""}



 def __init__(   self		:   Dict			,				a__		:   Optional[int]=5_1865			,				a__		:   str=80			,				a__		:   List[str]=6			,				a__		:   List[str]=4			,				a__		:   List[Any]=6			,				a__		:   Union[str, Any]=4			,				a__		:   Tuple=1536			,				a__		:   Optional[int]=1536			,				a__		:   List[str]=0.0			,				a__		:   Union[str, Any]=0.0			,				a__		:   Union[str, Any]=5_0257			,				a__		:   Dict=True			,				a__		:   Optional[Any]=True			,				a__		:   Union[str, Any]="gelu"			,				a__		:   Tuple=256			,				a__		:   Dict=0.0			,				a__		:   str=0.0			,				a__		:   Optional[Any]=0.0			,				a__		:   int=0.02			,				a__		:   Any=False			,				a__		:   List[Any]=1500			,				a__		:   Optional[int]=448			,				a__		:   Dict=5_0256			,				a__		:   str=5_0256			,				a__		:   Tuple=5_0256			,				a__		:   List[str]=None			,				a__		:   List[Any]=[220, 5_0256]			,				a__		:   Any=False			,				a__		:   Dict=256			,				a__		:   Optional[Any]=False			,				a__		:   str=0.05			,				a__		:   List[Any]=10			,				a__		:   List[Any]=2			,				a__		:   Optional[int]=0.0			,				a__		:   List[Any]=10			,				a__		:   Union[str, Any]=0			,				a__		:   int=7			,				**a__		:   Any			,				):
      __magic_name__						       =    vocab_size
      __magic_name__						       =    num_mel_bins
      __magic_name__						       =    d_model
      __magic_name__						       =    encoder_layers
      __magic_name__						       =    encoder_attention_heads
      __magic_name__						       =    decoder_layers
      __magic_name__						       =    decoder_attention_heads
      __magic_name__						       =    decoder_ffn_dim
      __magic_name__						       =    encoder_ffn_dim
      __magic_name__						       =    dropout
      __magic_name__						       =    attention_dropout
      __magic_name__						       =    activation_dropout
      __magic_name__						       =    activation_function
      __magic_name__						       =    init_std
      __magic_name__						       =    encoder_layerdrop
      __magic_name__						       =    decoder_layerdrop
      __magic_name__						       =    use_cache
      __magic_name__						       =    encoder_layers
      __magic_name__						       =    scale_embedding  # scale factor will be sqrt(d_model) if True
      __magic_name__						       =    max_source_positions
      __magic_name__						       =    max_target_positions

      # Audio Classification-specific parameters. Feel free to ignore for other classes.
      __magic_name__						       =    classifier_proj_size
      __magic_name__						       =    use_weighted_layer_sum

      # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
      __magic_name__						       =    apply_spec_augment
      __magic_name__						       =    mask_time_prob
      __magic_name__						       =    mask_time_length
      __magic_name__						       =    mask_time_min_masks
      __magic_name__						       =    mask_feature_prob
      __magic_name__						       =    mask_feature_length
      __magic_name__						       =    mask_feature_min_masks

      __magic_name__						       =    median_filter_width

      super().__init__(
          pad_token_id=a__			,				bos_token_id=a__			,				eos_token_id=a__			,				is_encoder_decoder=a__			,				decoder_start_token_id=a__			,				suppress_tokens=a__			,				begin_suppress_tokens=a__			,				**a__			,				)






class   _SCREAMING_SNAKE_CASE   (   __a     ):



 @property
 def 			snake_case__  (   self		:   List[str]      ):
      __magic_name__						       =    OrderedDict(
          [
              ('''input_features''', {0: '''batch''', 1: '''feature_size''', 2: '''encoder_sequence'''}),
          ]      )
      if self.use_past:
           __magic_name__						       =    {0: '''batch'''}
      else:
           __magic_name__						       =    {0: '''batch''', 1: '''decoder_sequence'''}

      if self.use_past:
           self.fill_with_past_key_values_(a__			,				direction='''inputs'''      )

      return common_inputs



 def 			snake_case__  (   self		:   Optional[int]			,				a__		:   Union["PreTrainedTokenizerBase", "FeatureExtractionMixin"]			,				a__		:   int = -1			,				a__		:   int = -1			,				a__		:   bool = False			,				a__		:   Optional["TensorType"] = None			,				a__		:   int = 2_2050			,				a__		:   float = 5.0			,				a__		:   int = 220			,				):
      __magic_name__						       =    OrderedDict()
      __magic_name__						       =    OnnxConfig.generate_dummy_inputs(
          self			,				preprocessor=preprocessor.feature_extractor			,				batch_size=a__			,				framework=a__			,				sampling_rate=a__			,				time_duration=a__			,				frequency=a__			,				)
      __magic_name__						       =    encoder_inputs['''input_features'''].shape[2]
      __magic_name__						       =    encoder_sequence_length // 2 if self.use_past else seq_length

      __magic_name__						       =    super().generate_dummy_inputs(
          preprocessor.tokenizer			,				a__			,				a__			,				a__			,				a__      )

      __magic_name__						       =    encoder_inputs.pop('''input_features'''      )
      __magic_name__						       =    decoder_inputs.pop('''decoder_input_ids'''      )

      if "past_key_values" in decoder_inputs:
           __magic_name__						       =    decoder_inputs.pop('''past_key_values'''      )

      return dummy_inputs



 @property
 def 			snake_case__  (   self		:   Dict      ):
      return 1E-3






