




'''simple docstring'''





import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import MaMaaaTokenizer, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
    slow,
)
from transformers.utils import is_sentencepiece_available


if is_sentencepiece_available():
  from transformers.models.mam_aaa.tokenization_mam_aaa import VOCAB_FILES_NAMES, save_json

from ...test_tokenization_common import TokenizerTesterMixin


if is_sentencepiece_available():
  UpperCamelCase_						:      List[Any]       =						get_tests_dir('''fixtures/test_sentencepiece.model''')


if is_torch_available():
  from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

UpperCamelCase_						:      Optional[Any]       =						128022
UpperCamelCase_						:      Optional[int]       =						128028

@require_sentencepiece
class        _a    (						__lowerCAmelCase , unittest.TestCase							):
     SCREAMING_SNAKE_CASE_							:   str       =						MaMaaaTokenizer
     SCREAMING_SNAKE_CASE_							:   List[Any]       =						False
     SCREAMING_SNAKE_CASE_							:   Tuple       =						False
     SCREAMING_SNAKE_CASE_							:   int       =						True






     def     _lowercase     (			self   )							->      Dict:
          super().setUp()

          _snake_case							=				["</s>", "<unk>", "▁This", "▁is", "▁a", "▁t", "est", "\u0120", "<pad>"]
          _snake_case							=				dict(zip(_SCREAMING_SNAKE_CASE ,range(len(_SCREAMING_SNAKE_CASE   )   )   )   )
          _snake_case							=				Path(self.tmpdirname   )
          save_json(_SCREAMING_SNAKE_CASE ,save_dir / VOCAB_FILES_NAMES["vocab_file"]   )
          if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
               copyfile(_SCREAMING_SNAKE_CASE ,save_dir / VOCAB_FILES_NAMES["spm_file"]   )

          _snake_case							=				MaMaaaTokenizer.from_pretrained(self.tmpdirname   )
          tokenizer.save_pretrained(self.tmpdirname   )






     def     _lowercase     (			self ,**_SCREAMING_SNAKE_CASE   )							->      Optional[Any]:
          return MaMaaaTokenizer.from_pretrained(self.tmpdirname ,**_SCREAMING_SNAKE_CASE   )






     def     _lowercase     (			self ,_SCREAMING_SNAKE_CASE   )							->      str:
          return (
              "This is a test",
              "This is a test",
          )






     def     _lowercase     (			self   )							->      Optional[int]:
          _snake_case							=				"</s>"
          _snake_case							=				0

          self.assertEqual(self.get_tokenizer()._convert_token_to_id(_SCREAMING_SNAKE_CASE   ) ,_SCREAMING_SNAKE_CASE   )
          self.assertEqual(self.get_tokenizer()._convert_id_to_token(_SCREAMING_SNAKE_CASE   ) ,_SCREAMING_SNAKE_CASE   )






     def     _lowercase     (			self   )							->      Union[str, Any]:
          _snake_case							=				self.get_tokenizer()
          _snake_case							=				list(tokenizer.get_vocab().keys()   )

          self.assertEqual(vocab_keys[0] ,"</s>"   )
          self.assertEqual(vocab_keys[1] ,"<unk>"   )
          self.assertEqual(vocab_keys[-1] ,"<s>"   )
          self.assertEqual(len(_SCREAMING_SNAKE_CASE   ) ,tokenizer.vocab_size + len(tokenizer.get_added_vocab()   )   )






     @unittest.skip("Skip this test while all models are still to be uploaded."   )
     def     _lowercase     (			self   )							->      List[str]:
          pass






     def     _lowercase     (			self   )							->      Optional[int]:
          _snake_case							=				self.get_tokenizer()

          _snake_case							=				tokenizer.tokenize("This is a test"   )
          self.assertListEqual(_SCREAMING_SNAKE_CASE ,["▁This", "▁is", "▁a", "▁t", "est"]   )

          self.assertListEqual(
              tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE   ) ,[2, 3, 4, 5, 6] ,)

          _snake_case							=				tokenizer.convert_ids_to_tokens([2, 3, 4, 5, 6]   )
          self.assertListEqual(_SCREAMING_SNAKE_CASE ,["▁This", "▁is", "▁a", "▁t", "est"]   )

          _snake_case							=				tokenizer.convert_tokens_to_string(_SCREAMING_SNAKE_CASE   )
          self.assertEqual(_SCREAMING_SNAKE_CASE ,"This is a test"   )






     @slow
     def     _lowercase     (			self   )							->      Dict:
          # fmt: off
          _snake_case							=				{"input_ids": [[128_022, 110_108, 397, 11, 38_272, 2_247, 124_811, 285, 18_105, 1_586, 207, 7, 39_534, 4_428, 397, 1_019, 18_105, 1_586, 207, 7, 41_337, 16_786, 241, 7, 20_214, 17, 125_690, 10_398, 7, 44_378, 58_069, 68_342, 7_798, 7_343, 11, 299, 33_310, 4, 158, 37_350, 94_077, 4_569, 299, 33_310, 90, 4, 52_840, 290, 4, 31_270, 112, 299, 682, 4, 52_840, 39_953, 14_079, 193, 52_519, 90_894, 17_894, 120_697, 11, 40_445, 551, 17, 1_019, 52_519, 90_894, 17_756, 963, 11, 40_445, 480, 17, 9_792, 1_120, 5_173, 1_393, 6_240, 16_786, 241, 120_996, 28, 1_245, 1_393, 118_240, 11_123, 1_019, 93_612, 2_691, 10_618, 98_058, 120_409, 1_928, 279, 4, 40_683, 367, 178, 207, 1_019, 103, 103_121, 506, 65_296, 5, 2], [128_022, 21_217, 367, 117, 125_450, 128, 719, 7, 7_308, 40, 93_612, 12_669, 1_116, 16_704, 71, 17_785, 3_699, 15_592, 35, 144, 9_584, 241, 11_943, 713, 950, 799, 2_247, 88_427, 150, 149, 118_813, 120_706, 1_019, 106_906, 81_518, 28, 1_224, 22_799, 397, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [128_022, 1_658, 123_311, 5_155, 5_578, 4_722, 279, 14_947, 2_366, 1_120, 1_197, 14, 1_348, 9_232, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
          # fmt: on

          self.tokenizer_integration_test_util(
              expected_encoding=_SCREAMING_SNAKE_CASE ,model_name="facebook/m2m100_418M" ,revision="c168bae485c864188cf9aa0e4108b0b6934dc91e" ,)







@require_torch
@require_sentencepiece
@require_tokenizers
class        _a    (						unittest.TestCase							):
     SCREAMING_SNAKE_CASE_							:   Optional[int]       =						"""facebook/m2m100_418M"""
     SCREAMING_SNAKE_CASE_							:   List[str]       =						[
         """In my opinion, there are two levels of response from the French government.""",
         """NSA Affair Emphasizes Complete Lack of Debate on Intelligence""",
     ]
     SCREAMING_SNAKE_CASE_							:   Optional[int]       =						[
         """Selon moi, il y a deux niveaux de réponse de la part du gouvernement français.""",
         """L'affaire NSA souligne l'absence totale de débat sur le renseignement""",
     ]

     # fmt: off
     SCREAMING_SNAKE_CASE_							:   int       =						[EN_CODE, 5_93, 19_49, 11_57_81, 4, 7_15_86, 42_34, 6_06_33, 12_62_33, 4_32, 12_38_08, 1_55_92, 11_97, 11_71_32, 12_06_18, 5, 2]






     @classmethod
     def     _lowercase     (			cls   )							->      Tuple:
          _snake_case							=				MaMaaaTokenizer.from_pretrained(
              cls.checkpoint_name ,src_lang="en" ,tgt_lang="fr"   )
          _snake_case							=				1
          return cls






     def     _lowercase     (			self   )							->      List[str]:
          self.assertEqual(self.tokenizer.get_lang_id("ar"   ) ,128_006   )
          self.assertEqual(self.tokenizer.get_lang_id("en"   ) ,128_022   )
          self.assertEqual(self.tokenizer.get_lang_id("ro"   ) ,128_076   )
          self.assertEqual(self.tokenizer.get_lang_id("mr"   ) ,128_063   )






     def     _lowercase     (			self   )							->      str:
          _snake_case							=				self.tokenizer.get_vocab()
          self.assertEqual(len(_SCREAMING_SNAKE_CASE   ) ,self.tokenizer.vocab_size   )
          self.assertEqual(vocab["<unk>"] ,3   )
          self.assertIn(self.tokenizer.get_lang_token("en"   ) ,_SCREAMING_SNAKE_CASE   )






     def     _lowercase     (			self   )							->      Optional[Any]:
          _snake_case							=				"en"
          _snake_case							=				self.tokenizer.batch_encode_plus(self.src_text   ).input_ids[0]
          self.assertListEqual(self.expected_src_tokens ,_SCREAMING_SNAKE_CASE   )






     def     _lowercase     (			self   )							->      Union[str, Any]:
          self.assertIn(_SCREAMING_SNAKE_CASE ,self.tokenizer.all_special_ids   )
          # fmt: off
          _snake_case							=				[FR_CODE, 5_364, 82, 8_642, 4, 294, 47, 8, 14_028, 136, 3_286, 9_706, 6, 90_797, 6, 144_012, 162, 88_128, 30_061, 5, 2]
          # fmt: on
          _snake_case							=				self.tokenizer.decode(_SCREAMING_SNAKE_CASE ,skip_special_tokens=_SCREAMING_SNAKE_CASE   )
          _snake_case							=				self.tokenizer.decode(generated_ids[1:] ,skip_special_tokens=_SCREAMING_SNAKE_CASE   )
          self.assertEqual(_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE   )
          self.assertNotIn(self.tokenizer.eos_token ,_SCREAMING_SNAKE_CASE   )






     def     _lowercase     (			self   )							->      Union[str, Any]:
          _snake_case							=				tempfile.mkdtemp()
          _snake_case							=				self.tokenizer.lang_token_to_id
          self.tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE   )
          _snake_case							=				MaMaaaTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE   )
          self.assertDictEqual(new_tok.lang_token_to_id ,_SCREAMING_SNAKE_CASE   )






     @require_torch
     def     _lowercase     (			self   )							->      List[Any]:
          _snake_case							=				"en"
          _snake_case							=				"fr"

          _snake_case							=				self.tokenizer(self.src_text ,text_target=self.tgt_text ,padding=_SCREAMING_SNAKE_CASE ,return_tensors="pt"   )

          _snake_case							=				shift_tokens_right(
              batch["labels"] ,self.tokenizer.pad_token_id ,self.tokenizer.eos_token_id   )

          for k in batch:
               _snake_case							=				batch[k].tolist()
          # batch = {k: v.tolist() for k,v in batch.items()}
          # fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
          # batch.decoder_inputs_ids[0][0] ==
          assert batch.input_ids[1][0] == EN_CODE
          assert batch.input_ids[1][-1] == 2
          assert batch.labels[1][0] == FR_CODE
          assert batch.labels[1][-1] == 2
          assert batch.decoder_input_ids[1][:2] == [2, FR_CODE]






     @require_torch
     def     _lowercase     (			self   )							->      str:
          _snake_case							=				"mr"
          self.assertListEqual(self.tokenizer.prefix_tokens ,[self.tokenizer.get_lang_id("mr"   )]   )
          self.assertListEqual(self.tokenizer.suffix_tokens ,[self.tokenizer.eos_token_id]   )

          _snake_case							=				"zh"
          self.assertListEqual(self.tokenizer.prefix_tokens ,[self.tokenizer.get_lang_id("zh"   )]   )
          self.assertListEqual(self.tokenizer.suffix_tokens ,[self.tokenizer.eos_token_id]   )






     @require_torch
     def     _lowercase     (			self   )							->      Optional[int]:
          _snake_case							=				"mr"
          self.tokenizer._switch_to_target_mode()
          self.assertListEqual(self.tokenizer.prefix_tokens ,[self.tokenizer.get_lang_id("mr"   )]   )
          self.assertListEqual(self.tokenizer.suffix_tokens ,[self.tokenizer.eos_token_id]   )
          self.tokenizer._switch_to_input_mode()
          self.assertListEqual(self.tokenizer.prefix_tokens ,[self.tokenizer.get_lang_id(self.tokenizer.src_lang   )]   )

          _snake_case							=				"zh"
          self.tokenizer._switch_to_target_mode()
          self.assertListEqual(self.tokenizer.prefix_tokens ,[self.tokenizer.get_lang_id("zh"   )]   )
          self.assertListEqual(self.tokenizer.suffix_tokens ,[self.tokenizer.eos_token_id]   )
          self.tokenizer._switch_to_input_mode()
          self.assertListEqual(self.tokenizer.prefix_tokens ,[self.tokenizer.get_lang_id(self.tokenizer.src_lang   )]   )






     @require_torch
     def     _lowercase     (			self   )							->      List[str]:
          _snake_case							=				self.tokenizer._build_translation_inputs("A test" ,return_tensors="pt" ,src_lang="en" ,tgt_lang="ar"   )

          self.assertEqual(
              nested_simplify(_SCREAMING_SNAKE_CASE   ) ,{
                  # en_XX, A, test, EOS
                  "input_ids": [[128_022, 58, 4_183, 2]],
                  "attention_mask": [[1, 1, 1, 1]],
                  # ar_AR
                  "forced_bos_token_id": 128_006,
              } ,)








'''simple docstring'''





def      __a     (       _UpperCamelCase:   int   )						->			None:

     """simple docstring"""




     _snake_case							=				generate_pascal_triangle(_UpperCamelCase   )
     for row_idx in range(_UpperCamelCase   ):
          # Print left spaces
          for _ in range(num_rows - row_idx - 1   ):
               print(end=" "   )
          # Print row values
          for col_idx in range(row_idx + 1   ):
               if col_idx != row_idx:
                    print(triangle[row_idx][col_idx] ,						end=" "   )
               else:
                    print(triangle[row_idx][col_idx] ,						end=""   )
          print()





def      __a     (       _UpperCamelCase:   int   )						->			list[list[int]]:

     """simple docstring"""




     if not isinstance(_UpperCamelCase ,						_UpperCamelCase   ):
          raise TypeError("The input value of 'num_rows' should be 'int'"   )

     if num_rows == 0:
          return []
     elif num_rows < 0:
          raise ValueError(
              "The input value of 'num_rows' should be greater than or equal to 0"   )

     _snake_case							=				[]
     for current_row_idx in range(_UpperCamelCase   ):
          _snake_case							=				populate_current_row(_UpperCamelCase ,						_UpperCamelCase   )
          triangle.append(_UpperCamelCase   )
     return triangle





def      __a     (       _UpperCamelCase:   list[list[int]] ,						_UpperCamelCase:   int   )						->			list[int]:

     """simple docstring"""




     _snake_case							=				[-1] * (current_row_idx + 1)
     # first and last elements of current row are equal to 1
     _snake_case							, _snake_case							=				1, 1
     for current_col_idx in range(1 ,						_UpperCamelCase   ):
          calculate_current_element(
              _UpperCamelCase ,						_UpperCamelCase ,						_UpperCamelCase ,						_UpperCamelCase   )
     return current_row





def      __a     (       _UpperCamelCase:   list[list[int]] ,						_UpperCamelCase:   list[int] ,						_UpperCamelCase:   int ,						_UpperCamelCase:   int ,						)						->			None:

     """simple docstring"""




     _snake_case							=				triangle[current_row_idx - 1][current_col_idx - 1]
     _snake_case							=				triangle[current_row_idx - 1][current_col_idx]
     _snake_case							=				above_to_left_elt + above_to_right_elt





def      __a     (       _UpperCamelCase:   int   )						->			list[list[int]]:

     """simple docstring"""




     if not isinstance(_UpperCamelCase ,						_UpperCamelCase   ):
          raise TypeError("The input value of 'num_rows' should be 'int'"   )

     if num_rows == 0:
          return []
     elif num_rows < 0:
          raise ValueError(
              "The input value of 'num_rows' should be greater than or equal to 0"   )

     _snake_case							=				[[1]]

     for row_index in range(1 ,						_UpperCamelCase   ):
          _snake_case							=				[0] + result[-1] + [0]
          _snake_case							=				row_index + 1
          # Calculate the number of distinct elements in a row
          _snake_case							=				sum(divmod(_UpperCamelCase ,						2   )   )
          _snake_case							=				[
              temp_row[i - 1] + temp_row[i] for i in range(1 ,						distinct_elements + 1   )
          ]
          _snake_case							=				row_first_half[: (row_index + 1) // 2]
          row_second_half.reverse()
          _snake_case							=				row_first_half + row_second_half
          result.append(_UpperCamelCase   )

     return result





def      __a     (       )						->			None:

     """simple docstring"""




     from collections.abc import Callable
     from timeit import timeit

     def benchmark_a_function(_UpperCamelCase:   Callable ,						_UpperCamelCase:   int   ) -> None:
          _snake_case							=				F"""{func.__name__}({value})"""
          _snake_case							=				timeit(F"""__main__.{call}""" ,						setup="import __main__"   )
          # print(f"{call:38} = {func(value)} -- {timing:.4f} seconds")
          print(F"""{call:38} -- {timing:.4f} seconds"""   )

     for value in range(15   ):  # (1, 7, 14):
          for func in (generate_pascal_triangle, generate_pascal_triangle_optimized):
               benchmark_a_function(_UpperCamelCase ,						_UpperCamelCase   )
          print()


if __name__ == "__main__":
   import doctest

   doctest.testmod()
   benchmark()


