"""simple docstring"""




import unittest

from huggingface_hub import hf_hub_download

from transformers import MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING, VideoMAEFeatureExtractor
from transformers.pipelines import VideoClassificationPipeline, pipeline
from transformers.testing_utils import (
    is_pipeline_test,
    nested_simplify,
    require_decord,
    require_tf,
    require_torch,
    require_torch_or_tf,
    require_vision,
)

from .test_pipelines_common import ANY
@is_pipeline_test
@require_torch_or_tf
@require_vision
@require_decord
class 						__A							(			unittest.TestCase		):
     _UpperCamelCase							:      Optional[Any]		 =		MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING






     def    __A  (		self    ,					a__    ,					a__    ,					a__  ):
       _lowerCAmelCase						:	Optional[Any]        =					hf_hub_download(
           repo_id="""nateraw/video-demo"""    ,					filename="""archery.mp4"""    ,					repo_type="""dataset"""  )
       _lowerCAmelCase						:	Union[str, Any]        =					VideoClassificationPipeline(model=snake_case_    ,					image_processor=snake_case_    ,					top_k=2  )
       _lowerCAmelCase						:	Dict        =					[
           example_video_filepath,
           '''https://huggingface.co/datasets/nateraw/video-demo/resolve/main/archery.mp4''',
       ]
       return video_classifier, examples






     def    __A  (		self    ,					a__    ,					a__  ):
       for example in examples:
         _lowerCAmelCase						:	List[Any]        =					video_classifier(snake_case_  )

         self.assertEqual(
             snake_case_    ,					[
                 {"""score""": ANY(snake_case_  ), """label""": ANY(snake_case_  )},
                 {"""score""": ANY(snake_case_  ), """label""": ANY(snake_case_  )},
             ]    ,					)






     @require_torch
     def    __A  (		self  ):
       _lowerCAmelCase						:	List[Any]        =					'''hf-internal-testing/tiny-random-VideoMAEForVideoClassification'''
       _lowerCAmelCase						:	Any        =					VideoMAEFeatureExtractor(
           size={"""shortest_edge""": 10}    ,					crop_size={"""height""": 10, """width""": 10}  )
       _lowerCAmelCase						:	Union[str, Any]        =					pipeline(
           """video-classification"""    ,					model=snake_case_    ,					feature_extractor=snake_case_    ,					frame_sampling_rate=4  )

       _lowerCAmelCase						:	Optional[int]        =					hf_hub_download(repo_id="""nateraw/video-demo"""    ,					filename="""archery.mp4"""    ,					repo_type="""dataset"""  )
       _lowerCAmelCase						:	List[Any]        =					video_classifier(snake_case_    ,					top_k=2  )
       self.assertEqual(
           nested_simplify(snake_case_    ,					decimals=4  )    ,					[{"""score""": 0.5_1_9_9, """label""": """LABEL_0"""}, {"""score""": 0.4_8_0_1, """label""": """LABEL_1"""}]    ,					)

       _lowerCAmelCase						:	List[Any]        =					video_classifier(
           [
               video_file_path,
               video_file_path,
           ]    ,					top_k=2    ,					)
       self.assertEqual(
           nested_simplify(snake_case_    ,					decimals=4  )    ,					[
               [{"""score""": 0.5_1_9_9, """label""": """LABEL_0"""}, {"""score""": 0.4_8_0_1, """label""": """LABEL_1"""}],
               [{"""score""": 0.5_1_9_9, """label""": """LABEL_0"""}, {"""score""": 0.4_8_0_1, """label""": """LABEL_1"""}],
           ]    ,					)






     @require_tf
     def    __A  (		self  ):
       pass




"""simple docstring"""




import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, MBartaaTokenizer, MBartaaTokenizerFast, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
    slow,
)

from ...test_tokenization_common import TokenizerTesterMixin


_a				:		Optional[int]	   =       get_tests_dir('fixtures/test_sentencepiece.model')

if is_torch_available():
     from transformers.models.mbart.modeling_mbart import shift_tokens_right

_a				:		Union[str, Any]	   =       250_004
_a				:		Optional[int]	   =       250_020
@require_sentencepiece
@require_tokenizers
class 						__A							(			SCREAMING_SNAKE_CASE_   ,     unittest.TestCase		):
     _UpperCamelCase							:      Optional[Any]		 =		MBartaaTokenizer
     _UpperCamelCase							:      Any		 =		MBartaaTokenizerFast
     _UpperCamelCase							:      List[str]		 =		True
     _UpperCamelCase							:      Optional[int]		 =		True






     def    __A  (		self  ):
       super().setUp()

       # We have a SentencePiece fixture for testing
       _lowerCAmelCase						:	Tuple        =					MBartaaTokenizer(a__    ,					src_lang="""en_XX"""    ,					tgt_lang="""ro_RO"""    ,					keep_accents=a__  )
       tokenizer.save_pretrained(self.tmpdirname  )






     def    __A  (		self  ):
       _lowerCAmelCase						:	Any        =					"""<s>"""
       _lowerCAmelCase						:	Optional[int]        =					0

       self.assertEqual(self.get_tokenizer()._convert_token_to_id(a__  )    ,					a__  )
       self.assertEqual(self.get_tokenizer()._convert_id_to_token(a__  )    ,					a__  )






     def    __A  (		self  ):
       _lowerCAmelCase						:	Optional[Any]        =					list(self.get_tokenizer().get_vocab().keys()  )

       self.assertEqual(vocab_keys[0]    ,					"""<s>"""  )
       self.assertEqual(vocab_keys[1]    ,					"""<pad>"""  )
       self.assertEqual(vocab_keys[-1]    ,					"""<mask>"""  )
       self.assertEqual(len(a__  )    ,					1054  )






     def    __A  (		self  ):
       self.assertEqual(self.get_tokenizer().vocab_size    ,					1054  )






     def    __A  (		self  ):
       _lowerCAmelCase						:	str        =					MBartaaTokenizer(a__    ,					src_lang="""en_XX"""    ,					tgt_lang="""ro_RO"""    ,					keep_accents=a__  )

       _lowerCAmelCase						:	Optional[int]        =					tokenizer.tokenize("""This is a test"""  )
       self.assertListEqual(a__    ,					["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]  )

       self.assertListEqual(
           tokenizer.convert_tokens_to_ids(a__  )    ,					[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]    ,					)

       _lowerCAmelCase						:	int        =					tokenizer.tokenize("""I was born in 92000, and this is falsé."""  )
       self.assertListEqual(
           a__    ,					[SPIECE_UNDERLINE + """I""", SPIECE_UNDERLINE + """was""", SPIECE_UNDERLINE + """b""", """or""", """n""", SPIECE_UNDERLINE + """in""", SPIECE_UNDERLINE + """""", """9""", """2""", """0""", """0""", """0""", """,""", SPIECE_UNDERLINE + """and""", SPIECE_UNDERLINE + """this""", SPIECE_UNDERLINE + """is""", SPIECE_UNDERLINE + """f""", """al""", """s""", """é""", """."""]    ,					)
       _lowerCAmelCase						:	Union[str, Any]        =					tokenizer.convert_tokens_to_ids(a__  )
       self.assertListEqual(
           a__    ,					[
               value + tokenizer.fairseq_offset
               for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
           ]    ,					)

       _lowerCAmelCase						:	int        =					tokenizer.convert_ids_to_tokens(a__  )
       self.assertListEqual(
           a__    ,					[SPIECE_UNDERLINE + """I""", SPIECE_UNDERLINE + """was""", SPIECE_UNDERLINE + """b""", """or""", """n""", SPIECE_UNDERLINE + """in""", SPIECE_UNDERLINE + """""", """<unk>""", """2""", """0""", """0""", """0""", """,""", SPIECE_UNDERLINE + """and""", SPIECE_UNDERLINE + """this""", SPIECE_UNDERLINE + """is""", SPIECE_UNDERLINE + """f""", """al""", """s""", """<unk>""", """."""]    ,					)






     @slow
     def    __A  (		self  ):
       # fmt: off
       _lowerCAmelCase						:	Any        =					{"""input_ids""": [[250004, 11062, 82772, 7, 15, 82772, 538, 51529, 237, 17198, 1290, 206, 9, 215175, 1314, 136, 17198, 1290, 206, 9, 56359, 42, 122009, 9, 16466, 16, 87344, 4537, 9, 4717, 78381, 6, 159958, 7, 15, 24480, 618, 4, 527, 22693, 5428, 4, 2777, 24480, 9874, 4, 43523, 594, 4, 803, 18392, 33189, 18, 4, 43523, 24447, 12399, 100, 24955, 83658, 9626, 144057, 15, 839, 22335, 16, 136, 24955, 83658, 83479, 15, 39102, 724, 16, 678, 645, 2789, 1328, 4589, 42, 122009, 115774, 23, 805, 1328, 46876, 7, 136, 53894, 1940, 42227, 41159, 17721, 823, 425, 4, 27512, 98722, 206, 136, 5531, 4970, 919, 17336, 5, 2], [250004, 20080, 618, 83, 82775, 47, 479, 9, 1517, 73, 53894, 333, 80581, 110117, 18811, 5256, 1295, 51, 152526, 297, 7986, 390, 124416, 538, 35431, 214, 98, 15044, 25737, 136, 7108, 43701, 23, 756, 135355, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [250004, 581, 63773, 119455, 6, 147797, 88203, 7, 645, 70, 21, 3285, 10269, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
       # fmt: on

       self.tokenizer_integration_test_util(
           expected_encoding=a__    ,					model_name="""facebook/mbart-large-50"""    ,					revision="""d3913889c59cd5c9e456b269c376325eabad57e2"""    ,					)






     def    __A  (		self  ):
       if not self.test_slow_tokenizer:
         # as we don't have a slow version, we can't compare the outputs between slow and fast versions
         return

       _lowerCAmelCase						:	List[str]        =					(self.rust_tokenizer_class, """hf-internal-testing/tiny-random-mbart50""", {})
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"  ):
           _lowerCAmelCase						:	List[Any]        =					self.rust_tokenizer_class.from_pretrained(a__    ,					**a__  )
           _lowerCAmelCase						:	List[str]        =					self.tokenizer_class.from_pretrained(a__    ,					**a__  )

           _lowerCAmelCase						:	Optional[int]        =					tempfile.mkdtemp()

           _lowerCAmelCase						:	Optional[Any]        =					tokenizer_r.save_pretrained(a__  )
           _lowerCAmelCase						:	List[str]        =					tokenizer_p.save_pretrained(a__  )

           # Checks it save with the same files + the tokenizer.json file for the fast one
           self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files  )  )
           _lowerCAmelCase						:	Any        =					tuple(f for f in tokenizer_r_files if """tokenizer.json""" not in f  )
           self.assertSequenceEqual(a__    ,					a__  )

           # Checks everything loads correctly in the same way
           _lowerCAmelCase						:	List[Any]        =					tokenizer_r.from_pretrained(a__  )
           _lowerCAmelCase						:	int        =					tokenizer_p.from_pretrained(a__  )

           # Check special tokens are set accordingly on Rust and Python
           for key in tokenizer_pp.special_tokens_map:
             self.assertTrue(hasattr(a__    ,					a__  )  )
             # self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
             # self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

           shutil.rmtree(a__  )

           # Save tokenizer rust, legacy_format=True
           _lowerCAmelCase						:	List[Any]        =					tempfile.mkdtemp()

           _lowerCAmelCase						:	Dict        =					tokenizer_r.save_pretrained(a__    ,					legacy_format=a__  )
           _lowerCAmelCase						:	str        =					tokenizer_p.save_pretrained(a__  )

           # Checks it save with the same files
           self.assertSequenceEqual(a__    ,					a__  )

           # Checks everything loads correctly in the same way
           _lowerCAmelCase						:	List[str]        =					tokenizer_r.from_pretrained(a__  )
           _lowerCAmelCase						:	Any        =					tokenizer_p.from_pretrained(a__  )

           # Check special tokens are set accordingly on Rust and Python
           for key in tokenizer_pp.special_tokens_map:
             self.assertTrue(hasattr(a__    ,					a__  )  )

           shutil.rmtree(a__  )

           # Save tokenizer rust, legacy_format=False
           _lowerCAmelCase						:	Optional[Any]        =					tempfile.mkdtemp()

           _lowerCAmelCase						:	Tuple        =					tokenizer_r.save_pretrained(a__    ,					legacy_format=a__  )
           _lowerCAmelCase						:	str        =					tokenizer_p.save_pretrained(a__  )

           # Checks it saved the tokenizer.json file
           self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files  )  )

           # Checks everything loads correctly in the same way
           _lowerCAmelCase						:	Any        =					tokenizer_r.from_pretrained(a__  )
           _lowerCAmelCase						:	int        =					tokenizer_p.from_pretrained(a__  )

           # Check special tokens are set accordingly on Rust and Python
           for key in tokenizer_pp.special_tokens_map:
             self.assertTrue(hasattr(a__    ,					a__  )  )

           shutil.rmtree(a__  )







@require_torch
@require_sentencepiece
@require_tokenizers
class 						__A							(			unittest.TestCase		):
     _UpperCamelCase							:      Union[str, Any]		 =		"facebook/mbart-large-50-one-to-many-mmt"
     _UpperCamelCase							:      int		 =		[
         " UN Chief Says There Is No Military Solution in Syria",
         " Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.",
     ]
     _UpperCamelCase							:      Any		 =		[
         "Şeful ONU declară că nu există o soluţie militară în Siria",
         "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"
         " pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"
         " face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.",
     ]
     _UpperCamelCase							:      List[str]		 =		[EN_CODE, 8_274, 127_873, 25_916, 7, 8_622, 2_071, 438, 67_485, 53, 187_895, 23, 51_712, 2]






     @classmethod
     def    __A  (		cls  ):
       _lowerCAmelCase						:	MBartaaTokenizer        =					MBartaaTokenizer.from_pretrained(
           cls.checkpoint_name    ,					src_lang="""en_XX"""    ,					tgt_lang="""ro_RO"""  )
       _lowerCAmelCase						:	str        =					1
       return cls






     def    __A  (		self  ):
       self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ar_AR"""]    ,					250001  )
       self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""en_EN"""]    ,					250004  )
       self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ro_RO"""]    ,					250020  )
       self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""mr_IN"""]    ,					250038  )






     def    __A  (		self  ):
       _lowerCAmelCase						:	int        =					self.tokenizer.batch_encode_plus(self.src_text  ).input_ids[0]
       self.assertListEqual(self.expected_src_tokens    ,					a__  )






     def    __A  (		self  ):
       self.assertIn(a__    ,					self.tokenizer.all_special_ids  )
       _lowerCAmelCase						:	Union[str, Any]        =					[RO_CODE, 884, 9019, 96, 9, 916, 86792, 36, 18743, 15596, 5, 2]
       _lowerCAmelCase						:	int        =					self.tokenizer.decode(a__    ,					skip_special_tokens=a__  )
       _lowerCAmelCase						:	str        =					self.tokenizer.decode(generated_ids[1:]    ,					skip_special_tokens=a__  )
       self.assertEqual(a__    ,					a__  )
       self.assertNotIn(self.tokenizer.eos_token    ,					a__  )






     def    __A  (		self  ):
       _lowerCAmelCase						:	Any        =					["""this is gunna be a long sentence """ * 20]
       assert isinstance(src_text[0]    ,					a__  )
       _lowerCAmelCase						:	Optional[Any]        =					10
       _lowerCAmelCase						:	Optional[int]        =					self.tokenizer(a__    ,					max_length=a__    ,					truncation=a__  ).input_ids[0]
       self.assertEqual(ids[0]    ,					a__  )
       self.assertEqual(ids[-1]    ,					2  )
       self.assertEqual(len(a__  )    ,					a__  )






     def    __A  (		self  ):
       self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["""<mask>""", """ar_AR"""]  )    ,					[250053, 250001]  )






     def    __A  (		self  ):
       _lowerCAmelCase						:	Optional[Any]        =					tempfile.mkdtemp()
       _lowerCAmelCase						:	Tuple        =					self.tokenizer.fairseq_tokens_to_ids
       self.tokenizer.save_pretrained(a__  )
       _lowerCAmelCase						:	List[Any]        =					MBartaaTokenizer.from_pretrained(a__  )
       self.assertDictEqual(new_tok.fairseq_tokens_to_ids    ,					a__  )






     @require_torch
     def    __A  (		self  ):
       _lowerCAmelCase						:	List[str]        =					self.tokenizer(self.src_text    ,					text_target=self.tgt_text    ,					padding=a__    ,					return_tensors="""pt"""  )
       _lowerCAmelCase						:	Any        =					shift_tokens_right(batch["""labels"""]    ,					self.tokenizer.pad_token_id  )

       # fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
       assert batch.input_ids[1][0] == EN_CODE
       assert batch.input_ids[1][-1] == 2
       assert batch.labels[1][0] == RO_CODE
       assert batch.labels[1][-1] == 2
       assert batch.decoder_input_ids[1][:2].tolist() == [2, RO_CODE]






     @require_torch
     def    __A  (		self  ):
       _lowerCAmelCase						:	Optional[int]        =					self.tokenizer(
           self.src_text    ,					text_target=self.tgt_text    ,					padding=a__    ,					truncation=a__    ,					max_length=len(self.expected_src_tokens  )    ,					return_tensors="""pt"""    ,					)
       _lowerCAmelCase						:	Tuple        =					shift_tokens_right(batch["""labels"""]    ,					self.tokenizer.pad_token_id  )

       self.assertIsInstance(a__    ,					a__  )

       self.assertEqual((2, 14)    ,					batch.input_ids.shape  )
       self.assertEqual((2, 14)    ,					batch.attention_mask.shape  )
       _lowerCAmelCase						:	str        =					batch.input_ids.tolist()[0]
       self.assertListEqual(self.expected_src_tokens    ,					a__  )
       self.assertEqual(2    ,					batch.decoder_input_ids[0, 0]  )  # decoder_start_token_id
       # Test that special tokens are reset
       self.assertEqual(self.tokenizer.prefix_tokens    ,					[EN_CODE]  )
       self.assertEqual(self.tokenizer.suffix_tokens    ,					[self.tokenizer.eos_token_id]  )






     def    __A  (		self  ):
       _lowerCAmelCase						:	str        =					self.tokenizer(self.src_text    ,					padding=a__    ,					truncation=a__    ,					max_length=3    ,					return_tensors="""pt"""  )
       _lowerCAmelCase						:	List[Any]        =					self.tokenizer(
           text_target=self.tgt_text    ,					padding=a__    ,					truncation=a__    ,					max_length=10    ,					return_tensors="""pt"""  )
       _lowerCAmelCase						:	List[str]        =					targets["""input_ids"""]
       _lowerCAmelCase						:	Any        =					shift_tokens_right(a__    ,					self.tokenizer.pad_token_id  )

       self.assertEqual(batch.input_ids.shape[1]    ,					3  )
       self.assertEqual(batch.decoder_input_ids.shape[1]    ,					10  )






     @require_torch
     def    __A  (		self  ):
       _lowerCAmelCase						:	str        =					self.tokenizer._build_translation_inputs(
           """A test"""    ,					return_tensors="""pt"""    ,					src_lang="""en_XX"""    ,					tgt_lang="""ar_AR"""  )

       self.assertEqual(
           nested_simplify(a__  )    ,					{
               # en_XX, A, test, EOS
               """input_ids""": [[250004, 62, 3034, 2]],
               """attention_mask""": [[1, 1, 1, 1]],
               # ar_AR
               """forced_bos_token_id""": 250001,
           }    ,					)




