



import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


__snake_case    :		Optional[int]  =get_tests_dir('fixtures/test_sentencepiece.model')


if is_torch_available():
							from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

__snake_case    :		Optional[Any]  =2_5_6_0_4_7
__snake_case    :		List[Any]  =2_5_6_1_4_5



@require_sentencepiece
@require_tokenizers
class 	lowerCamelCase__      (							lowerCamelCase__				,    unittest.TestCase):




		'''simple docstring'''



		snake_case_     =NllbTokenizer
		snake_case_     =NllbTokenizerFast
		snake_case_     =True
		snake_case_     =True
		snake_case_     ={}



		def 		lowerCAmelCase__				(self			)					->      Optional[Any]:

						"""simple docstring"""






						super().setUp()

						# We have a SentencePiece fixture for testing
						lowerCAmelCase__					:    Optional[Any]												=					NllbTokenizer(__lowerCamelCase		,keep_accents=__lowerCamelCase			)
						tokenizer.save_pretrained(self.tmpdirname			)



		def 		lowerCAmelCase__				(self			)					->      Union[str, Any]:

						"""simple docstring"""






						lowerCAmelCase__					:    Optional[int]												=					NllbTokenizer(__lowerCamelCase		,keep_accents=__lowerCamelCase			)

						lowerCAmelCase__					:    Tuple												=					tokenizer.tokenize('''This is a test'''			)
						self.assertListEqual(__lowerCamelCase		,['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']			)

						self.assertListEqual(
						    tokenizer.convert_tokens_to_ids(__lowerCamelCase			)		,[value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]		,)

						lowerCAmelCase__					:    Dict												=					tokenizer.tokenize('''I was born in 92000, and this is falsé.'''			)
						self.assertListEqual(
						    __lowerCamelCase		,[
						        SPIECE_UNDERLINE + '''I''',
						        SPIECE_UNDERLINE + '''was''',
						        SPIECE_UNDERLINE + '''b''',
						        '''or''',
						        '''n''',
						        SPIECE_UNDERLINE + '''in''',
						        SPIECE_UNDERLINE + '''''',
						        '''9''',
						        '''2''',
						        '''0''',
						        '''0''',
						        '''0''',
						        ''',''',
						        SPIECE_UNDERLINE + '''and''',
						        SPIECE_UNDERLINE + '''this''',
						        SPIECE_UNDERLINE + '''is''',
						        SPIECE_UNDERLINE + '''f''',
						        '''al''',
						        '''s''',
						        '''é''',
						        '''.''',
						    ]		,)
						lowerCAmelCase__					:    Tuple												=					tokenizer.convert_tokens_to_ids(__lowerCamelCase			)
						self.assertListEqual(
						    __lowerCamelCase		,[
						        value + tokenizer.fairseq_offset
						        for value in [8, 21, 84, 55, 24, 19, 7, 2, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 2, 4]
						    ]		,)

						lowerCAmelCase__					:    Tuple												=					tokenizer.convert_ids_to_tokens(__lowerCamelCase			)
						self.assertListEqual(
						    __lowerCamelCase		,[
						        SPIECE_UNDERLINE + '''I''',
						        SPIECE_UNDERLINE + '''was''',
						        SPIECE_UNDERLINE + '''b''',
						        '''or''',
						        '''n''',
						        SPIECE_UNDERLINE + '''in''',
						        SPIECE_UNDERLINE + '''''',
						        '''<unk>''',
						        '''2''',
						        '''0''',
						        '''0''',
						        '''0''',
						        ''',''',
						        SPIECE_UNDERLINE + '''and''',
						        SPIECE_UNDERLINE + '''this''',
						        SPIECE_UNDERLINE + '''is''',
						        SPIECE_UNDERLINE + '''f''',
						        '''al''',
						        '''s''',
						        '''<unk>''',
						        '''.''',
						    ]		,)



		def 		lowerCAmelCase__				(self			)					->      Any:

						"""simple docstring"""






						lowerCAmelCase__					:    int												=					(self.rust_tokenizer_class, '''hf-internal-testing/tiny-random-nllb''', {})
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""			):
														lowerCAmelCase__					:    Optional[Any]												=					self.rust_tokenizer_class.from_pretrained(__lowerCamelCase		,**__lowerCamelCase			)
														lowerCAmelCase__					:    int												=					self.tokenizer_class.from_pretrained(__lowerCamelCase		,**__lowerCamelCase			)

														lowerCAmelCase__					:    Dict												=					tempfile.mkdtemp()

														lowerCAmelCase__					:    Optional[Any]												=					tokenizer_r.save_pretrained(__lowerCamelCase			)
														lowerCAmelCase__					:    Any												=					tokenizer_p.save_pretrained(__lowerCamelCase			)

														# Checks it save with the same files + the tokenizer.json file for the fast one
														self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files			)			)
														lowerCAmelCase__					:    Tuple												=					tuple(f for f in tokenizer_r_files if '''tokenizer.json''' not in f			)
														self.assertSequenceEqual(__lowerCamelCase		,__lowerCamelCase			)

														# Checks everything loads correctly in the same way
														lowerCAmelCase__					:    Any												=					tokenizer_r.from_pretrained(__lowerCamelCase			)
														lowerCAmelCase__					:    Dict												=					tokenizer_p.from_pretrained(__lowerCamelCase			)

														# Check special tokens are set accordingly on Rust and Python
														for key in tokenizer_pp.special_tokens_map:
																		self.assertTrue(hasattr(__lowerCamelCase		,__lowerCamelCase			)			)

														shutil.rmtree(__lowerCamelCase			)

														# Save tokenizer rust, legacy_format=True
														lowerCAmelCase__					:    Tuple												=					tempfile.mkdtemp()

														lowerCAmelCase__					:    List[Any]												=					tokenizer_r.save_pretrained(__lowerCamelCase		,legacy_format=__lowerCamelCase			)
														lowerCAmelCase__					:    Union[str, Any]												=					tokenizer_p.save_pretrained(__lowerCamelCase			)

														# Checks it save with the same files
														self.assertSequenceEqual(__lowerCamelCase		,__lowerCamelCase			)

														# Checks everything loads correctly in the same way
														lowerCAmelCase__					:    List[Any]												=					tokenizer_r.from_pretrained(__lowerCamelCase			)
														lowerCAmelCase__					:    int												=					tokenizer_p.from_pretrained(__lowerCamelCase			)

														# Check special tokens are set accordingly on Rust and Python
														for key in tokenizer_pp.special_tokens_map:
																		self.assertTrue(hasattr(__lowerCamelCase		,__lowerCamelCase			)			)

														shutil.rmtree(__lowerCamelCase			)

														# Save tokenizer rust, legacy_format=False
														lowerCAmelCase__					:    Any												=					tempfile.mkdtemp()

														lowerCAmelCase__					:    Any												=					tokenizer_r.save_pretrained(__lowerCamelCase		,legacy_format=__lowerCamelCase			)
														lowerCAmelCase__					:    Dict												=					tokenizer_p.save_pretrained(__lowerCamelCase			)

														# Checks it saved the tokenizer.json file
														self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files			)			)

														# Checks everything loads correctly in the same way
														lowerCAmelCase__					:    Dict												=					tokenizer_r.from_pretrained(__lowerCamelCase			)
														lowerCAmelCase__					:    Tuple												=					tokenizer_p.from_pretrained(__lowerCamelCase			)

														# Check special tokens are set accordingly on Rust and Python
														for key in tokenizer_pp.special_tokens_map:
																		self.assertTrue(hasattr(__lowerCamelCase		,__lowerCamelCase			)			)

														shutil.rmtree(__lowerCamelCase			)



		@require_torch
		def 		lowerCAmelCase__				(self			)					->      Tuple:

						"""simple docstring"""






						if not self.test_seqaseq:
										return

						lowerCAmelCase__					:    int												=					self.get_tokenizers()
						for tokenizer in tokenizers:
										with self.subTest(f"""{tokenizer.__class__.__name__}"""			):
														# Longer text that will definitely require truncation.
														lowerCAmelCase__					:    List[str]												=					[
														    ''' UN Chief Says There Is No Military Solution in Syria''',
														    ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for'''
														    ''' Syria is that \'there is no military solution\' to the nearly five-year conflict and more weapons'''
														    ''' will only worsen the violence and misery for millions of people.''',
														]
														lowerCAmelCase__					:    Any												=					[
														    '''Şeful ONU declară că nu există o soluţie militară în Siria''',
														    '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al'''
														    ''' Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi'''
														    ''' că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
														]
														try:
																		lowerCAmelCase__					:    List[str]												=					tokenizer.prepare_seqaseq_batch(
																		    src_texts=__lowerCamelCase		,tgt_texts=__lowerCamelCase		,max_length=3		,max_target_length=10		,return_tensors='''pt'''		,src_lang='''eng_Latn'''		,tgt_lang='''ron_Latn'''		,)
														except NotImplementedError:
																		return
														self.assertEqual(batch.input_ids.shape[1]		,3			)
														self.assertEqual(batch.labels.shape[1]		,10			)
														# max_target_length will default to max_length if not specified
														lowerCAmelCase__					:    Tuple												=					tokenizer.prepare_seqaseq_batch(
														    __lowerCamelCase		,tgt_texts=__lowerCamelCase		,max_length=3		,return_tensors='''pt'''			)
														self.assertEqual(batch.input_ids.shape[1]		,3			)
														self.assertEqual(batch.labels.shape[1]		,3			)

														lowerCAmelCase__					:    Tuple												=					tokenizer.prepare_seqaseq_batch(
														    src_texts=__lowerCamelCase		,max_length=3		,max_target_length=10		,return_tensors='''pt'''			)
														self.assertEqual(batch_encoder_only.input_ids.shape[1]		,3			)
														self.assertEqual(batch_encoder_only.attention_mask.shape[1]		,3			)
														self.assertNotIn('''decoder_input_ids'''		,__lowerCamelCase			)



		@unittest.skip('''Unfortunately way too slow to build a BPE with SentencePiece.'''			)
		def 		lowerCAmelCase__				(self			)					->      Any:

						"""simple docstring"""






						pass



		def 		lowerCAmelCase__				(self			)					->      Tuple:

						"""simple docstring"""






						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""			):
														lowerCAmelCase__					:    List[str]												=					[AddedToken('''<special>'''		,lstrip=__lowerCamelCase			)]

														lowerCAmelCase__					:    List[str]												=					self.rust_tokenizer_class.from_pretrained(
														    __lowerCamelCase		,additional_special_tokens=__lowerCamelCase		,**__lowerCamelCase			)
														lowerCAmelCase__					:    Dict												=					tokenizer_r.encode('''Hey this is a <special> token'''			)

														lowerCAmelCase__					:    int												=					tokenizer_r.encode('''<special>'''		,add_special_tokens=__lowerCamelCase			)[0]

														self.assertTrue(special_token_id in r_output			)

														if self.test_slow_tokenizer:
																		lowerCAmelCase__					:    List[str]												=					self.rust_tokenizer_class.from_pretrained(
																		    __lowerCamelCase		,additional_special_tokens=__lowerCamelCase		,**__lowerCamelCase		,)
																		lowerCAmelCase__					:    Union[str, Any]												=					self.tokenizer_class.from_pretrained(
																		    __lowerCamelCase		,additional_special_tokens=__lowerCamelCase		,**__lowerCamelCase			)

																		lowerCAmelCase__					:    int												=					tokenizer_p.encode('''Hey this is a <special> token'''			)

																		lowerCAmelCase__					:    List[str]												=					tokenizer_cr.encode('''Hey this is a <special> token'''			)

																		self.assertEqual(__lowerCamelCase		,__lowerCamelCase			)
																		self.assertEqual(__lowerCamelCase		,__lowerCamelCase			)
																		self.assertTrue(special_token_id in p_output			)
																		self.assertTrue(special_token_id in cr_output			)



@require_torch
@require_sentencepiece
@require_tokenizers
class 	lowerCamelCase__      (							unittest.TestCase):




		'''simple docstring'''



		snake_case_     ="""facebook/nllb-200-distilled-600M"""
		snake_case_     =[
		    """ UN Chief Says There Is No Military Solution in Syria""",
		    """ Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.""",
		]
		snake_case_     =[
		    """Şeful ONU declară că nu există o soluţie militară în Siria""",
		    """Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"""
		    """ pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"""
		    """ face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.""",
		]
		snake_case_     =[
		    25_6047,
		    1_6297,
		    13_4408,
		    8165,
		    24_8066,
		    1_4734,
		    950,
		    1135,
		    10_5721,
		    3573,
		    83,
		    2_7352,
		    108,
		    4_9486,
		    2,
		]



		@classmethod
		def 		lowerCAmelCase__				(cls			)					->      Optional[int]:

						"""simple docstring"""






						lowerCAmelCase__					:    NllbTokenizer												=					NllbTokenizer.from_pretrained(
						    cls.checkpoint_name		,src_lang='''eng_Latn'''		,tgt_lang='''ron_Latn'''			)
						lowerCAmelCase__					:    int												=					1
						return cls



		def 		lowerCAmelCase__				(self			)					->      Optional[int]:

						"""simple docstring"""






						self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Arab''']		,25_60_01			)
						self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Latn''']		,25_60_02			)
						self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''fra_Latn''']		,25_60_57			)



		def 		lowerCAmelCase__				(self			)					->      int:

						"""simple docstring"""






						lowerCAmelCase__					:    str												=					self.tokenizer.batch_encode_plus(self.src_text			).input_ids[0]
						self.assertListEqual(self.expected_src_tokens		,__lowerCamelCase			)



		def 		lowerCAmelCase__				(self			)					->      List[Any]:

						"""simple docstring"""






						self.assertIn(__lowerCamelCase		,self.tokenizer.all_special_ids			)
						# fmt: off
						lowerCAmelCase__					:    List[str]												=					[RO_CODE, 42_54, 9_80_68, 11_29_23, 3_90_72, 39_09, 7_13, 10_27_67, 26, 1_73_14, 3_56_42, 1_46_83, 3_31_18, 20_22, 6_69_87, 2, 25_60_47]
						# fmt: on

						lowerCAmelCase__					:    int												=					self.tokenizer.decode(__lowerCamelCase		,skip_special_tokens=__lowerCamelCase			)
						lowerCAmelCase__					:    str												=					self.tokenizer.decode(generated_ids[1:]		,skip_special_tokens=__lowerCamelCase			)
						self.assertEqual(__lowerCamelCase		,__lowerCamelCase			)
						self.assertNotIn(self.tokenizer.eos_token		,__lowerCamelCase			)



		def 		lowerCAmelCase__				(self			)					->      List[Any]:

						"""simple docstring"""






						lowerCAmelCase__					:    Union[str, Any]												=					['''this is gunna be a long sentence ''' * 20]
						assert isinstance(src_text[0]		,__lowerCamelCase			)
						lowerCAmelCase__					:    List[str]												=					10
						lowerCAmelCase__					:    int												=					self.tokenizer(__lowerCamelCase		,max_length=__lowerCamelCase		,truncation=__lowerCamelCase			).input_ids[0]
						self.assertEqual(ids[-1]		,2			)
						self.assertEqual(ids[0]		,__lowerCamelCase			)
						self.assertEqual(len(__lowerCamelCase			)		,__lowerCamelCase			)



		def 		lowerCAmelCase__				(self			)					->      Tuple:

						"""simple docstring"""






						self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['''<mask>''', '''ar_AR''']			)		,[25_62_03, 3]			)



		def 		lowerCAmelCase__				(self			)					->      Dict:

						"""simple docstring"""






						lowerCAmelCase__					:    Dict												=					tempfile.mkdtemp()
						lowerCAmelCase__					:    Tuple												=					self.tokenizer.fairseq_tokens_to_ids
						self.tokenizer.save_pretrained(__lowerCamelCase			)
						lowerCAmelCase__					:    List[str]												=					NllbTokenizer.from_pretrained(__lowerCamelCase			)
						self.assertDictEqual(new_tok.fairseq_tokens_to_ids		,__lowerCamelCase			)



		@require_torch
		def 		lowerCAmelCase__				(self			)					->      Union[str, Any]:

						"""simple docstring"""






						lowerCAmelCase__					:    Any												=					self.tokenizer(
						    self.src_text		,text_target=self.tgt_text		,padding=__lowerCamelCase		,truncation=__lowerCamelCase		,max_length=len(self.expected_src_tokens			)		,return_tensors='''pt'''		,)
						lowerCAmelCase__					:    int												=					shift_tokens_right(
						    batch['''labels''']		,self.tokenizer.pad_token_id		,self.tokenizer.lang_code_to_id['''ron_Latn''']			)

						self.assertIsInstance(__lowerCamelCase		,__lowerCamelCase			)

						self.assertEqual((2, 15)		,batch.input_ids.shape			)
						self.assertEqual((2, 15)		,batch.attention_mask.shape			)
						lowerCAmelCase__					:    List[str]												=					batch.input_ids.tolist()[0]
						self.assertListEqual(self.expected_src_tokens		,__lowerCamelCase			)
						self.assertEqual(__lowerCamelCase		,batch.decoder_input_ids[0, 0]			)  # EOS
						# Test that special tokens are reset
						self.assertEqual(self.tokenizer.prefix_tokens		,[EN_CODE]			)
						self.assertEqual(self.tokenizer.suffix_tokens		,[self.tokenizer.eos_token_id]			)



		def 		lowerCAmelCase__				(self			)					->      int:

						"""simple docstring"""






						lowerCAmelCase__					:    Optional[Any]												=					self.tokenizer(self.src_text		,padding=__lowerCamelCase		,truncation=__lowerCamelCase		,max_length=3		,return_tensors='''pt'''			)
						lowerCAmelCase__					:    Optional[Any]												=					self.tokenizer(
						    text_target=self.tgt_text		,padding=__lowerCamelCase		,truncation=__lowerCamelCase		,max_length=10		,return_tensors='''pt'''			)
						lowerCAmelCase__					:    str												=					targets['''input_ids''']
						lowerCAmelCase__					:    List[Any]												=					shift_tokens_right(
						    __lowerCamelCase		,self.tokenizer.pad_token_id		,decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]		,)

						self.assertEqual(batch.input_ids.shape[1]		,3			)
						self.assertEqual(batch.decoder_input_ids.shape[1]		,10			)



		@require_torch
		def 		lowerCAmelCase__				(self			)					->      List[str]:

						"""simple docstring"""






						lowerCAmelCase__					:    str												=					self.tokenizer._build_translation_inputs(
						    '''A test'''		,return_tensors='''pt'''		,src_lang='''eng_Latn'''		,tgt_lang='''fra_Latn'''			)

						self.assertEqual(
						    nested_simplify(__lowerCamelCase			)		,{
						        # A, test, EOS, en_XX
						        '''input_ids''': [[25_60_47, 70, 73_56, 2]],
						        '''attention_mask''': [[1, 1, 1, 1]],
						        # ar_AR
						        '''forced_bos_token_id''': 25_60_57,
						    }		,)



		@require_torch
		def 		lowerCAmelCase__				(self			)					->      Dict:

						"""simple docstring"""






						lowerCAmelCase__					:    List[Any]												=					True
						lowerCAmelCase__					:    Any												=					self.tokenizer(
						    '''UN Chief says there is no military solution in Syria'''		,src_lang='''eng_Latn'''		,tgt_lang='''fra_Latn'''			)
						self.assertEqual(
						    inputs.input_ids		,[1_62_97, 13_44_08, 2_56_53, 63_70, 2_48, 2_54, 10_39_29, 9_49_95, 1_08, 4_94_86, 2, 25_60_47]			)

						lowerCAmelCase__					:    List[Any]												=					False
						lowerCAmelCase__					:    Dict												=					self.tokenizer(
						    '''UN Chief says there is no military solution in Syria'''		,src_lang='''eng_Latn'''		,tgt_lang='''fra_Latn'''			)
						self.assertEqual(
						    inputs.input_ids		,[25_60_47, 1_62_97, 13_44_08, 2_56_53, 63_70, 2_48, 2_54, 10_39_29, 9_49_95, 1_08, 4_94_86, 2]			)






__snake_case    :		Any  ='\n# Transformers installation\n! pip install transformers datasets\n# To install from source instead of the last release, comment the command above and uncomment the following one.\n# ! pip install git+https://github.com/huggingface/transformers.git\n'

__snake_case    :		Tuple  =[{'type': 'code', 'content': INSTALL_CONTENT}]
__snake_case    :		Tuple  ={
    '{processor_class}': 'FakeProcessorClass',
    '{model_class}': 'FakeModelClass',
    '{object_class}': 'FakeObjectClass',
}

