import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


lowerCAmelCase__     						=			get_tests_dir('fixtures/test_sentencepiece.model')


if is_torch_available():
    from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

lowerCAmelCase__     						=			256047
lowerCAmelCase__     						=			256145
@require_sentencepiece
@require_tokenizers
class     a_			(						__SCREAMING_SNAKE_CASE	,					unittest.TestCase						):






 '''simple docstring'''






 UpperCAmelCase_     			= NllbTokenizer
 UpperCAmelCase_     			= NllbTokenizerFast
 UpperCAmelCase_     			= True
 UpperCAmelCase_     			= True
 UpperCAmelCase_     			= {}




 def 		__snake_case						(      self :  List[Any]):







        '''simple docstring'''



        super().setUp()

        # We have a SentencePiece fixture for testing
        lowerCAmelCase__					     = NllbTokenizer(_snake_case      ,   keep_accents=_snake_case)
        tokenizer.save_pretrained(self.tmpdirname)




 def 		__snake_case						(      self :  Optional[Any]):







        '''simple docstring'''



        lowerCAmelCase__					     = NllbTokenizer(_snake_case      ,   keep_accents=_snake_case)

        lowerCAmelCase__					     = tokenizer.tokenize('This is a test')
        self.assertListEqual(_snake_case      ,   ['▁This', '▁is', '▁a', '▁t', 'est'])

        self.assertListEqual(
            tokenizer.convert_tokens_to_ids(_snake_case)      ,   [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]      ,   )

        lowerCAmelCase__					     = tokenizer.tokenize('I was born in 92000, and this is falsé.')
        self.assertListEqual(
            _snake_case      ,   [
                SPIECE_UNDERLINE + 'I',
                SPIECE_UNDERLINE + 'was',
                SPIECE_UNDERLINE + 'b',
                'or',
                'n',
                SPIECE_UNDERLINE + 'in',
                SPIECE_UNDERLINE + '',
                '9',
                '2',
                '0',
                '0',
                '0',
                ',',
                SPIECE_UNDERLINE + 'and',
                SPIECE_UNDERLINE + 'this',
                SPIECE_UNDERLINE + 'is',
                SPIECE_UNDERLINE + 'f',
                'al',
                's',
                'é',
                '.',
            ]      ,   )
        lowerCAmelCase__					     = tokenizer.convert_tokens_to_ids(_snake_case)
        self.assertListEqual(
            _snake_case      ,   [
                value + tokenizer.fairseq_offset
                for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
            ]      ,   )

        lowerCAmelCase__					     = tokenizer.convert_ids_to_tokens(_snake_case)
        self.assertListEqual(
            _snake_case      ,   [
                SPIECE_UNDERLINE + 'I',
                SPIECE_UNDERLINE + 'was',
                SPIECE_UNDERLINE + 'b',
                'or',
                'n',
                SPIECE_UNDERLINE + 'in',
                SPIECE_UNDERLINE + '',
                '<unk>',
                '2',
                '0',
                '0',
                '0',
                ',',
                SPIECE_UNDERLINE + 'and',
                SPIECE_UNDERLINE + 'this',
                SPIECE_UNDERLINE + 'is',
                SPIECE_UNDERLINE + 'f',
                'al',
                's',
                '<unk>',
                '.',
            ]      ,   )




 def 		__snake_case						(      self :  List[Any]):







        '''simple docstring'''



        lowerCAmelCase__					     = (self.rust_tokenizer_class, "hf-internal-testing/tiny-random-nllb", {})
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
                      lowerCAmelCase__					     = self.rust_tokenizer_class.from_pretrained(_snake_case      ,   **_snake_case)
                      lowerCAmelCase__					     = self.tokenizer_class.from_pretrained(_snake_case      ,   **_snake_case)

                      lowerCAmelCase__					     = tempfile.mkdtemp()

                      lowerCAmelCase__					     = tokenizer_r.save_pretrained(_snake_case)
                      lowerCAmelCase__					     = tokenizer_p.save_pretrained(_snake_case)

                      # Checks it save with the same files + the tokenizer.json file for the fast one
                      self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files))
                      lowerCAmelCase__					     = tuple(f for f in tokenizer_r_files if 'tokenizer.json' not in f)
                      self.assertSequenceEqual(_snake_case      ,   _snake_case)

                      # Checks everything loads correctly in the same way
                      lowerCAmelCase__					     = tokenizer_r.from_pretrained(_snake_case)
                      lowerCAmelCase__					     = tokenizer_p.from_pretrained(_snake_case)

                      # Check special tokens are set accordingly on Rust and Python
                      for key in tokenizer_pp.special_tokens_map:
                             self.assertTrue(hasattr(_snake_case      ,   _snake_case))

                      shutil.rmtree(_snake_case)

                      # Save tokenizer rust, legacy_format=True
                      lowerCAmelCase__					     = tempfile.mkdtemp()

                      lowerCAmelCase__					     = tokenizer_r.save_pretrained(_snake_case      ,   legacy_format=_snake_case)
                      lowerCAmelCase__					     = tokenizer_p.save_pretrained(_snake_case)

                      # Checks it save with the same files
                      self.assertSequenceEqual(_snake_case      ,   _snake_case)

                      # Checks everything loads correctly in the same way
                      lowerCAmelCase__					     = tokenizer_r.from_pretrained(_snake_case)
                      lowerCAmelCase__					     = tokenizer_p.from_pretrained(_snake_case)

                      # Check special tokens are set accordingly on Rust and Python
                      for key in tokenizer_pp.special_tokens_map:
                             self.assertTrue(hasattr(_snake_case      ,   _snake_case))

                      shutil.rmtree(_snake_case)

                      # Save tokenizer rust, legacy_format=False
                      lowerCAmelCase__					     = tempfile.mkdtemp()

                      lowerCAmelCase__					     = tokenizer_r.save_pretrained(_snake_case      ,   legacy_format=_snake_case)
                      lowerCAmelCase__					     = tokenizer_p.save_pretrained(_snake_case)

                      # Checks it saved the tokenizer.json file
                      self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files))

                      # Checks everything loads correctly in the same way
                      lowerCAmelCase__					     = tokenizer_r.from_pretrained(_snake_case)
                      lowerCAmelCase__					     = tokenizer_p.from_pretrained(_snake_case)

                      # Check special tokens are set accordingly on Rust and Python
                      for key in tokenizer_pp.special_tokens_map:
                             self.assertTrue(hasattr(_snake_case      ,   _snake_case))

                      shutil.rmtree(_snake_case)




 @require_torch
 def 		__snake_case						(      self :  str):







        '''simple docstring'''



        if not self.test_seqaseq:
               return

        lowerCAmelCase__					     = self.get_tokenizers()
        for tokenizer in tokenizers:
               with self.subTest(F"""{tokenizer.__class__.__name__}"""):
                      # Longer text that will definitely require truncation.
                      lowerCAmelCase__					     = [
                          " UN Chief Says There Is No Military Solution in Syria",
                          " Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for"
                          " Syria is that 'there is no military solution' to the nearly five-year conflict and more weapons"
                          " will only worsen the violence and misery for millions of people.",
                      ]
                      lowerCAmelCase__					     = [
                          "Şeful ONU declară că nu există o soluţie militară în Siria",
                          "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al"
                          " Rusiei pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi"
                          " că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.",
                      ]
                      try:
                             lowerCAmelCase__					     = tokenizer.prepare_seqaseq_batch(
                                 src_texts=_snake_case      ,   tgt_texts=_snake_case      ,   max_length=3      ,   max_target_length=10      ,   return_tensors='pt'      ,   src_lang='eng_Latn'      ,   tgt_lang='ron_Latn'      ,   )
                      except NotImplementedError:
                             return
                      self.assertEqual(batch.input_ids.shape[1]      ,   3)
                      self.assertEqual(batch.labels.shape[1]      ,   10)
                      # max_target_length will default to max_length if not specified
                      lowerCAmelCase__					     = tokenizer.prepare_seqaseq_batch(
                          _snake_case      ,   tgt_texts=_snake_case      ,   max_length=3      ,   return_tensors='pt')
                      self.assertEqual(batch.input_ids.shape[1]      ,   3)
                      self.assertEqual(batch.labels.shape[1]      ,   3)

                      lowerCAmelCase__					     = tokenizer.prepare_seqaseq_batch(
                          src_texts=_snake_case      ,   max_length=3      ,   max_target_length=10      ,   return_tensors='pt')
                      self.assertEqual(batch_encoder_only.input_ids.shape[1]      ,   3)
                      self.assertEqual(batch_encoder_only.attention_mask.shape[1]      ,   3)
                      self.assertNotIn('decoder_input_ids'      ,   _snake_case)




 @unittest.skip('Unfortunately way too slow to build a BPE with SentencePiece.')
 def 		__snake_case						(      self :  Any):







        '''simple docstring'''



        pass




 def 		__snake_case						(      self :  Tuple):







        '''simple docstring'''



        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
                      lowerCAmelCase__					     = [AddedToken('<special>'      ,   lstrip=_snake_case)]

                      lowerCAmelCase__					     = self.rust_tokenizer_class.from_pretrained(
                          _snake_case      ,   additional_special_tokens=_snake_case      ,   **_snake_case)
                      lowerCAmelCase__					     = tokenizer_r.encode('Hey this is a <special> token')

                      lowerCAmelCase__					     = tokenizer_r.encode('<special>'      ,   add_special_tokens=_snake_case)[0]

                      self.assertTrue(special_token_id in r_output)

                      if self.test_slow_tokenizer:
                             lowerCAmelCase__					     = self.rust_tokenizer_class.from_pretrained(
                                 _snake_case      ,   additional_special_tokens=_snake_case      ,   **_snake_case      ,   )
                             lowerCAmelCase__					     = self.tokenizer_class.from_pretrained(
                                 _snake_case      ,   additional_special_tokens=_snake_case      ,   **_snake_case)

                             lowerCAmelCase__					     = tokenizer_p.encode('Hey this is a <special> token')

                             lowerCAmelCase__					     = tokenizer_cr.encode('Hey this is a <special> token')

                             self.assertEqual(_snake_case      ,   _snake_case)
                             self.assertEqual(_snake_case      ,   _snake_case)
                             self.assertTrue(special_token_id in p_output)
                             self.assertTrue(special_token_id in cr_output)






@require_torch
@require_sentencepiece
@require_tokenizers
class     a_			(						unittest.TestCase						):






 '''simple docstring'''






 UpperCAmelCase_     			= "facebook/nllb-200-distilled-600M"
 UpperCAmelCase_     			= [
     " UN Chief Says There Is No Military Solution in Syria",
     " Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.",
 ]
 UpperCAmelCase_     			= [
     "Şeful ONU declară că nu există o soluţie militară în Siria",
     "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"
     " pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"
     " face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.",
 ]
 UpperCAmelCase_     			= [
     256_047,
     16_297,
     134_408,
     8_165,
     248_066,
     14_734,
     950,
     1_135,
     105_721,
     3_573,
     83,
     27_352,
     108,
     49_486,
     2,
 ]




 @classmethod
 def 		__snake_case						(      cls :  Optional[Any]):







        '''simple docstring'''



        lowerCAmelCase__					     = NllbTokenizer.from_pretrained(
            cls.checkpoint_name      ,   src_lang='eng_Latn'      ,   tgt_lang='ron_Latn')
        lowerCAmelCase__					     = 1
        return cls




 def 		__snake_case						(      self :  Dict):







        '''simple docstring'''



        self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ace_Arab']      ,   256_001)
        self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ace_Latn']      ,   256_002)
        self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['fra_Latn']      ,   256_057)




 def 		__snake_case						(      self :  int):







        '''simple docstring'''



        lowerCAmelCase__					     = self.tokenizer.batch_encode_plus(self.src_text).input_ids[0]
        self.assertListEqual(self.expected_src_tokens      ,   _snake_case)




 def 		__snake_case						(      self :  Any):







        '''simple docstring'''



        self.assertIn(_snake_case      ,   self.tokenizer.all_special_ids)
        # fmt: off
        lowerCAmelCase__					     = [RO_CODE, 4_254, 98_068, 112_923, 39_072, 3_909, 713, 102_767, 26, 17_314, 35_642, 14_683, 33_118, 2_022, 66_987, 2, 256_047]
        # fmt: on

        lowerCAmelCase__					     = self.tokenizer.decode(_snake_case      ,   skip_special_tokens=_snake_case)
        lowerCAmelCase__					     = self.tokenizer.decode(generated_ids[1:]      ,   skip_special_tokens=_snake_case)
        self.assertEqual(_snake_case      ,   _snake_case)
        self.assertNotIn(self.tokenizer.eos_token      ,   _snake_case)




 def 		__snake_case						(      self :  Union[str, Any]):







        '''simple docstring'''



        lowerCAmelCase__					     = ["this is gunna be a long sentence " * 20]
        assert isinstance(src_text[0]      ,   _snake_case)
        lowerCAmelCase__					     = 10
        lowerCAmelCase__					     = self.tokenizer(_snake_case      ,   max_length=_snake_case      ,   truncation=_snake_case).input_ids[0]
        self.assertEqual(ids[-1]      ,   2)
        self.assertEqual(ids[0]      ,   _snake_case)
        self.assertEqual(len(_snake_case)      ,   _snake_case)




 def 		__snake_case						(      self :  str):







        '''simple docstring'''



        self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['<mask>', 'ar_AR'])      ,   [256_203, 3])




 def 		__snake_case						(      self :  Tuple):







        '''simple docstring'''



        lowerCAmelCase__					     = tempfile.mkdtemp()
        lowerCAmelCase__					     = self.tokenizer.fairseq_tokens_to_ids
        self.tokenizer.save_pretrained(_snake_case)
        lowerCAmelCase__					     = NllbTokenizer.from_pretrained(_snake_case)
        self.assertDictEqual(new_tok.fairseq_tokens_to_ids      ,   _snake_case)




 @require_torch
 def 		__snake_case						(      self :  Tuple):







        '''simple docstring'''



        lowerCAmelCase__					     = self.tokenizer(
            self.src_text      ,   text_target=self.tgt_text      ,   padding=_snake_case      ,   truncation=_snake_case      ,   max_length=len(self.expected_src_tokens)      ,   return_tensors='pt'      ,   )
        lowerCAmelCase__					     = shift_tokens_right(
            batch['labels']      ,   self.tokenizer.pad_token_id      ,   self.tokenizer.lang_code_to_id['ron_Latn'])

        self.assertIsInstance(_snake_case      ,   _snake_case)

        self.assertEqual((2, 15)      ,   batch.input_ids.shape)
        self.assertEqual((2, 15)      ,   batch.attention_mask.shape)
        lowerCAmelCase__					     = batch.input_ids.tolist()[0]
        self.assertListEqual(self.expected_src_tokens      ,   _snake_case)
        self.assertEqual(_snake_case      ,   batch.decoder_input_ids[0, 0])  # EOS
        # Test that special tokens are reset
        self.assertEqual(self.tokenizer.prefix_tokens      ,   [EN_CODE])
        self.assertEqual(self.tokenizer.suffix_tokens      ,   [self.tokenizer.eos_token_id])




 def 		__snake_case						(      self :  List[Any]):







        '''simple docstring'''



        lowerCAmelCase__					     = self.tokenizer(self.src_text      ,   padding=_snake_case      ,   truncation=_snake_case      ,   max_length=3      ,   return_tensors='pt')
        lowerCAmelCase__					     = self.tokenizer(
            text_target=self.tgt_text      ,   padding=_snake_case      ,   truncation=_snake_case      ,   max_length=10      ,   return_tensors='pt')
        lowerCAmelCase__					     = targets["input_ids"]
        lowerCAmelCase__					     = shift_tokens_right(
            _snake_case      ,   self.tokenizer.pad_token_id      ,   decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]      ,   )

        self.assertEqual(batch.input_ids.shape[1]      ,   3)
        self.assertEqual(batch.decoder_input_ids.shape[1]      ,   10)




 @require_torch
 def 		__snake_case						(      self :  Optional[int]):







        '''simple docstring'''



        lowerCAmelCase__					     = self.tokenizer._build_translation_inputs(
            'A test'      ,   return_tensors='pt'      ,   src_lang='eng_Latn'      ,   tgt_lang='fra_Latn')

        self.assertEqual(
            nested_simplify(_snake_case)      ,   {
                # A, test, EOS, en_XX
                'input_ids': [[256_047, 70, 7_356, 2]],
                'attention_mask': [[1, 1, 1, 1]],
                # ar_AR
                'forced_bos_token_id': 256_057,
            }      ,   )




 @require_torch
 def 		__snake_case						(      self :  int):







        '''simple docstring'''



        lowerCAmelCase__					     = True
        lowerCAmelCase__					     = self.tokenizer(
            'UN Chief says there is no military solution in Syria'      ,   src_lang='eng_Latn'      ,   tgt_lang='fra_Latn')
        self.assertEqual(
            inputs.input_ids      ,   [16_297, 134_408, 25_653, 6_370, 248, 254, 103_929, 94_995, 108, 49_486, 2, 256_047])

        lowerCAmelCase__					     = False
        lowerCAmelCase__					     = self.tokenizer(
            'UN Chief says there is no military solution in Syria'      ,   src_lang='eng_Latn'      ,   tgt_lang='fra_Latn')
        self.assertEqual(
            inputs.input_ids      ,   [256_047, 16_297, 134_408, 25_653, 6_370, 248, 254, 103_929, 94_995, 108, 49_486, 2])


from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


lowerCAmelCase__     						=			{
    'configuration_x_clip': [
        'XCLIP_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'XCLIPConfig',
        'XCLIPTextConfig',
        'XCLIPVisionConfig',
    ],
    'processing_x_clip': ['XCLIPProcessor'],
}

try:
    if not is_torch_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    lowerCAmelCase__     						=			[
        'XCLIP_PRETRAINED_MODEL_ARCHIVE_LIST',
        'XCLIPModel',
        'XCLIPPreTrainedModel',
        'XCLIPTextModel',
        'XCLIPVisionModel',
    ]

if TYPE_CHECKING:
    from .configuration_x_clip import (
        XCLIP_PRETRAINED_CONFIG_ARCHIVE_MAP,
        XCLIPConfig,
        XCLIPTextConfig,
        XCLIPVisionConfig,
    )
    from .processing_x_clip import XCLIPProcessor

    try:
        if not is_torch_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_x_clip import (
            XCLIP_PRETRAINED_MODEL_ARCHIVE_LIST,
            XCLIPModel,
            XCLIPPreTrainedModel,
            XCLIPTextModel,
            XCLIPVisionModel,
        )

else:
    import sys

    lowerCAmelCase__     						=			_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)
