




import json
import os
import shutil
import tempfile
import unittest

from transformers import BatchEncoding, CanineTokenizer
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.tokenization_utils import AddedToken
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin






class SCREAMING_SNAKE_CASE__					(		SCREAMING_SNAKE_CASE__			,						unittest.TestCase   ):




    '''simple docstring'''






    __lowerCamelCase      :			Dict =				CanineTokenizer
    __lowerCamelCase      :			Optional[Any] =				False






    def       _lowerCAmelCase (	self		):
        super().setUp()
        A			:	Union[str, Any]										=       CanineTokenizer()
        tokenizer.save_pretrained(self.tmpdirname		)






    @cached_property
    def       _lowerCAmelCase (	self		):
        return CanineTokenizer.from_pretrained("""google/canine-s"""		)






    def       _lowerCAmelCase (	self,		**lowerCamelCase__		):
        A			:	Optional[Any]										=       self.tokenizer_class.from_pretrained(self.tmpdirname,		**lowerCamelCase__		)
        A			:	Dict										=       1024
        return tokenizer






    @require_torch
    def       _lowerCAmelCase (	self		):
        A			:	List[Any]										=       self.canine_tokenizer
        A			:	Union[str, Any]										=       ['''Life is like a box of chocolates.''', '''You never know what you\'re gonna get.''']
        # fmt: off
        A			:	Tuple										=       [5_7344, 76, 105, 102, 101, 32, 105, 115, 32, 108, 105, 107, 101, 32, 97, 32, 98, 111, 120, 32, 111, 102, 32, 99, 104, 111, 99, 111, 108, 97, 116, 101, 115, 46, 5_7345, 0, 0, 0, 0]
        # fmt: on
        A			:	Optional[Any]										=       tokenizer(lowerCamelCase__,		padding=lowerCamelCase__,		return_tensors="""pt"""		)
        self.assertIsInstance(lowerCamelCase__,		lowerCamelCase__		)

        A			:	List[str]										=       list(batch.input_ids.numpy()[0]		)

        self.assertListEqual(lowerCamelCase__,		lowerCamelCase__		)

        self.assertEqual((2, 39),		batch.input_ids.shape		)
        self.assertEqual((2, 39),		batch.attention_mask.shape		)






    @require_torch
    def       _lowerCAmelCase (	self		):
        A			:	List[str]										=       self.canine_tokenizer
        A			:	str										=       ['''Once there was a man.''', '''He wrote a test in HuggingFace Tranformers.''']
        A			:	List[Any]										=       tokenizer(lowerCamelCase__,		padding=lowerCamelCase__,		return_tensors="""pt"""		)
        # check if input_ids, attention_mask and token_type_ids are returned
        self.assertIn("""input_ids""",		lowerCamelCase__		)
        self.assertIn("""attention_mask""",		lowerCamelCase__		)
        self.assertIn("""token_type_ids""",		lowerCamelCase__		)






    @require_torch
    def       _lowerCAmelCase (	self		):
        A			:	str										=       self.canine_tokenizer
        A			:	Tuple										=       [
            '''What\'s the weater?''',
            '''It\'s about 25 degrees.''',
        ]
        A			:	Any										=       tokenizer(
            text_target=lowerCamelCase__,		max_length=32,		padding="""max_length""",		truncation=lowerCamelCase__,		return_tensors="""pt"""		)
        self.assertEqual(32,		targets["""input_ids"""].shape[1]		)






    def       _lowerCAmelCase (	self		):
        # safety check on max_len default value so we are sure the test works
        A			:	Tuple										=       self.get_tokenizers()
        for tokenizer in tokenizers:
            with self.subTest(f'''{tokenizer.__class__.__name__}'''		):
                self.assertNotEqual(tokenizer.model_max_length,		42		)

        # Now let's start the test
        A			:	Tuple										=       self.get_tokenizers()
        for tokenizer in tokenizers:
            with self.subTest(f'''{tokenizer.__class__.__name__}'''		):
                # Isolate this from the other tests because we save additional tokens/etc
                A			:	Optional[Any]										=       tempfile.mkdtemp()

                A			:	Optional[int]										=       ''' He is very happy, UNwant\u00E9d,running'''
                A			:	Union[str, Any]										=       tokenizer.encode(lowerCamelCase__,		add_special_tokens=lowerCamelCase__		)
                tokenizer.save_pretrained(lowerCamelCase__		)

                A			:	str										=       tokenizer.__class__.from_pretrained(lowerCamelCase__		)
                A			:	str										=       after_tokenizer.encode(lowerCamelCase__,		add_special_tokens=lowerCamelCase__		)
                self.assertListEqual(lowerCamelCase__,		lowerCamelCase__		)

                shutil.rmtree(lowerCamelCase__		)

        A			:	List[Any]										=       self.get_tokenizers(model_max_length=42		)
        for tokenizer in tokenizers:
            with self.subTest(f'''{tokenizer.__class__.__name__}'''		):
                # Isolate this from the other tests because we save additional tokens/etc
                A			:	Optional[Any]										=       tempfile.mkdtemp()

                A			:	Union[str, Any]										=       ''' He is very happy, UNwant\u00E9d,running'''

                A			:	Dict										=       tokenizer.additional_special_tokens

                # We can add a new special token for Canine as follows:
                A			:	Tuple										=       chr(0xe_0_0_7		)
                additional_special_tokens.append(lowerCamelCase__		)
                tokenizer.add_special_tokens({"""additional_special_tokens""": additional_special_tokens}		)
                A			:	Union[str, Any]										=       tokenizer.encode(lowerCamelCase__,		add_special_tokens=lowerCamelCase__		)
                tokenizer.save_pretrained(lowerCamelCase__		)

                A			:	str										=       tokenizer.__class__.from_pretrained(lowerCamelCase__		)
                A			:	Dict										=       after_tokenizer.encode(lowerCamelCase__,		add_special_tokens=lowerCamelCase__		)
                self.assertListEqual(lowerCamelCase__,		lowerCamelCase__		)
                self.assertIn(lowerCamelCase__,		after_tokenizer.additional_special_tokens		)
                self.assertEqual(after_tokenizer.model_max_length,		42		)

                A			:	List[Any]										=       tokenizer.__class__.from_pretrained(lowerCamelCase__,		model_max_length=43		)
                self.assertEqual(tokenizer.model_max_length,		43		)

                shutil.rmtree(lowerCamelCase__		)






    def       _lowerCAmelCase (	self		):
        A			:	str										=       self.get_tokenizers(do_lower_case=lowerCamelCase__		)
        for tokenizer in tokenizers:
            with self.subTest(f'''{tokenizer.__class__.__name__}'''		):
                A			:	Optional[int]										=       self.get_clean_sequence(lowerCamelCase__		)

                # a special token for Canine can be defined as follows:
                A			:	List[str]										=       0xe_0_0_5
                A			:	Dict										=       chr(lowerCamelCase__		)

                tokenizer.add_special_tokens({"""cls_token""": special_token}		)
                A			:	int										=       tokenizer.encode(lowerCamelCase__,		add_special_tokens=lowerCamelCase__		)
                self.assertEqual(len(lowerCamelCase__		),		1		)

                A			:	Optional[Any]										=       tokenizer.decode(ids + encoded_special_token,		clean_up_tokenization_spaces=lowerCamelCase__		)
                A			:	Tuple										=       tokenizer.encode(lowerCamelCase__,		add_special_tokens=lowerCamelCase__		)

                A			:	Optional[int]										=       tokenizer.encode(lowerCamelCase__,		add_special_tokens=lowerCamelCase__		)
                A			:	List[Any]										=       tokenizer.encode(lowerCamelCase__,		add_special_tokens=lowerCamelCase__		)
                self.assertEqual(lowerCamelCase__,		input_encoded + special_token_id		)

                A			:	Dict										=       tokenizer.decode(lowerCamelCase__,		skip_special_tokens=lowerCamelCase__		)
                self.assertTrue(special_token not in decoded		)






    def       _lowerCAmelCase (	self		):
        A			:	Any										=       self.get_tokenizers(do_lower_case=lowerCamelCase__		)
        for tokenizer in tokenizers:
            with self.subTest(f'''{tokenizer.__class__.__name__}'''		):
                A			:	List[Any]										=       chr(0xe_0_0_5		)
                A			:	str										=       chr(0xe_0_0_6		)

                # `add_tokens` method stores special tokens only in `tokenizer.unique_no_split_tokens`. (in tokenization_utils.py)
                tokenizer.add_tokens([SPECIAL_TOKEN_1],		special_tokens=lowerCamelCase__		)
                # `add_special_tokens` method stores special tokens in `tokenizer.additional_special_tokens`,
                # which also occur in `tokenizer.all_special_tokens`. (in tokenization_utils_base.py)
                tokenizer.add_special_tokens({"""additional_special_tokens""": [SPECIAL_TOKEN_2]}		)

                A			:	List[str]										=       tokenizer.tokenize(lowerCamelCase__		)
                A			:	List[Any]										=       tokenizer.tokenize(lowerCamelCase__		)

                self.assertEqual(len(lowerCamelCase__		),		1		)
                self.assertEqual(len(lowerCamelCase__		),		1		)
                self.assertEqual(token_a[0],		lowerCamelCase__		)
                self.assertEqual(token_a[0],		lowerCamelCase__		)






    @require_tokenizers
    def       _lowerCAmelCase (	self		):
        A			:	Tuple										=       self.get_tokenizers(do_lower_case=lowerCamelCase__		)
        for tokenizer in tokenizers:
            with self.subTest(f'''{tokenizer.__class__.__name__}'''		):
                # a special token for Canine can be defined as follows:
                A			:	str										=       0xe_0_0_6
                A			:	Optional[int]										=       chr(lowerCamelCase__		)

                A			:	Dict										=       AddedToken(lowerCamelCase__,		lstrip=lowerCamelCase__		)
                tokenizer.add_special_tokens({"""additional_special_tokens""": [new_token]}		)

                with tempfile.TemporaryDirectory() as tmp_dir_name:
                    tokenizer.save_pretrained(lowerCamelCase__		)
                    tokenizer.from_pretrained(lowerCamelCase__		)






    def       _lowerCAmelCase (	self		):
        A			:	int										=       []
        if self.test_slow_tokenizer:
            tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())		)

        if self.test_rust_tokenizer:
            tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())		)

        for tokenizer_class, tokenizer_utils in tokenizer_list:
            with tempfile.TemporaryDirectory() as tmp_dir:
                tokenizer_utils.save_pretrained(lowerCamelCase__		)

                with open(os.path.join(lowerCamelCase__,		"""special_tokens_map.json"""		),		encoding="""utf-8"""		) as json_file:
                    A			:	str										=       json.load(lowerCamelCase__		)

                with open(os.path.join(lowerCamelCase__,		"""tokenizer_config.json"""		),		encoding="""utf-8"""		) as json_file:
                    A			:	Dict										=       json.load(lowerCamelCase__		)

                # a special token for Canine can be defined as follows:
                A			:	Any										=       0xe_0_0_6
                A			:	int										=       chr(lowerCamelCase__		)

                A			:	Any										=       [new_token_a]
                A			:	Optional[int]										=       [new_token_a]

                with open(os.path.join(lowerCamelCase__,		"""special_tokens_map.json"""		),		"""w""",		encoding="""utf-8"""		) as outfile:
                    json.dump(lowerCamelCase__,		lowerCamelCase__		)
                with open(os.path.join(lowerCamelCase__,		"""tokenizer_config.json"""		),		"""w""",		encoding="""utf-8"""		) as outfile:
                    json.dump(lowerCamelCase__,		lowerCamelCase__		)

                # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
                # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
                # "special_tokens_map.json" files
                A			:	str										=       tokenizer_class.from_pretrained(lowerCamelCase__,		extra_ids=0		)
                self.assertIn(lowerCamelCase__,		tokenizer_without_change_in_init.additional_special_tokens		)
                # self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
                self.assertEqual(
                    [new_token_a],		tokenizer_without_change_in_init.convert_ids_to_tokens(
                        tokenizer_without_change_in_init.convert_tokens_to_ids([new_token_a]		)		),		)

                A			:	List[str]										=       0xe_0_0_7
                A			:	str										=       chr(lowerCamelCase__		)
                # Now we test that we can change the value of additional_special_tokens in the from_pretrained
                A			:	Union[str, Any]										=       [AddedToken(lowerCamelCase__,		lstrip=lowerCamelCase__		)]
                A			:	Union[str, Any]										=       tokenizer_class.from_pretrained(
                    lowerCamelCase__,		additional_special_tokens=lowerCamelCase__,		extra_ids=0		)

                self.assertIn(lowerCamelCase__,		tokenizer.additional_special_tokens		)
                # self.assertIn(new_token_2,tokenizer.get_vocab()) # ByT5Tokenization no vocab
                self.assertEqual(
                    [new_token_a],		tokenizer.convert_ids_to_tokens(tokenizer.convert_tokens_to_ids([new_token_a]		)		)		)






    @require_tokenizers
    def       _lowerCAmelCase (	self		):
        A			:	int										=       self.get_tokenizers(do_lower_case=lowerCamelCase__		)
        for tokenizer in tokenizers:
            with self.subTest(f'''{tokenizer.__class__.__name__}'''		):
                A			:	List[str]										=       '''hello world'''
                if self.space_between_special_tokens:
                    A			:	List[str]										=       '''[CLS] hello world [SEP]'''
                else:
                    A			:	str										=       input
                A			:	str										=       tokenizer.encode(lowerCamelCase__,		add_special_tokens=lowerCamelCase__		)
                A			:	Optional[int]										=       tokenizer.decode(lowerCamelCase__,		spaces_between_special_tokens=self.space_between_special_tokens		)
                self.assertIn(lowerCamelCase__,		[output, output.lower()]		)






    def       _lowerCAmelCase (	self		):
        A			:	Optional[int]										=       self.get_tokenizers()
        for tokenizer in tokenizers:
            with self.subTest(f'''{tokenizer.__class__.__name__}'''		):
                A			:	Optional[int]										=       [
                    '''bos_token''',
                    '''eos_token''',
                    '''unk_token''',
                    '''sep_token''',
                    '''pad_token''',
                    '''cls_token''',
                    '''mask_token''',
                ]

        A			:	List[Any]										=       '''a'''
        A			:	Dict										=       ord(lowerCamelCase__		)

        for attr in attributes_list:
            setattr(lowerCamelCase__,		attr + """_id""",		lowerCamelCase__		)
            self.assertEqual(getattr(lowerCamelCase__,		lowerCamelCase__		),		lowerCamelCase__		)
            self.assertEqual(getattr(lowerCamelCase__,		attr + """_id"""		),		lowerCamelCase__		)

            setattr(lowerCamelCase__,		attr + """_id""",		lowerCamelCase__		)
            self.assertEqual(getattr(lowerCamelCase__,		lowerCamelCase__		),		lowerCamelCase__		)
            self.assertEqual(getattr(lowerCamelCase__,		attr + """_id"""		),		lowerCamelCase__		)

        setattr(lowerCamelCase__,		"""additional_special_tokens_ids""",		[]		)
        self.assertListEqual(getattr(lowerCamelCase__,		"""additional_special_tokens"""		),		[]		)
        self.assertListEqual(getattr(lowerCamelCase__,		"""additional_special_tokens_ids"""		),		[]		)

        A			:	int										=       0xe_0_0_6
        A			:	Any										=       chr(lowerCamelCase__		)
        setattr(lowerCamelCase__,		"""additional_special_tokens_ids""",		[additional_special_token_id]		)
        self.assertListEqual(getattr(lowerCamelCase__,		"""additional_special_tokens"""		),		[additional_special_token]		)
        self.assertListEqual(getattr(lowerCamelCase__,		"""additional_special_tokens_ids"""		),		[additional_special_token_id]		)






    def       _lowerCAmelCase (	self		):
        pass






    def       _lowerCAmelCase (	self		):
        pass






    def       _lowerCAmelCase (	self		):
        pass






    def       _lowerCAmelCase (	self		):
        pass






    def       _lowerCAmelCase (	self		):
        pass






    def       _lowerCAmelCase (	self		):
        pass






    def       _lowerCAmelCase (	self		):
        pass






    def       _lowerCAmelCase (	self		):
        pass







from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


SCREAMING_SNAKE_CASE_:Any					=						logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_:Any					=						{
    """xlm-mlm-en-2048""": """https://huggingface.co/xlm-mlm-en-2048/resolve/main/config.json""",
    """xlm-mlm-ende-1024""": """https://huggingface.co/xlm-mlm-ende-1024/resolve/main/config.json""",
    """xlm-mlm-enfr-1024""": """https://huggingface.co/xlm-mlm-enfr-1024/resolve/main/config.json""",
    """xlm-mlm-enro-1024""": """https://huggingface.co/xlm-mlm-enro-1024/resolve/main/config.json""",
    """xlm-mlm-tlm-xnli15-1024""": """https://huggingface.co/xlm-mlm-tlm-xnli15-1024/resolve/main/config.json""",
    """xlm-mlm-xnli15-1024""": """https://huggingface.co/xlm-mlm-xnli15-1024/resolve/main/config.json""",
    """xlm-clm-enfr-1024""": """https://huggingface.co/xlm-clm-enfr-1024/resolve/main/config.json""",
    """xlm-clm-ende-1024""": """https://huggingface.co/xlm-clm-ende-1024/resolve/main/config.json""",
    """xlm-mlm-17-1280""": """https://huggingface.co/xlm-mlm-17-1280/resolve/main/config.json""",
    """xlm-mlm-100-1280""": """https://huggingface.co/xlm-mlm-100-1280/resolve/main/config.json""",
}






class SCREAMING_SNAKE_CASE__					(		SCREAMING_SNAKE_CASE__   ):




    '''simple docstring'''






    __lowerCamelCase      :			List[str] =				"xlm"
    __lowerCamelCase      :			Tuple =				{
        "hidden_size": "emb_dim",
        "num_attention_heads": "n_heads",
        "num_hidden_layers": "n_layers",
        "n_words": "vocab_size",  # For backward compatibility
    }






    def __init__(	self,		lowerCamelCase__=3_0145,		lowerCamelCase__=2048,		lowerCamelCase__=12,		lowerCamelCase__=16,		lowerCamelCase__=0.1,		lowerCamelCase__=0.1,		lowerCamelCase__=True,		lowerCamelCase__=False,		lowerCamelCase__=False,		lowerCamelCase__=False,		lowerCamelCase__=1,		lowerCamelCase__=True,		lowerCamelCase__=512,		lowerCamelCase__=2048**-0.5,		lowerCamelCase__=1e-12,		lowerCamelCase__=0.02,		lowerCamelCase__=0,		lowerCamelCase__=1,		lowerCamelCase__=2,		lowerCamelCase__=3,		lowerCamelCase__=5,		lowerCamelCase__=True,		lowerCamelCase__="first",		lowerCamelCase__=True,		lowerCamelCase__=None,		lowerCamelCase__=True,		lowerCamelCase__=0.1,		lowerCamelCase__=5,		lowerCamelCase__=5,		lowerCamelCase__=0,		lowerCamelCase__=0,		lowerCamelCase__=2,		lowerCamelCase__=0,		**lowerCamelCase__,		):
        A			:	Dict										=       vocab_size
        A			:	int										=       emb_dim
        A			:	str										=       n_layers
        A			:	Union[str, Any]										=       n_heads
        A			:	Optional[int]										=       dropout
        A			:	Union[str, Any]										=       attention_dropout
        A			:	Optional[Any]										=       gelu_activation
        A			:	Dict										=       sinusoidal_embeddings
        A			:	int										=       causal
        A			:	Optional[Any]										=       asm
        A			:	Any										=       n_langs
        A			:	List[str]										=       use_lang_emb
        A			:	Union[str, Any]										=       layer_norm_eps
        A			:	str										=       bos_index
        A			:	int										=       eos_index
        A			:	Tuple										=       pad_index
        A			:	str										=       unk_index
        A			:	Optional[Any]										=       mask_index
        A			:	Union[str, Any]										=       is_encoder
        A			:	Tuple										=       max_position_embeddings
        A			:	List[str]										=       embed_init_std
        A			:	Tuple										=       init_std
        A			:	Tuple										=       summary_type
        A			:	int										=       summary_use_proj
        A			:	List[Any]										=       summary_activation
        A			:	Optional[Any]										=       summary_proj_to_labels
        A			:	Optional[Any]										=       summary_first_dropout
        A			:	Optional[int]										=       start_n_top
        A			:	Optional[Any]										=       end_n_top
        A			:	List[str]										=       mask_token_id
        A			:	Tuple										=       lang_id

        if "n_words" in kwargs:
            A			:	List[str]										=       kwargs["""n_words"""]

        super().__init__(pad_token_id=lowerCamelCase__,		bos_token_id=lowerCamelCase__,		**lowerCamelCase__		)






class SCREAMING_SNAKE_CASE__					(		SCREAMING_SNAKE_CASE__   ):




    '''simple docstring'''






    @property
    def       _lowerCAmelCase (	self		):
        if self.task == "multiple-choice":
            A			:	Any										=       {0: """batch""", 1: """choice""", 2: """sequence"""}
        else:
            A			:	Union[str, Any]										=       {0: """batch""", 1: """sequence"""}
        return OrderedDict(
            [
                ("""input_ids""", dynamic_axis),
                ("""attention_mask""", dynamic_axis),
                ("""token_type_ids""", dynamic_axis),
            ]		)


