





import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / """utils"""))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
     from test_module.custom_tokenization_fast import CustomTokenizerFast







class     __lowerCAmelCase	(   unittest.TestCase):







      def        _lowercase  (   self							)       ->						List[Any]:



        '''simple docstring'''




        a__     : Dict  =0







      @slow
      def        _lowercase  (   self							)       ->						Optional[int]:



        '''simple docstring'''




        for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
          a__     : List[str]  =AutoTokenizer.from_pretrained(lowerCAmelCase__							)
          self.assertIsNotNone(lowerCAmelCase__							)
          self.assertIsInstance(lowerCAmelCase__		,				(BertTokenizer, BertTokenizerFast)							)
          self.assertGreater(len(lowerCAmelCase__							)		,				0							)

        for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
          a__     : str  =AutoTokenizer.from_pretrained(lowerCAmelCase__							)
          self.assertIsNotNone(lowerCAmelCase__							)
          self.assertIsInstance(lowerCAmelCase__		,				(GPTaTokenizer, GPTaTokenizerFast)							)
          self.assertGreater(len(lowerCAmelCase__							)		,				0							)







      def        _lowercase  (   self							)       ->						List[str]:



        '''simple docstring'''




        a__     : List[Any]  =AutoTokenizer.from_pretrained(lowerCAmelCase__							)
        self.assertIsInstance(lowerCAmelCase__		,				(BertTokenizer, BertTokenizerFast)							)
        self.assertEqual(tokenizer.vocab_size		,				1_2							)







      def        _lowercase  (   self							)       ->						Dict:



        '''simple docstring'''




        a__     : Optional[Any]  =AutoTokenizer.from_pretrained(lowerCAmelCase__							)
        self.assertIsInstance(lowerCAmelCase__		,				(RobertaTokenizer, RobertaTokenizerFast)							)
        self.assertEqual(tokenizer.vocab_size		,				2_0							)







      def        _lowercase  (   self							)       ->						str:



        '''simple docstring'''




        a__     : int  =AutoConfig.from_pretrained(lowerCAmelCase__							)
        self.assertIsInstance(lowerCAmelCase__		,				lowerCAmelCase__							)
        # Check that tokenizer_type ≠ model_type
        a__     : Union[str, Any]  =AutoTokenizer.from_pretrained(lowerCAmelCase__		,				config=lowerCAmelCase__							)
        self.assertIsInstance(lowerCAmelCase__		,				(BertTokenizer, BertTokenizerFast)							)
        self.assertEqual(tokenizer.vocab_size		,				1_2							)







      def        _lowercase  (   self							)       ->						Tuple:



        '''simple docstring'''




        with tempfile.TemporaryDirectory() as tmp_dir:
          shutil.copy("./tests/fixtures/vocab.txt"		,				os.path.join(lowerCAmelCase__		,				"vocab.txt"							)							)

          a__     : Any  =AutoTokenizer.from_pretrained(lowerCAmelCase__		,				tokenizer_type="bert"		,				use_fast=lowerCAmelCase__							)
          self.assertIsInstance(lowerCAmelCase__		,				lowerCAmelCase__							)

        with tempfile.TemporaryDirectory() as tmp_dir:
          shutil.copy("./tests/fixtures/vocab.json"		,				os.path.join(lowerCAmelCase__		,				"vocab.json"							)							)
          shutil.copy("./tests/fixtures/merges.txt"		,				os.path.join(lowerCAmelCase__		,				"merges.txt"							)							)

          a__     : Dict  =AutoTokenizer.from_pretrained(lowerCAmelCase__		,				tokenizer_type="gpt2"		,				use_fast=lowerCAmelCase__							)
          self.assertIsInstance(lowerCAmelCase__		,				lowerCAmelCase__							)







      @require_tokenizers
      def        _lowercase  (   self							)       ->						int:



        '''simple docstring'''




        with tempfile.TemporaryDirectory() as tmp_dir:
          shutil.copy("./tests/fixtures/vocab.txt"		,				os.path.join(lowerCAmelCase__		,				"vocab.txt"							)							)

          a__     : List[str]  =AutoTokenizer.from_pretrained(lowerCAmelCase__		,				tokenizer_type="bert"							)
          self.assertIsInstance(lowerCAmelCase__		,				lowerCAmelCase__							)

        with tempfile.TemporaryDirectory() as tmp_dir:
          shutil.copy("./tests/fixtures/vocab.json"		,				os.path.join(lowerCAmelCase__		,				"vocab.json"							)							)
          shutil.copy("./tests/fixtures/merges.txt"		,				os.path.join(lowerCAmelCase__		,				"merges.txt"							)							)

          a__     : Any  =AutoTokenizer.from_pretrained(lowerCAmelCase__		,				tokenizer_type="gpt2"							)
          self.assertIsInstance(lowerCAmelCase__		,				lowerCAmelCase__							)







      def        _lowercase  (   self							)       ->						Dict:



        '''simple docstring'''




        with pytest.raises(lowerCAmelCase__							):
          AutoTokenizer.from_pretrained("./"		,				tokenizer_type="xxx"							)







      @require_tokenizers
      def        _lowercase  (   self							)       ->						List[str]:



        '''simple docstring'''




        for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
          a__     : List[Any]  =tokenizer_class.from_pretrained("wietsedv/bert-base-dutch-cased"							)
          self.assertIsInstance(lowerCAmelCase__		,				(BertTokenizer, BertTokenizerFast)							)

          if isinstance(lowerCAmelCase__		,				lowerCAmelCase__							):
            self.assertEqual(tokenizer.basic_tokenizer.do_lower_case		,				lowerCAmelCase__							)
          else:
            self.assertEqual(tokenizer.do_lower_case		,				lowerCAmelCase__							)

          self.assertEqual(tokenizer.model_max_length		,				5_1_2							)







      @require_tokenizers
      def        _lowercase  (   self							)       ->						Union[str, Any]:



        '''simple docstring'''




        for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
          with self.assertRaisesRegex(
              lowerCAmelCase__		,				"julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier"		,				):
            a__     : str  =tokenizer_class.from_pretrained("julien-c/herlolip-not-exists"							)







      def        _lowercase  (   self							)       ->						Optional[int]:



        '''simple docstring'''




        a__     : Optional[int]  =TOKENIZER_MAPPING.values()
        a__     : Any  =[]

        for slow_tok, fast_tok in tokenizers:
          if slow_tok is not None:
            tokenizer_names.append(slow_tok.__name__							)

          if fast_tok is not None:
            tokenizer_names.append(fast_tok.__name__							)

        for tokenizer_name in tokenizer_names:
          # must find the right class
          tokenizer_class_from_name(lowerCAmelCase__							)







      @require_tokenizers
      def        _lowercase  (   self							)       ->						int:



        '''simple docstring'''




        self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased"		,				use_fast=lowerCAmelCase__							)		,				lowerCAmelCase__							)
        self.assertIsInstance(AutoTokenizer.from_pretrained("bert-base-cased"							)		,				lowerCAmelCase__							)







      @require_tokenizers
      def        _lowercase  (   self							)       ->						Any:



        '''simple docstring'''




        a__     : Dict  =AutoTokenizer.from_pretrained("distilbert-base-uncased"		,				do_lower_case=lowerCAmelCase__							)
        a__     : Dict  ="Hello, world. How are you?"
        a__     : List[str]  =tokenizer.tokenize(lowerCAmelCase__							)
        self.assertEqual("[UNK]"		,				tokens[0]							)

        a__     : str  =AutoTokenizer.from_pretrained("microsoft/mpnet-base"		,				do_lower_case=lowerCAmelCase__							)
        a__     : Optional[int]  =tokenizer.tokenize(lowerCAmelCase__							)
        self.assertEqual("[UNK]"		,				tokens[0]							)







      @require_tokenizers
      def        _lowercase  (   self							)       ->						Tuple:



        '''simple docstring'''




        a__     : Tuple  =AutoTokenizer.from_pretrained("robot-test/dummy-tokenizer-fast-with-model-config"							)
        self.assertEqual(type(lowerCAmelCase__							)		,				lowerCAmelCase__							)
        self.assertEqual(tokenizer.model_max_length		,				5_1_2							)
        self.assertEqual(tokenizer.vocab_size		,				3_0_0_0_0							)
        self.assertEqual(tokenizer.unk_token		,				"[UNK]"							)
        self.assertEqual(tokenizer.padding_side		,				"right"							)
        self.assertEqual(tokenizer.truncation_side		,				"right"							)







      def        _lowercase  (   self							)       ->						Optional[Any]:



        '''simple docstring'''




        a__     : List[str]  =AutoTokenizer.from_pretrained(lowerCAmelCase__							)
        self.assertIsInstance(lowerCAmelCase__		,				(BertTokenizer, BertTokenizerFast)							)
        with tempfile.TemporaryDirectory() as tmp_dir:
          tokenizer.save_pretrained(lowerCAmelCase__							)
          a__     : Any  =AutoTokenizer.from_pretrained(lowerCAmelCase__							)

        self.assertIsInstance(lowerCAmelCase__		,				tokenizer.__class__							)
        self.assertEqual(tokenizera.vocab_size		,				1_2							)







      def        _lowercase  (   self							)       ->						List[str]:



        '''simple docstring'''




        a__     : List[str]  =AutoTokenizer.from_pretrained("ctrl"							)
        # There is no fast CTRL so this always gives us a slow tokenizer.
        self.assertIsInstance(lowerCAmelCase__		,				lowerCAmelCase__							)







      def        _lowercase  (   self							)       ->						str:



        '''simple docstring'''




        a__     : Tuple  =get_tokenizer_config("bert-base-cased"							)
        a__     : Tuple  =config.pop("_commit_hash"		,				lowerCAmelCase__							)
        # If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
        self.assertEqual(lowerCAmelCase__		,				{"do_lower_case": False}							)

        # This model does not have a tokenizer_config so we get back an empty dict.
        a__     : Optional[Any]  =get_tokenizer_config(lowerCAmelCase__							)
        self.assertDictEqual(lowerCAmelCase__		,				{}							)

        # A tokenizer saved with `save_pretrained` always creates a tokenizer config.
        a__     : int  =AutoTokenizer.from_pretrained(lowerCAmelCase__							)
        with tempfile.TemporaryDirectory() as tmp_dir:
          tokenizer.save_pretrained(lowerCAmelCase__							)
          a__     : Tuple  =get_tokenizer_config(lowerCAmelCase__							)

        # Check the class of the tokenizer was properly saved (note that it always saves the slow class).
        self.assertEqual(config["tokenizer_class"]		,				"BertTokenizer"							)







      def        _lowercase  (   self							)       ->						Dict:



        '''simple docstring'''




        try:
          AutoConfig.register("custom"		,				lowerCAmelCase__							)

          AutoTokenizer.register(lowerCAmelCase__		,				slow_tokenizer_class=lowerCAmelCase__							)
          # Trying to register something existing in the Transformers library will raise an error
          with self.assertRaises(lowerCAmelCase__							):
            AutoTokenizer.register(lowerCAmelCase__		,				slow_tokenizer_class=lowerCAmelCase__							)

          a__     : Union[str, Any]  =CustomTokenizer.from_pretrained(lowerCAmelCase__							)
          with tempfile.TemporaryDirectory() as tmp_dir:
            tokenizer.save_pretrained(lowerCAmelCase__							)

            a__     : List[str]  =AutoTokenizer.from_pretrained(lowerCAmelCase__							)
            self.assertIsInstance(lowerCAmelCase__		,				lowerCAmelCase__							)

        finally:
          if "custom" in CONFIG_MAPPING._extra_content:
            del CONFIG_MAPPING._extra_content["custom"]
          if CustomConfig in TOKENIZER_MAPPING._extra_content:
            del TOKENIZER_MAPPING._extra_content[CustomConfig]







      @require_tokenizers
      def        _lowercase  (   self							)       ->						List[Any]:



        '''simple docstring'''




        try:
          AutoConfig.register("custom"		,				lowerCAmelCase__							)

          # Can register in two steps
          AutoTokenizer.register(lowerCAmelCase__		,				slow_tokenizer_class=lowerCAmelCase__							)
          self.assertEqual(TOKENIZER_MAPPING[CustomConfig]		,				(CustomTokenizer, None)							)
          AutoTokenizer.register(lowerCAmelCase__		,				fast_tokenizer_class=lowerCAmelCase__							)
          self.assertEqual(TOKENIZER_MAPPING[CustomConfig]		,				(CustomTokenizer, CustomTokenizerFast)							)

          del TOKENIZER_MAPPING._extra_content[CustomConfig]
          # Can register in one step
          AutoTokenizer.register(
              lowerCAmelCase__		,				slow_tokenizer_class=lowerCAmelCase__		,				fast_tokenizer_class=lowerCAmelCase__							)
          self.assertEqual(TOKENIZER_MAPPING[CustomConfig]		,				(CustomTokenizer, CustomTokenizerFast)							)

          # Trying to register something existing in the Transformers library will raise an error
          with self.assertRaises(lowerCAmelCase__							):
            AutoTokenizer.register(lowerCAmelCase__		,				fast_tokenizer_class=lowerCAmelCase__							)

          # We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
          # and that model does not have a tokenizer.json
          with tempfile.TemporaryDirectory() as tmp_dir:
            a__     : Optional[int]  =BertTokenizerFast.from_pretrained(lowerCAmelCase__							)
            bert_tokenizer.save_pretrained(lowerCAmelCase__							)
            a__     : Dict  =CustomTokenizerFast.from_pretrained(lowerCAmelCase__							)

          with tempfile.TemporaryDirectory() as tmp_dir:
            tokenizer.save_pretrained(lowerCAmelCase__							)

            a__     : int  =AutoTokenizer.from_pretrained(lowerCAmelCase__							)
            self.assertIsInstance(lowerCAmelCase__		,				lowerCAmelCase__							)

            a__     : Union[str, Any]  =AutoTokenizer.from_pretrained(lowerCAmelCase__		,				use_fast=lowerCAmelCase__							)
            self.assertIsInstance(lowerCAmelCase__		,				lowerCAmelCase__							)

        finally:
          if "custom" in CONFIG_MAPPING._extra_content:
            del CONFIG_MAPPING._extra_content["custom"]
          if CustomConfig in TOKENIZER_MAPPING._extra_content:
            del TOKENIZER_MAPPING._extra_content[CustomConfig]







      def        _lowercase  (   self							)       ->						Optional[Any]:



        '''simple docstring'''




        with self.assertRaises(lowerCAmelCase__							):
          a__     : Optional[Any]  =AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"							)
        # If remote code is disabled, we can't load this config.
        with self.assertRaises(lowerCAmelCase__							):
          a__     : Any  =AutoTokenizer.from_pretrained(
              "hf-internal-testing/test_dynamic_tokenizer"		,				trust_remote_code=lowerCAmelCase__							)

        a__     : str  =AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"		,				trust_remote_code=lowerCAmelCase__							)
        self.assertTrue(tokenizer.special_attribute_present							)
        # Test tokenizer can be reloaded.
        with tempfile.TemporaryDirectory() as tmp_dir:
          tokenizer.save_pretrained(lowerCAmelCase__							)
          a__     : int  =AutoTokenizer.from_pretrained(lowerCAmelCase__		,				trust_remote_code=lowerCAmelCase__							)
        self.assertTrue(reloaded_tokenizer.special_attribute_present							)

        if is_tokenizers_available():
          self.assertEqual(tokenizer.__class__.__name__		,				"NewTokenizerFast"							)
          self.assertEqual(reloaded_tokenizer.__class__.__name__		,				"NewTokenizerFast"							)

          # Test we can also load the slow version
          a__     : Dict  =AutoTokenizer.from_pretrained(
              "hf-internal-testing/test_dynamic_tokenizer"		,				trust_remote_code=lowerCAmelCase__		,				use_fast=lowerCAmelCase__							)
          self.assertTrue(tokenizer.special_attribute_present							)
          self.assertEqual(tokenizer.__class__.__name__		,				"NewTokenizer"							)
          # Test tokenizer can be reloaded.
          with tempfile.TemporaryDirectory() as tmp_dir:
            tokenizer.save_pretrained(lowerCAmelCase__							)
            a__     : Tuple  =AutoTokenizer.from_pretrained(lowerCAmelCase__		,				trust_remote_code=lowerCAmelCase__		,				use_fast=lowerCAmelCase__							)
          self.assertEqual(reloaded_tokenizer.__class__.__name__		,				"NewTokenizer"							)
          self.assertTrue(reloaded_tokenizer.special_attribute_present							)
        else:
          self.assertEqual(tokenizer.__class__.__name__		,				"NewTokenizer"							)
          self.assertEqual(reloaded_tokenizer.__class__.__name__		,				"NewTokenizer"							)







      @require_tokenizers
      def        _lowercase  (   self							)       ->						Tuple:



        '''simple docstring'''




        class     __lowerCAmelCase	(   UpperCamelCase__):
              _lowercase    : str            =      False







        class     __lowerCAmelCase	(   UpperCamelCase__):
              _lowercase    : int            =      NewTokenizer
              _lowercase    : Any            =      False

        try:
          AutoConfig.register("custom"		,				lowerCAmelCase__							)
          AutoTokenizer.register(lowerCAmelCase__		,				slow_tokenizer_class=lowerCAmelCase__							)
          AutoTokenizer.register(lowerCAmelCase__		,				fast_tokenizer_class=lowerCAmelCase__							)
          # If remote code is not set, the default is to use local
          a__     : str  =AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"							)
          self.assertEqual(tokenizer.__class__.__name__		,				"NewTokenizerFast"							)
          self.assertFalse(tokenizer.special_attribute_present							)
          a__     : Dict  =AutoTokenizer.from_pretrained("hf-internal-testing/test_dynamic_tokenizer"		,				use_fast=lowerCAmelCase__							)
          self.assertEqual(tokenizer.__class__.__name__		,				"NewTokenizer"							)
          self.assertFalse(tokenizer.special_attribute_present							)

          # If remote code is disabled, we load the local one.
          a__     : List[Any]  =AutoTokenizer.from_pretrained(
              "hf-internal-testing/test_dynamic_tokenizer"		,				trust_remote_code=lowerCAmelCase__							)
          self.assertEqual(tokenizer.__class__.__name__		,				"NewTokenizerFast"							)
          self.assertFalse(tokenizer.special_attribute_present							)
          a__     : Dict  =AutoTokenizer.from_pretrained(
              "hf-internal-testing/test_dynamic_tokenizer"		,				trust_remote_code=lowerCAmelCase__		,				use_fast=lowerCAmelCase__							)
          self.assertEqual(tokenizer.__class__.__name__		,				"NewTokenizer"							)
          self.assertFalse(tokenizer.special_attribute_present							)

          # If remote is enabled, we load from the Hub
          a__     : Optional[int]  =AutoTokenizer.from_pretrained(
              "hf-internal-testing/test_dynamic_tokenizer"		,				trust_remote_code=lowerCAmelCase__							)
          self.assertEqual(tokenizer.__class__.__name__		,				"NewTokenizerFast"							)
          self.assertTrue(tokenizer.special_attribute_present							)
          a__     : int  =AutoTokenizer.from_pretrained(
              "hf-internal-testing/test_dynamic_tokenizer"		,				trust_remote_code=lowerCAmelCase__		,				use_fast=lowerCAmelCase__							)
          self.assertEqual(tokenizer.__class__.__name__		,				"NewTokenizer"							)
          self.assertTrue(tokenizer.special_attribute_present							)

        finally:
          if "custom" in CONFIG_MAPPING._extra_content:
            del CONFIG_MAPPING._extra_content["custom"]
          if CustomConfig in TOKENIZER_MAPPING._extra_content:
            del TOKENIZER_MAPPING._extra_content[CustomConfig]







      def        _lowercase  (   self							)       ->						int:



        '''simple docstring'''




        a__     : Optional[Any]  =AutoTokenizer.from_pretrained(
            "hf-internal-testing/test_dynamic_tokenizer_legacy"		,				trust_remote_code=lowerCAmelCase__							)
        self.assertTrue(tokenizer.special_attribute_present							)
        if is_tokenizers_available():
          self.assertEqual(tokenizer.__class__.__name__		,				"NewTokenizerFast"							)

          # Test we can also load the slow version
          a__     : Optional[int]  =AutoTokenizer.from_pretrained(
              "hf-internal-testing/test_dynamic_tokenizer_legacy"		,				trust_remote_code=lowerCAmelCase__		,				use_fast=lowerCAmelCase__							)
          self.assertTrue(tokenizer.special_attribute_present							)
          self.assertEqual(tokenizer.__class__.__name__		,				"NewTokenizer"							)
        else:
          self.assertEqual(tokenizer.__class__.__name__		,				"NewTokenizer"							)







      def        _lowercase  (   self							)       ->						List[str]:



        '''simple docstring'''




        with self.assertRaisesRegex(
            lowerCAmelCase__		,				"bert-base is not a local folder and is not a valid model identifier"							):
          a__     : Optional[int]  =AutoTokenizer.from_pretrained("bert-base"							)







      def        _lowercase  (   self							)       ->						Dict:



        '''simple docstring'''




        with self.assertRaisesRegex(
            lowerCAmelCase__		,				r"aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"							):
          a__     : List[str]  =AutoTokenizer.from_pretrained(lowerCAmelCase__		,				revision="aaaaaa"							)







      def        _lowercase  (   self							)       ->						Optional[Any]:



        '''simple docstring'''




        a__     : Tuple  =AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"							)
        with RequestCounter() as counter:
          a__     : Union[str, Any]  =AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"							)
          self.assertEqual(counter.get_request_count		,				0							)
          self.assertEqual(counter.head_request_count		,				1							)
          self.assertEqual(counter.other_request_count		,				0							)













from maths.prime_check import is_prime
def 					_A	(		SCREAMING_SNAKE_CASE					:       int						):





  """simple docstring"""



  if not isinstance(SCREAMING_SNAKE_CASE	,	SCREAMING_SNAKE_CASE						):
    a__     : Dict  =f'''Input value of [number={number}] must be an integer'''
    raise TypeError(SCREAMING_SNAKE_CASE						)
  if is_prime(SCREAMING_SNAKE_CASE						) and is_prime(number + 2						):
    return number + 2
  else:
    return -1


if __name__ == "__main__":
     import doctest

     doctest.testmod()






