



import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / '''utils'''))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
 from test_module.custom_tokenization_fast import CustomTokenizerFast

class    _A	(     unittest.TestCase   ):



       def 		_lowerCamelCase      (		self							:					int):


           '''simple docstring'''

           __a							=    0



       @slow
       def 		_lowerCamelCase      (		self							:					Union[str, Any]):


           '''simple docstring'''

           for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
               __a							=    AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE)
               self.assertIsNotNone(__SCREAMING_SNAKE_CASE)
               self.assertIsInstance(__SCREAMING_SNAKE_CASE					,		(BertTokenizer, BertTokenizerFast))
               self.assertGreater(len(__SCREAMING_SNAKE_CASE)					,		0)

           for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
               __a							=    AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE)
               self.assertIsNotNone(__SCREAMING_SNAKE_CASE)
               self.assertIsInstance(__SCREAMING_SNAKE_CASE					,		(GPTaTokenizer, GPTaTokenizerFast))
               self.assertGreater(len(__SCREAMING_SNAKE_CASE)					,		0)



       def 		_lowerCamelCase      (		self							:					Any):


           '''simple docstring'''

           __a							=    AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE)
           self.assertIsInstance(__SCREAMING_SNAKE_CASE					,		(BertTokenizer, BertTokenizerFast))
           self.assertEqual(tokenizer.vocab_size					,		12)



       def 		_lowerCamelCase      (		self							:					List[Any]):


           '''simple docstring'''

           __a							=    AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE)
           self.assertIsInstance(__SCREAMING_SNAKE_CASE					,		(RobertaTokenizer, RobertaTokenizerFast))
           self.assertEqual(tokenizer.vocab_size					,		20)



       def 		_lowerCamelCase      (		self							:					Tuple):


           '''simple docstring'''

           __a							=    AutoConfig.from_pretrained(__SCREAMING_SNAKE_CASE)
           self.assertIsInstance(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE)
           # Check that tokenizer_type ≠ model_type
           __a							=    AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE					,		config=__SCREAMING_SNAKE_CASE)
           self.assertIsInstance(__SCREAMING_SNAKE_CASE					,		(BertTokenizer, BertTokenizerFast))
           self.assertEqual(tokenizer.vocab_size					,		12)



       def 		_lowerCamelCase      (		self							:					Any):


           '''simple docstring'''

           with tempfile.TemporaryDirectory() as tmp_dir:
               shutil.copy('''./tests/fixtures/vocab.txt'''					,		os.path.join(__SCREAMING_SNAKE_CASE					,		'''vocab.txt'''))

               __a							=    AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE					,		tokenizer_type='''bert'''					,		use_fast=__SCREAMING_SNAKE_CASE)
               self.assertIsInstance(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE)

           with tempfile.TemporaryDirectory() as tmp_dir:
               shutil.copy('''./tests/fixtures/vocab.json'''					,		os.path.join(__SCREAMING_SNAKE_CASE					,		'''vocab.json'''))
               shutil.copy('''./tests/fixtures/merges.txt'''					,		os.path.join(__SCREAMING_SNAKE_CASE					,		'''merges.txt'''))

               __a							=    AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE					,		tokenizer_type='''gpt2'''					,		use_fast=__SCREAMING_SNAKE_CASE)
               self.assertIsInstance(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE)



       @require_tokenizers
       def 		_lowerCamelCase      (		self							:					Tuple):


           '''simple docstring'''

           with tempfile.TemporaryDirectory() as tmp_dir:
               shutil.copy('''./tests/fixtures/vocab.txt'''					,		os.path.join(__SCREAMING_SNAKE_CASE					,		'''vocab.txt'''))

               __a							=    AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE					,		tokenizer_type='''bert''')
               self.assertIsInstance(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE)

           with tempfile.TemporaryDirectory() as tmp_dir:
               shutil.copy('''./tests/fixtures/vocab.json'''					,		os.path.join(__SCREAMING_SNAKE_CASE					,		'''vocab.json'''))
               shutil.copy('''./tests/fixtures/merges.txt'''					,		os.path.join(__SCREAMING_SNAKE_CASE					,		'''merges.txt'''))

               __a							=    AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE					,		tokenizer_type='''gpt2''')
               self.assertIsInstance(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE)



       def 		_lowerCamelCase      (		self							:					Optional[Any]):


           '''simple docstring'''

           with pytest.raises(__SCREAMING_SNAKE_CASE):
               AutoTokenizer.from_pretrained('''./'''					,		tokenizer_type='''xxx''')



       @require_tokenizers
       def 		_lowerCamelCase      (		self							:					Union[str, Any]):


           '''simple docstring'''

           for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
               __a							=    tokenizer_class.from_pretrained('''wietsedv/bert-base-dutch-cased''')
               self.assertIsInstance(__SCREAMING_SNAKE_CASE					,		(BertTokenizer, BertTokenizerFast))

               if isinstance(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE):
                   self.assertEqual(tokenizer.basic_tokenizer.do_lower_case					,		__SCREAMING_SNAKE_CASE)
               else:
                   self.assertEqual(tokenizer.do_lower_case					,		__SCREAMING_SNAKE_CASE)

               self.assertEqual(tokenizer.model_max_length					,		512)



       @require_tokenizers
       def 		_lowerCamelCase      (		self							:					Optional[Any]):


           '''simple docstring'''

           for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
               with self.assertRaisesRegex(
                   __SCREAMING_SNAKE_CASE					,		'''julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier'''					,		):
                   __a							=    tokenizer_class.from_pretrained('''julien-c/herlolip-not-exists''')



       def 		_lowerCamelCase      (		self							:					str):


           '''simple docstring'''

           __a							=    TOKENIZER_MAPPING.values()
           __a							=    []

           for slow_tok, fast_tok in tokenizers:
               if slow_tok is not None:
                   tokenizer_names.append(slow_tok.__name__)

               if fast_tok is not None:
                   tokenizer_names.append(fast_tok.__name__)

           for tokenizer_name in tokenizer_names:
               # must find the right class
               tokenizer_class_from_name(__SCREAMING_SNAKE_CASE)



       @require_tokenizers
       def 		_lowerCamelCase      (		self							:					List[str]):


           '''simple docstring'''

           self.assertIsInstance(AutoTokenizer.from_pretrained('''bert-base-cased'''					,		use_fast=__SCREAMING_SNAKE_CASE)					,		__SCREAMING_SNAKE_CASE)
           self.assertIsInstance(AutoTokenizer.from_pretrained('''bert-base-cased''')					,		__SCREAMING_SNAKE_CASE)



       @require_tokenizers
       def 		_lowerCamelCase      (		self							:					int):


           '''simple docstring'''

           __a							=    AutoTokenizer.from_pretrained('''distilbert-base-uncased'''					,		do_lower_case=__SCREAMING_SNAKE_CASE)
           __a							=    '''Hello, world. How are you?'''
           __a							=    tokenizer.tokenize(__SCREAMING_SNAKE_CASE)
           self.assertEqual('''[UNK]'''					,		tokens[0])

           __a							=    AutoTokenizer.from_pretrained('''microsoft/mpnet-base'''					,		do_lower_case=__SCREAMING_SNAKE_CASE)
           __a							=    tokenizer.tokenize(__SCREAMING_SNAKE_CASE)
           self.assertEqual('''[UNK]'''					,		tokens[0])



       @require_tokenizers
       def 		_lowerCamelCase      (		self							:					Union[str, Any]):


           '''simple docstring'''

           __a							=    AutoTokenizer.from_pretrained('''robot-test/dummy-tokenizer-fast-with-model-config''')
           self.assertEqual(type(__SCREAMING_SNAKE_CASE)					,		__SCREAMING_SNAKE_CASE)
           self.assertEqual(tokenizer.model_max_length					,		512)
           self.assertEqual(tokenizer.vocab_size					,		30_000)
           self.assertEqual(tokenizer.unk_token					,		'''[UNK]''')
           self.assertEqual(tokenizer.padding_side					,		'''right''')
           self.assertEqual(tokenizer.truncation_side					,		'''right''')



       def 		_lowerCamelCase      (		self							:					Optional[int]):


           '''simple docstring'''

           __a							=    AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE)
           self.assertIsInstance(__SCREAMING_SNAKE_CASE					,		(BertTokenizer, BertTokenizerFast))
           with tempfile.TemporaryDirectory() as tmp_dir:
               tokenizer.save_pretrained(__SCREAMING_SNAKE_CASE)
               __a							=    AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE)

           self.assertIsInstance(__SCREAMING_SNAKE_CASE					,		tokenizer.__class__)
           self.assertEqual(tokenizera.vocab_size					,		12)



       def 		_lowerCamelCase      (		self							:					Optional[Any]):


           '''simple docstring'''

           __a							=    AutoTokenizer.from_pretrained('''ctrl''')
           # There is no fast CTRL so this always gives us a slow tokenizer.
           self.assertIsInstance(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE)



       def 		_lowerCamelCase      (		self							:					Tuple):


           '''simple docstring'''

           __a							=    get_tokenizer_config('''bert-base-cased''')
           __a							=    config.pop('''_commit_hash'''					,		__SCREAMING_SNAKE_CASE)
           # If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
           self.assertEqual(__SCREAMING_SNAKE_CASE					,		{'''do_lower_case''': False})

           # This model does not have a tokenizer_config so we get back an empty dict.
           __a							=    get_tokenizer_config(__SCREAMING_SNAKE_CASE)
           self.assertDictEqual(__SCREAMING_SNAKE_CASE					,		{})

           # A tokenizer saved with `save_pretrained` always creates a tokenizer config.
           __a							=    AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE)
           with tempfile.TemporaryDirectory() as tmp_dir:
               tokenizer.save_pretrained(__SCREAMING_SNAKE_CASE)
               __a							=    get_tokenizer_config(__SCREAMING_SNAKE_CASE)

           # Check the class of the tokenizer was properly saved (note that it always saves the slow class).
           self.assertEqual(config['''tokenizer_class''']					,		'''BertTokenizer''')



       def 		_lowerCamelCase      (		self							:					List[Any]):


           '''simple docstring'''

           try:
               AutoConfig.register('''custom'''					,		__SCREAMING_SNAKE_CASE)

               AutoTokenizer.register(__SCREAMING_SNAKE_CASE					,		slow_tokenizer_class=__SCREAMING_SNAKE_CASE)
               # Trying to register something existing in the Transformers library will raise an error
               with self.assertRaises(__SCREAMING_SNAKE_CASE):
                   AutoTokenizer.register(__SCREAMING_SNAKE_CASE					,		slow_tokenizer_class=__SCREAMING_SNAKE_CASE)

               __a							=    CustomTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE)
               with tempfile.TemporaryDirectory() as tmp_dir:
                   tokenizer.save_pretrained(__SCREAMING_SNAKE_CASE)

                   __a							=    AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE)
                   self.assertIsInstance(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE)

           finally:
               if "custom" in CONFIG_MAPPING._extra_content:
                   del CONFIG_MAPPING._extra_content["custom"]
               if CustomConfig in TOKENIZER_MAPPING._extra_content:
                   del TOKENIZER_MAPPING._extra_content[CustomConfig]



       @require_tokenizers
       def 		_lowerCamelCase      (		self							:					Dict):


           '''simple docstring'''

           try:
               AutoConfig.register('''custom'''					,		__SCREAMING_SNAKE_CASE)

               # Can register in two steps
               AutoTokenizer.register(__SCREAMING_SNAKE_CASE					,		slow_tokenizer_class=__SCREAMING_SNAKE_CASE)
               self.assertEqual(TOKENIZER_MAPPING[CustomConfig]					,		(CustomTokenizer, None))
               AutoTokenizer.register(__SCREAMING_SNAKE_CASE					,		fast_tokenizer_class=__SCREAMING_SNAKE_CASE)
               self.assertEqual(TOKENIZER_MAPPING[CustomConfig]					,		(CustomTokenizer, CustomTokenizerFast))

               del TOKENIZER_MAPPING._extra_content[CustomConfig]
               # Can register in one step
               AutoTokenizer.register(
                   __SCREAMING_SNAKE_CASE					,		slow_tokenizer_class=__SCREAMING_SNAKE_CASE					,		fast_tokenizer_class=__SCREAMING_SNAKE_CASE)
               self.assertEqual(TOKENIZER_MAPPING[CustomConfig]					,		(CustomTokenizer, CustomTokenizerFast))

               # Trying to register something existing in the Transformers library will raise an error
               with self.assertRaises(__SCREAMING_SNAKE_CASE):
                   AutoTokenizer.register(__SCREAMING_SNAKE_CASE					,		fast_tokenizer_class=__SCREAMING_SNAKE_CASE)

               # We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
               # and that model does not have a tokenizer.json
               with tempfile.TemporaryDirectory() as tmp_dir:
                   __a							=    BertTokenizerFast.from_pretrained(__SCREAMING_SNAKE_CASE)
                   bert_tokenizer.save_pretrained(__SCREAMING_SNAKE_CASE)
                   __a							=    CustomTokenizerFast.from_pretrained(__SCREAMING_SNAKE_CASE)

               with tempfile.TemporaryDirectory() as tmp_dir:
                   tokenizer.save_pretrained(__SCREAMING_SNAKE_CASE)

                   __a							=    AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE)
                   self.assertIsInstance(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE)

                   __a							=    AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE					,		use_fast=__SCREAMING_SNAKE_CASE)
                   self.assertIsInstance(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE)

           finally:
               if "custom" in CONFIG_MAPPING._extra_content:
                   del CONFIG_MAPPING._extra_content["custom"]
               if CustomConfig in TOKENIZER_MAPPING._extra_content:
                   del TOKENIZER_MAPPING._extra_content[CustomConfig]



       def 		_lowerCamelCase      (		self							:					str):


           '''simple docstring'''

           with self.assertRaises(__SCREAMING_SNAKE_CASE):
               __a							=    AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer''')
           # If remote code is disabled, we can't load this config.
           with self.assertRaises(__SCREAMING_SNAKE_CASE):
               __a							=    AutoTokenizer.from_pretrained(
                   '''hf-internal-testing/test_dynamic_tokenizer'''					,		trust_remote_code=__SCREAMING_SNAKE_CASE)

           __a							=    AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer'''					,		trust_remote_code=__SCREAMING_SNAKE_CASE)
           self.assertTrue(tokenizer.special_attribute_present)
           # Test tokenizer can be reloaded.
           with tempfile.TemporaryDirectory() as tmp_dir:
               tokenizer.save_pretrained(__SCREAMING_SNAKE_CASE)
               __a							=    AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE					,		trust_remote_code=__SCREAMING_SNAKE_CASE)
           self.assertTrue(reloaded_tokenizer.special_attribute_present)

           if is_tokenizers_available():
               self.assertEqual(tokenizer.__class__.__name__					,		'''NewTokenizerFast''')
               self.assertEqual(reloaded_tokenizer.__class__.__name__					,		'''NewTokenizerFast''')

               # Test we can also load the slow version
               __a							=    AutoTokenizer.from_pretrained(
                   '''hf-internal-testing/test_dynamic_tokenizer'''					,		trust_remote_code=__SCREAMING_SNAKE_CASE					,		use_fast=__SCREAMING_SNAKE_CASE)
               self.assertTrue(tokenizer.special_attribute_present)
               self.assertEqual(tokenizer.__class__.__name__					,		'''NewTokenizer''')
               # Test tokenizer can be reloaded.
               with tempfile.TemporaryDirectory() as tmp_dir:
                   tokenizer.save_pretrained(__SCREAMING_SNAKE_CASE)
                   __a							=    AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE					,		trust_remote_code=__SCREAMING_SNAKE_CASE					,		use_fast=__SCREAMING_SNAKE_CASE)
               self.assertEqual(reloaded_tokenizer.__class__.__name__					,		'''NewTokenizer''')
               self.assertTrue(reloaded_tokenizer.special_attribute_present)
           else:
               self.assertEqual(tokenizer.__class__.__name__					,		'''NewTokenizer''')
               self.assertEqual(reloaded_tokenizer.__class__.__name__					,		'''NewTokenizer''')



       @require_tokenizers
       def 		_lowerCamelCase      (		self							:					Optional[int]):


           '''simple docstring'''

           class    _A	(     __UpperCAmelCase   ):
                  UpperCamelCase__  :   List[Any]						 =						False


           class    _A	(     __UpperCAmelCase   ):
                  UpperCamelCase__  :   Union[str, Any]						 =						NewTokenizer
                  UpperCamelCase__  :   str						 =						False

           try:
               AutoConfig.register('''custom'''					,		__SCREAMING_SNAKE_CASE)
               AutoTokenizer.register(__SCREAMING_SNAKE_CASE					,		slow_tokenizer_class=__SCREAMING_SNAKE_CASE)
               AutoTokenizer.register(__SCREAMING_SNAKE_CASE					,		fast_tokenizer_class=__SCREAMING_SNAKE_CASE)
               # If remote code is not set, the default is to use local
               __a							=    AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer''')
               self.assertEqual(tokenizer.__class__.__name__					,		'''NewTokenizerFast''')
               self.assertFalse(tokenizer.special_attribute_present)
               __a							=    AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer'''					,		use_fast=__SCREAMING_SNAKE_CASE)
               self.assertEqual(tokenizer.__class__.__name__					,		'''NewTokenizer''')
               self.assertFalse(tokenizer.special_attribute_present)

               # If remote code is disabled, we load the local one.
               __a							=    AutoTokenizer.from_pretrained(
                   '''hf-internal-testing/test_dynamic_tokenizer'''					,		trust_remote_code=__SCREAMING_SNAKE_CASE)
               self.assertEqual(tokenizer.__class__.__name__					,		'''NewTokenizerFast''')
               self.assertFalse(tokenizer.special_attribute_present)
               __a							=    AutoTokenizer.from_pretrained(
                   '''hf-internal-testing/test_dynamic_tokenizer'''					,		trust_remote_code=__SCREAMING_SNAKE_CASE					,		use_fast=__SCREAMING_SNAKE_CASE)
               self.assertEqual(tokenizer.__class__.__name__					,		'''NewTokenizer''')
               self.assertFalse(tokenizer.special_attribute_present)

               # If remote is enabled, we load from the Hub
               __a							=    AutoTokenizer.from_pretrained(
                   '''hf-internal-testing/test_dynamic_tokenizer'''					,		trust_remote_code=__SCREAMING_SNAKE_CASE)
               self.assertEqual(tokenizer.__class__.__name__					,		'''NewTokenizerFast''')
               self.assertTrue(tokenizer.special_attribute_present)
               __a							=    AutoTokenizer.from_pretrained(
                   '''hf-internal-testing/test_dynamic_tokenizer'''					,		trust_remote_code=__SCREAMING_SNAKE_CASE					,		use_fast=__SCREAMING_SNAKE_CASE)
               self.assertEqual(tokenizer.__class__.__name__					,		'''NewTokenizer''')
               self.assertTrue(tokenizer.special_attribute_present)

           finally:
               if "custom" in CONFIG_MAPPING._extra_content:
                   del CONFIG_MAPPING._extra_content["custom"]
               if CustomConfig in TOKENIZER_MAPPING._extra_content:
                   del TOKENIZER_MAPPING._extra_content[CustomConfig]



       def 		_lowerCamelCase      (		self							:					List[str]):


           '''simple docstring'''

           __a							=    AutoTokenizer.from_pretrained(
               '''hf-internal-testing/test_dynamic_tokenizer_legacy'''					,		trust_remote_code=__SCREAMING_SNAKE_CASE)
           self.assertTrue(tokenizer.special_attribute_present)
           if is_tokenizers_available():
               self.assertEqual(tokenizer.__class__.__name__					,		'''NewTokenizerFast''')

               # Test we can also load the slow version
               __a							=    AutoTokenizer.from_pretrained(
                   '''hf-internal-testing/test_dynamic_tokenizer_legacy'''					,		trust_remote_code=__SCREAMING_SNAKE_CASE					,		use_fast=__SCREAMING_SNAKE_CASE)
               self.assertTrue(tokenizer.special_attribute_present)
               self.assertEqual(tokenizer.__class__.__name__					,		'''NewTokenizer''')
           else:
               self.assertEqual(tokenizer.__class__.__name__					,		'''NewTokenizer''')



       def 		_lowerCamelCase      (		self							:					str):


           '''simple docstring'''

           with self.assertRaisesRegex(
               __SCREAMING_SNAKE_CASE					,		'''bert-base is not a local folder and is not a valid model identifier'''):
               __a							=    AutoTokenizer.from_pretrained('''bert-base''')



       def 		_lowerCamelCase      (		self							:					str):


           '''simple docstring'''

           with self.assertRaisesRegex(
               __SCREAMING_SNAKE_CASE					,		r'''aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'''):
               __a							=    AutoTokenizer.from_pretrained(__SCREAMING_SNAKE_CASE					,		revision='''aaaaaa''')





       def 		_lowerCamelCase      (		self							:					Tuple):


           '''simple docstring'''

           __a							=    AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert''')
           with RequestCounter() as counter:
               __a							=    AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert''')
               self.assertEqual(counter.get_request_count					,		0)
               self.assertEqual(counter.head_request_count					,		1)
               self.assertEqual(counter.other_request_count					,		0)










from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


__snake_case						:Tuple						=					{
    '''configuration_bloom''': ['''BLOOM_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''BloomConfig''', '''BloomOnnxConfig'''],
}
try:
 if not is_tokenizers_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __snake_case						:List[Any]						=					['''BloomTokenizerFast''']

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __snake_case						:Any						=					[
     '''BLOOM_PRETRAINED_MODEL_ARCHIVE_LIST''',
     '''BloomForCausalLM''',
     '''BloomModel''',
     '''BloomPreTrainedModel''',
     '''BloomForSequenceClassification''',
     '''BloomForTokenClassification''',
     '''BloomForQuestionAnswering''',
 ]

if TYPE_CHECKING:
 from .configuration_bloom import BLOOM_PRETRAINED_CONFIG_ARCHIVE_MAP, BloomConfig, BloomOnnxConfig

 try:
  if not is_tokenizers_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .tokenization_bloom_fast import BloomTokenizerFast

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_bloom import (
      BLOOM_PRETRAINED_MODEL_ARCHIVE_LIST,
      BloomForCausalLM,
      BloomForQuestionAnswering,
      BloomForSequenceClassification,
      BloomForTokenClassification,
      BloomModel,
      BloomPreTrainedModel,
  )

else:
 import sys

 __snake_case						:Any						=					_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)





