

import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / """utils"""))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
   from test_module.custom_tokenization_fast import CustomTokenizerFast



class SCREAMING_SNAKE_CASE     (  unittest.TestCase			):
  """simple docstring"""






  def 	SCREAMING_SNAKE_CASE   (      self			:   str   )							->     Tuple:






     """simple docstring"""







     __lowerCAmelCase  :      Any    =						0







  @slow
  def 	SCREAMING_SNAKE_CASE   (      self			:   List[Any]   )							->     List[str]:






     """simple docstring"""







     for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
        __lowerCAmelCase  :      str    =						AutoTokenizer.from_pretrained(lowerCAmelCase   )
        self.assertIsNotNone(lowerCAmelCase   )
        self.assertIsInstance(lowerCAmelCase	,							(BertTokenizer, BertTokenizerFast)   )
        self.assertGreater(len(lowerCAmelCase   )	,							0   )

     for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
        __lowerCAmelCase  :      Optional[Any]    =						AutoTokenizer.from_pretrained(lowerCAmelCase   )
        self.assertIsNotNone(lowerCAmelCase   )
        self.assertIsInstance(lowerCAmelCase	,							(GPTaTokenizer, GPTaTokenizerFast)   )
        self.assertGreater(len(lowerCAmelCase   )	,							0   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Optional[Any]   )							->     List[Any]:






     """simple docstring"""







     __lowerCAmelCase  :      str    =						AutoTokenizer.from_pretrained(lowerCAmelCase   )
     self.assertIsInstance(lowerCAmelCase	,							(BertTokenizer, BertTokenizerFast)   )
     self.assertEqual(tokenizer.vocab_size	,							12   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Dict   )							->     str:






     """simple docstring"""







     __lowerCAmelCase  :      Dict    =						AutoTokenizer.from_pretrained(lowerCAmelCase   )
     self.assertIsInstance(lowerCAmelCase	,							(RobertaTokenizer, RobertaTokenizerFast)   )
     self.assertEqual(tokenizer.vocab_size	,							20   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Any   )							->     Any:






     """simple docstring"""







     __lowerCAmelCase  :      Union[str, Any]    =						AutoConfig.from_pretrained(lowerCAmelCase   )
     self.assertIsInstance(lowerCAmelCase	,							lowerCAmelCase   )
     # Check that tokenizer_type ≠ model_type
     __lowerCAmelCase  :      List[Any]    =						AutoTokenizer.from_pretrained(lowerCAmelCase	,							config=lowerCAmelCase   )
     self.assertIsInstance(lowerCAmelCase	,							(BertTokenizer, BertTokenizerFast)   )
     self.assertEqual(tokenizer.vocab_size	,							12   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Tuple   )							->     List[Any]:






     """simple docstring"""







     with tempfile.TemporaryDirectory() as tmp_dir:
        shutil.copy("""./tests/fixtures/vocab.txt"""	,							os.path.join(lowerCAmelCase	,							"""vocab.txt"""   )   )

        __lowerCAmelCase  :      Any    =						AutoTokenizer.from_pretrained(lowerCAmelCase	,							tokenizer_type="""bert"""	,							use_fast=lowerCAmelCase   )
        self.assertIsInstance(lowerCAmelCase	,							lowerCAmelCase   )

     with tempfile.TemporaryDirectory() as tmp_dir:
        shutil.copy("""./tests/fixtures/vocab.json"""	,							os.path.join(lowerCAmelCase	,							"""vocab.json"""   )   )
        shutil.copy("""./tests/fixtures/merges.txt"""	,							os.path.join(lowerCAmelCase	,							"""merges.txt"""   )   )

        __lowerCAmelCase  :      Union[str, Any]    =						AutoTokenizer.from_pretrained(lowerCAmelCase	,							tokenizer_type="""gpt2"""	,							use_fast=lowerCAmelCase   )
        self.assertIsInstance(lowerCAmelCase	,							lowerCAmelCase   )







  @require_tokenizers
  def 	SCREAMING_SNAKE_CASE   (      self			:   Optional[int]   )							->     Tuple:






     """simple docstring"""







     with tempfile.TemporaryDirectory() as tmp_dir:
        shutil.copy("""./tests/fixtures/vocab.txt"""	,							os.path.join(lowerCAmelCase	,							"""vocab.txt"""   )   )

        __lowerCAmelCase  :      Dict    =						AutoTokenizer.from_pretrained(lowerCAmelCase	,							tokenizer_type="""bert"""   )
        self.assertIsInstance(lowerCAmelCase	,							lowerCAmelCase   )

     with tempfile.TemporaryDirectory() as tmp_dir:
        shutil.copy("""./tests/fixtures/vocab.json"""	,							os.path.join(lowerCAmelCase	,							"""vocab.json"""   )   )
        shutil.copy("""./tests/fixtures/merges.txt"""	,							os.path.join(lowerCAmelCase	,							"""merges.txt"""   )   )

        __lowerCAmelCase  :      Union[str, Any]    =						AutoTokenizer.from_pretrained(lowerCAmelCase	,							tokenizer_type="""gpt2"""   )
        self.assertIsInstance(lowerCAmelCase	,							lowerCAmelCase   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Tuple   )							->     Tuple:






     """simple docstring"""







     with pytest.raises(lowerCAmelCase   ):
        AutoTokenizer.from_pretrained("""./"""	,							tokenizer_type="""xxx"""   )







  @require_tokenizers
  def 	SCREAMING_SNAKE_CASE   (      self			:   List[Any]   )							->     Any:






     """simple docstring"""







     for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
        __lowerCAmelCase  :      str    =						tokenizer_class.from_pretrained("""wietsedv/bert-base-dutch-cased"""   )
        self.assertIsInstance(lowerCAmelCase	,							(BertTokenizer, BertTokenizerFast)   )

        if isinstance(lowerCAmelCase	,							lowerCAmelCase   ):
           self.assertEqual(tokenizer.basic_tokenizer.do_lower_case	,							lowerCAmelCase   )
        else:
           self.assertEqual(tokenizer.do_lower_case	,							lowerCAmelCase   )

        self.assertEqual(tokenizer.model_max_length	,							5_12   )







  @require_tokenizers
  def 	SCREAMING_SNAKE_CASE   (      self			:   List[str]   )							->     Optional[Any]:






     """simple docstring"""







     for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
        with self.assertRaisesRegex(
            lowerCAmelCase	,							"""julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier"""	,							):
           __lowerCAmelCase  :      List[str]    =						tokenizer_class.from_pretrained("""julien-c/herlolip-not-exists"""   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Optional[Any]   )							->     Any:






     """simple docstring"""







     __lowerCAmelCase  :      str    =						TOKENIZER_MAPPING.values()
     __lowerCAmelCase  :      Dict    =						[]

     for slow_tok, fast_tok in tokenizers:
        if slow_tok is not None:
           tokenizer_names.append(slow_tok.__name__   )

        if fast_tok is not None:
           tokenizer_names.append(fast_tok.__name__   )

     for tokenizer_name in tokenizer_names:
        # must find the right class
        tokenizer_class_from_name(lowerCAmelCase   )







  @require_tokenizers
  def 	SCREAMING_SNAKE_CASE   (      self			:   Any   )							->     List[Any]:






     """simple docstring"""







     self.assertIsInstance(AutoTokenizer.from_pretrained("""bert-base-cased"""	,							use_fast=lowerCAmelCase   )	,							lowerCAmelCase   )
     self.assertIsInstance(AutoTokenizer.from_pretrained("""bert-base-cased"""   )	,							lowerCAmelCase   )







  @require_tokenizers
  def 	SCREAMING_SNAKE_CASE   (      self			:   int   )							->     Union[str, Any]:






     """simple docstring"""







     __lowerCAmelCase  :      List[Any]    =						AutoTokenizer.from_pretrained("""distilbert-base-uncased"""	,							do_lower_case=lowerCAmelCase   )
     __lowerCAmelCase  :      str    =						"""Hello, world. How are you?"""
     __lowerCAmelCase  :      List[Any]    =						tokenizer.tokenize(lowerCAmelCase   )
     self.assertEqual("""[UNK]"""	,							tokens[0]   )

     __lowerCAmelCase  :      Dict    =						AutoTokenizer.from_pretrained("""microsoft/mpnet-base"""	,							do_lower_case=lowerCAmelCase   )
     __lowerCAmelCase  :      Union[str, Any]    =						tokenizer.tokenize(lowerCAmelCase   )
     self.assertEqual("""[UNK]"""	,							tokens[0]   )







  @require_tokenizers
  def 	SCREAMING_SNAKE_CASE   (      self			:   Optional[Any]   )							->     Union[str, Any]:






     """simple docstring"""







     __lowerCAmelCase  :      Union[str, Any]    =						AutoTokenizer.from_pretrained("""robot-test/dummy-tokenizer-fast-with-model-config"""   )
     self.assertEqual(type(lowerCAmelCase   )	,							lowerCAmelCase   )
     self.assertEqual(tokenizer.model_max_length	,							5_12   )
     self.assertEqual(tokenizer.vocab_size	,							3_00_00   )
     self.assertEqual(tokenizer.unk_token	,							"""[UNK]"""   )
     self.assertEqual(tokenizer.padding_side	,							"""right"""   )
     self.assertEqual(tokenizer.truncation_side	,							"""right"""   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   str   )							->     Tuple:






     """simple docstring"""







     __lowerCAmelCase  :      Dict    =						AutoTokenizer.from_pretrained(lowerCAmelCase   )
     self.assertIsInstance(lowerCAmelCase	,							(BertTokenizer, BertTokenizerFast)   )
     with tempfile.TemporaryDirectory() as tmp_dir:
        tokenizer.save_pretrained(lowerCAmelCase   )
        __lowerCAmelCase  :      Any    =						AutoTokenizer.from_pretrained(lowerCAmelCase   )

     self.assertIsInstance(lowerCAmelCase	,							tokenizer.__class__   )
     self.assertEqual(tokenizera.vocab_size	,							12   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   List[str]   )							->     Optional[int]:






     """simple docstring"""







     __lowerCAmelCase  :      Union[str, Any]    =						AutoTokenizer.from_pretrained("""ctrl"""   )
     # There is no fast CTRL so this always gives us a slow tokenizer.
     self.assertIsInstance(lowerCAmelCase	,							lowerCAmelCase   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Any   )							->     Dict:






     """simple docstring"""







     __lowerCAmelCase  :      Optional[int]    =						get_tokenizer_config("""bert-base-cased"""   )
     __lowerCAmelCase  :      Any    =						config.pop("""_commit_hash"""	,							lowerCAmelCase   )
     # If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
     self.assertEqual(lowerCAmelCase	,							{"""do_lower_case""": False}   )

     # This model does not have a tokenizer_config so we get back an empty dict.
     __lowerCAmelCase  :      List[Any]    =						get_tokenizer_config(lowerCAmelCase   )
     self.assertDictEqual(lowerCAmelCase	,							{}   )

     # A tokenizer saved with `save_pretrained` always creates a tokenizer config.
     __lowerCAmelCase  :      Dict    =						AutoTokenizer.from_pretrained(lowerCAmelCase   )
     with tempfile.TemporaryDirectory() as tmp_dir:
        tokenizer.save_pretrained(lowerCAmelCase   )
        __lowerCAmelCase  :      Tuple    =						get_tokenizer_config(lowerCAmelCase   )

     # Check the class of the tokenizer was properly saved (note that it always saves the slow class).
     self.assertEqual(config["""tokenizer_class"""]	,							"""BertTokenizer"""   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Any   )							->     Optional[int]:






     """simple docstring"""







     try:
        AutoConfig.register("""custom"""	,							lowerCAmelCase   )

        AutoTokenizer.register(lowerCAmelCase	,							slow_tokenizer_class=lowerCAmelCase   )
        # Trying to register something existing in the Transformers library will raise an error
        with self.assertRaises(lowerCAmelCase   ):
           AutoTokenizer.register(lowerCAmelCase	,							slow_tokenizer_class=lowerCAmelCase   )

        __lowerCAmelCase  :      Tuple    =						CustomTokenizer.from_pretrained(lowerCAmelCase   )
        with tempfile.TemporaryDirectory() as tmp_dir:
           tokenizer.save_pretrained(lowerCAmelCase   )

           __lowerCAmelCase  :      Union[str, Any]    =						AutoTokenizer.from_pretrained(lowerCAmelCase   )
           self.assertIsInstance(lowerCAmelCase	,							lowerCAmelCase   )

     finally:
        if "custom" in CONFIG_MAPPING._extra_content:
           del CONFIG_MAPPING._extra_content["custom"]
        if CustomConfig in TOKENIZER_MAPPING._extra_content:
           del TOKENIZER_MAPPING._extra_content[CustomConfig]







  @require_tokenizers
  def 	SCREAMING_SNAKE_CASE   (      self			:   str   )							->     Any:






     """simple docstring"""







     try:
        AutoConfig.register("""custom"""	,							lowerCAmelCase   )

        # Can register in two steps
        AutoTokenizer.register(lowerCAmelCase	,							slow_tokenizer_class=lowerCAmelCase   )
        self.assertEqual(TOKENIZER_MAPPING[CustomConfig]	,							(CustomTokenizer, None)   )
        AutoTokenizer.register(lowerCAmelCase	,							fast_tokenizer_class=lowerCAmelCase   )
        self.assertEqual(TOKENIZER_MAPPING[CustomConfig]	,							(CustomTokenizer, CustomTokenizerFast)   )

        del TOKENIZER_MAPPING._extra_content[CustomConfig]
        # Can register in one step
        AutoTokenizer.register(
            lowerCAmelCase	,							slow_tokenizer_class=lowerCAmelCase	,							fast_tokenizer_class=lowerCAmelCase   )
        self.assertEqual(TOKENIZER_MAPPING[CustomConfig]	,							(CustomTokenizer, CustomTokenizerFast)   )

        # Trying to register something existing in the Transformers library will raise an error
        with self.assertRaises(lowerCAmelCase   ):
           AutoTokenizer.register(lowerCAmelCase	,							fast_tokenizer_class=lowerCAmelCase   )

        # We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
        # and that model does not have a tokenizer.json
        with tempfile.TemporaryDirectory() as tmp_dir:
           __lowerCAmelCase  :      List[Any]    =						BertTokenizerFast.from_pretrained(lowerCAmelCase   )
           bert_tokenizer.save_pretrained(lowerCAmelCase   )
           __lowerCAmelCase  :      Union[str, Any]    =						CustomTokenizerFast.from_pretrained(lowerCAmelCase   )

        with tempfile.TemporaryDirectory() as tmp_dir:
           tokenizer.save_pretrained(lowerCAmelCase   )

           __lowerCAmelCase  :      List[str]    =						AutoTokenizer.from_pretrained(lowerCAmelCase   )
           self.assertIsInstance(lowerCAmelCase	,							lowerCAmelCase   )

           __lowerCAmelCase  :      List[Any]    =						AutoTokenizer.from_pretrained(lowerCAmelCase	,							use_fast=lowerCAmelCase   )
           self.assertIsInstance(lowerCAmelCase	,							lowerCAmelCase   )

     finally:
        if "custom" in CONFIG_MAPPING._extra_content:
           del CONFIG_MAPPING._extra_content["custom"]
        if CustomConfig in TOKENIZER_MAPPING._extra_content:
           del TOKENIZER_MAPPING._extra_content[CustomConfig]







  def 	SCREAMING_SNAKE_CASE   (      self			:   Tuple   )							->     Optional[int]:






     """simple docstring"""







     with self.assertRaises(lowerCAmelCase   ):
        __lowerCAmelCase  :      int    =						AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""   )
     # If remote code is disabled, we can't load this config.
     with self.assertRaises(lowerCAmelCase   ):
        __lowerCAmelCase  :      List[Any]    =						AutoTokenizer.from_pretrained(
            """hf-internal-testing/test_dynamic_tokenizer"""	,							trust_remote_code=lowerCAmelCase   )

     __lowerCAmelCase  :      Dict    =						AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""	,							trust_remote_code=lowerCAmelCase   )
     self.assertTrue(tokenizer.special_attribute_present   )
     # Test tokenizer can be reloaded.
     with tempfile.TemporaryDirectory() as tmp_dir:
        tokenizer.save_pretrained(lowerCAmelCase   )
        __lowerCAmelCase  :      List[str]    =						AutoTokenizer.from_pretrained(lowerCAmelCase	,							trust_remote_code=lowerCAmelCase   )
     self.assertTrue(reloaded_tokenizer.special_attribute_present   )

     if is_tokenizers_available():
        self.assertEqual(tokenizer.__class__.__name__	,							"""NewTokenizerFast"""   )
        self.assertEqual(reloaded_tokenizer.__class__.__name__	,							"""NewTokenizerFast"""   )

        # Test we can also load the slow version
        __lowerCAmelCase  :      List[str]    =						AutoTokenizer.from_pretrained(
            """hf-internal-testing/test_dynamic_tokenizer"""	,							trust_remote_code=lowerCAmelCase	,							use_fast=lowerCAmelCase   )
        self.assertTrue(tokenizer.special_attribute_present   )
        self.assertEqual(tokenizer.__class__.__name__	,							"""NewTokenizer"""   )
        # Test tokenizer can be reloaded.
        with tempfile.TemporaryDirectory() as tmp_dir:
           tokenizer.save_pretrained(lowerCAmelCase   )
           __lowerCAmelCase  :      Optional[Any]    =						AutoTokenizer.from_pretrained(lowerCAmelCase	,							trust_remote_code=lowerCAmelCase	,							use_fast=lowerCAmelCase   )
        self.assertEqual(reloaded_tokenizer.__class__.__name__	,							"""NewTokenizer"""   )
        self.assertTrue(reloaded_tokenizer.special_attribute_present   )
     else:
        self.assertEqual(tokenizer.__class__.__name__	,							"""NewTokenizer"""   )
        self.assertEqual(reloaded_tokenizer.__class__.__name__	,							"""NewTokenizer"""   )







  @require_tokenizers
  def 	SCREAMING_SNAKE_CASE   (      self			:   List[Any]   )							->     List[Any]:






     """simple docstring"""







     class SCREAMING_SNAKE_CASE     (  a_			):
       """simple docstring"""






       lowerCamelCase   :      int										=False






     class SCREAMING_SNAKE_CASE     (  a_			):
       """simple docstring"""






       lowerCamelCase   :      Dict										=NewTokenizer
       lowerCamelCase   :      Any										=False

     try:
        AutoConfig.register("""custom"""	,							lowerCAmelCase   )
        AutoTokenizer.register(lowerCAmelCase	,							slow_tokenizer_class=lowerCAmelCase   )
        AutoTokenizer.register(lowerCAmelCase	,							fast_tokenizer_class=lowerCAmelCase   )
        # If remote code is not set, the default is to use local
        __lowerCAmelCase  :      Union[str, Any]    =						AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""   )
        self.assertEqual(tokenizer.__class__.__name__	,							"""NewTokenizerFast"""   )
        self.assertFalse(tokenizer.special_attribute_present   )
        __lowerCAmelCase  :      Union[str, Any]    =						AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""	,							use_fast=lowerCAmelCase   )
        self.assertEqual(tokenizer.__class__.__name__	,							"""NewTokenizer"""   )
        self.assertFalse(tokenizer.special_attribute_present   )

        # If remote code is disabled, we load the local one.
        __lowerCAmelCase  :      Optional[Any]    =						AutoTokenizer.from_pretrained(
            """hf-internal-testing/test_dynamic_tokenizer"""	,							trust_remote_code=lowerCAmelCase   )
        self.assertEqual(tokenizer.__class__.__name__	,							"""NewTokenizerFast"""   )
        self.assertFalse(tokenizer.special_attribute_present   )
        __lowerCAmelCase  :      Union[str, Any]    =						AutoTokenizer.from_pretrained(
            """hf-internal-testing/test_dynamic_tokenizer"""	,							trust_remote_code=lowerCAmelCase	,							use_fast=lowerCAmelCase   )
        self.assertEqual(tokenizer.__class__.__name__	,							"""NewTokenizer"""   )
        self.assertFalse(tokenizer.special_attribute_present   )

        # If remote is enabled, we load from the Hub
        __lowerCAmelCase  :      int    =						AutoTokenizer.from_pretrained(
            """hf-internal-testing/test_dynamic_tokenizer"""	,							trust_remote_code=lowerCAmelCase   )
        self.assertEqual(tokenizer.__class__.__name__	,							"""NewTokenizerFast"""   )
        self.assertTrue(tokenizer.special_attribute_present   )
        __lowerCAmelCase  :      str    =						AutoTokenizer.from_pretrained(
            """hf-internal-testing/test_dynamic_tokenizer"""	,							trust_remote_code=lowerCAmelCase	,							use_fast=lowerCAmelCase   )
        self.assertEqual(tokenizer.__class__.__name__	,							"""NewTokenizer"""   )
        self.assertTrue(tokenizer.special_attribute_present   )

     finally:
        if "custom" in CONFIG_MAPPING._extra_content:
           del CONFIG_MAPPING._extra_content["custom"]
        if CustomConfig in TOKENIZER_MAPPING._extra_content:
           del TOKENIZER_MAPPING._extra_content[CustomConfig]







  def 	SCREAMING_SNAKE_CASE   (      self			:   int   )							->     Optional[int]:






     """simple docstring"""







     __lowerCAmelCase  :      Optional[Any]    =						AutoTokenizer.from_pretrained(
         """hf-internal-testing/test_dynamic_tokenizer_legacy"""	,							trust_remote_code=lowerCAmelCase   )
     self.assertTrue(tokenizer.special_attribute_present   )
     if is_tokenizers_available():
        self.assertEqual(tokenizer.__class__.__name__	,							"""NewTokenizerFast"""   )

        # Test we can also load the slow version
        __lowerCAmelCase  :      Optional[Any]    =						AutoTokenizer.from_pretrained(
            """hf-internal-testing/test_dynamic_tokenizer_legacy"""	,							trust_remote_code=lowerCAmelCase	,							use_fast=lowerCAmelCase   )
        self.assertTrue(tokenizer.special_attribute_present   )
        self.assertEqual(tokenizer.__class__.__name__	,							"""NewTokenizer"""   )
     else:
        self.assertEqual(tokenizer.__class__.__name__	,							"""NewTokenizer"""   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   str   )							->     int:






     """simple docstring"""







     with self.assertRaisesRegex(
         lowerCAmelCase	,							"""bert-base is not a local folder and is not a valid model identifier"""   ):
        __lowerCAmelCase  :      Dict    =						AutoTokenizer.from_pretrained("""bert-base"""   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Optional[int]   )							->     Optional[int]:






     """simple docstring"""







     with self.assertRaisesRegex(
         lowerCAmelCase	,							r"""aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"""   ):
        __lowerCAmelCase  :      str    =						AutoTokenizer.from_pretrained(lowerCAmelCase	,							revision="""aaaaaa"""   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   List[Any]   )							->     Tuple:






     """simple docstring"""







     __lowerCAmelCase  :      Optional[int]    =						AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""   )
     with RequestCounter() as counter:
        __lowerCAmelCase  :      List[Any]    =						AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""   )
        self.assertEqual(counter.get_request_count	,							0   )
        self.assertEqual(counter.head_request_count	,							1   )
        self.assertEqual(counter.other_request_count	,							0   )





import unittest

from transformers import MobileBertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
   import torch

   from transformers import (
       MODEL_FOR_PRETRAINING_MAPPING,
       MobileBertForMaskedLM,
       MobileBertForMultipleChoice,
       MobileBertForNextSentencePrediction,
       MobileBertForPreTraining,
       MobileBertForQuestionAnswering,
       MobileBertForSequenceClassification,
       MobileBertForTokenClassification,
       MobileBertModel,
   )



class SCREAMING_SNAKE_CASE     :
  """simple docstring"""






  def __init__(      self			:   Union[str, Any]	,							lowerCAmelCase			:   Dict	,							lowerCAmelCase			:   Dict=13	,							lowerCAmelCase			:   int=7	,							lowerCAmelCase			:   Any=True	,							lowerCAmelCase			:   str=True	,							lowerCAmelCase			:   Optional[Any]=True	,							lowerCAmelCase			:   str=True	,							lowerCAmelCase			:   Tuple=99	,							lowerCAmelCase			:   int=64	,							lowerCAmelCase			:   Any=32	,							lowerCAmelCase			:   str=5	,							lowerCAmelCase			:   List[str]=4	,							lowerCAmelCase			:   str=37	,							lowerCAmelCase			:   Union[str, Any]="gelu"	,							lowerCAmelCase			:   Dict=0.1	,							lowerCAmelCase			:   Dict=0.1	,							lowerCAmelCase			:   Optional[int]=5_12	,							lowerCAmelCase			:   List[str]=16	,							lowerCAmelCase			:   str=2	,							lowerCAmelCase			:   Union[str, Any]=0.02	,							lowerCAmelCase			:   Dict=3	,							lowerCAmelCase			:   int=4	,							lowerCAmelCase			:   Union[str, Any]=None	,							)							->     Optional[int]:






     """simple docstring"""







     __lowerCAmelCase  :      List[Any]    =						parent
     __lowerCAmelCase  :      Tuple    =						batch_size
     __lowerCAmelCase  :      Dict    =						seq_length
     __lowerCAmelCase  :      List[str]    =						is_training
     __lowerCAmelCase  :      Dict    =						use_input_mask
     __lowerCAmelCase  :      Optional[int]    =						use_token_type_ids
     __lowerCAmelCase  :      List[str]    =						use_labels
     __lowerCAmelCase  :      Dict    =						vocab_size
     __lowerCAmelCase  :      List[str]    =						hidden_size
     __lowerCAmelCase  :      Optional[int]    =						embedding_size
     __lowerCAmelCase  :      Optional[int]    =						num_hidden_layers
     __lowerCAmelCase  :      Optional[Any]    =						num_attention_heads
     __lowerCAmelCase  :      Optional[Any]    =						intermediate_size
     __lowerCAmelCase  :      Optional[int]    =						hidden_act
     __lowerCAmelCase  :      Any    =						hidden_dropout_prob
     __lowerCAmelCase  :      Optional[int]    =						attention_probs_dropout_prob
     __lowerCAmelCase  :      List[str]    =						max_position_embeddings
     __lowerCAmelCase  :      Optional[Any]    =						type_vocab_size
     __lowerCAmelCase  :      Optional[Any]    =						type_sequence_label_size
     __lowerCAmelCase  :      Optional[Any]    =						initializer_range
     __lowerCAmelCase  :      Optional[Any]    =						num_labels
     __lowerCAmelCase  :      Union[str, Any]    =						num_choices
     __lowerCAmelCase  :      Union[str, Any]    =						scope







  def 	SCREAMING_SNAKE_CASE   (      self			:   Dict   )							->     List[str]:






     """simple docstring"""







     __lowerCAmelCase  :      Union[str, Any]    =						ids_tensor([self.batch_size, self.seq_length]	,							self.vocab_size   )

     __lowerCAmelCase  :      Dict    =						None
     if self.use_input_mask:
        __lowerCAmelCase  :      Union[str, Any]    =						random_attention_mask([self.batch_size, self.seq_length]   )

     __lowerCAmelCase  :      List[str]    =						None
     if self.use_token_type_ids:
        __lowerCAmelCase  :      Tuple    =						ids_tensor([self.batch_size, self.seq_length]	,							self.type_vocab_size   )

     __lowerCAmelCase  :      List[Any]    =						None
     __lowerCAmelCase  :      Tuple    =						None
     __lowerCAmelCase  :      int    =						None
     if self.use_labels:
        __lowerCAmelCase  :      Tuple    =						ids_tensor([self.batch_size]	,							self.type_sequence_label_size   )
        __lowerCAmelCase  :      List[str]    =						ids_tensor([self.batch_size, self.seq_length]	,							self.num_labels   )
        __lowerCAmelCase  :      str    =						ids_tensor([self.batch_size]	,							self.num_choices   )

     __lowerCAmelCase  :      Optional[int]    =						self.get_config()

     return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels







  def 	SCREAMING_SNAKE_CASE   (      self			:   List[Any]   )							->     Union[str, Any]:






     """simple docstring"""







     return MobileBertConfig(
         vocab_size=self.vocab_size	,							hidden_size=self.hidden_size	,							num_hidden_layers=self.num_hidden_layers	,							num_attention_heads=self.num_attention_heads	,							intermediate_size=self.intermediate_size	,							embedding_size=self.embedding_size	,							hidden_act=self.hidden_act	,							hidden_dropout_prob=self.hidden_dropout_prob	,							attention_probs_dropout_prob=self.attention_probs_dropout_prob	,							max_position_embeddings=self.max_position_embeddings	,							type_vocab_size=self.type_vocab_size	,							is_decoder=lowerCAmelCase	,							initializer_range=self.initializer_range	,							)







  def 	SCREAMING_SNAKE_CASE   (      self			:   Optional[Any]	,							lowerCAmelCase			:   Dict	,							lowerCAmelCase			:   Optional[Any]	,							lowerCAmelCase			:   Optional[int]	,							lowerCAmelCase			:   int	,							lowerCAmelCase			:   Any	,							lowerCAmelCase			:   Tuple	,							lowerCAmelCase			:   List[Any]   )							->     Tuple:






     """simple docstring"""







     __lowerCAmelCase  :      Any    =						MobileBertModel(config=lowerCAmelCase   )
     model.to(lowerCAmelCase   )
     model.eval()
     __lowerCAmelCase  :      Union[str, Any]    =						model(lowerCAmelCase	,							attention_mask=lowerCAmelCase	,							token_type_ids=lowerCAmelCase   )
     __lowerCAmelCase  :      List[Any]    =						model(lowerCAmelCase	,							token_type_ids=lowerCAmelCase   )
     __lowerCAmelCase  :      Tuple    =						model(lowerCAmelCase   )

     self.parent.assertEqual(result.last_hidden_state.shape	,							(self.batch_size, self.seq_length, self.hidden_size)   )
     self.parent.assertEqual(result.pooler_output.shape	,							(self.batch_size, self.hidden_size)   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Optional[int]	,							lowerCAmelCase			:   Union[str, Any]	,							lowerCAmelCase			:   Any	,							lowerCAmelCase			:   str	,							lowerCAmelCase			:   List[Any]	,							lowerCAmelCase			:   Dict	,							lowerCAmelCase			:   Union[str, Any]	,							lowerCAmelCase			:   Optional[int]   )							->     Optional[Any]:






     """simple docstring"""







     __lowerCAmelCase  :      Any    =						MobileBertForMaskedLM(config=lowerCAmelCase   )
     model.to(lowerCAmelCase   )
     model.eval()
     __lowerCAmelCase  :      List[Any]    =						model(lowerCAmelCase	,							attention_mask=lowerCAmelCase	,							token_type_ids=lowerCAmelCase	,							labels=lowerCAmelCase   )
     self.parent.assertEqual(result.logits.shape	,							(self.batch_size, self.seq_length, self.vocab_size)   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Dict	,							lowerCAmelCase			:   List[str]	,							lowerCAmelCase			:   Union[str, Any]	,							lowerCAmelCase			:   Union[str, Any]	,							lowerCAmelCase			:   str	,							lowerCAmelCase			:   Optional[Any]	,							lowerCAmelCase			:   Dict	,							lowerCAmelCase			:   Optional[int]   )							->     List[str]:






     """simple docstring"""







     __lowerCAmelCase  :      Tuple    =						MobileBertForNextSentencePrediction(config=lowerCAmelCase   )
     model.to(lowerCAmelCase   )
     model.eval()
     __lowerCAmelCase  :      List[Any]    =						model(
         lowerCAmelCase	,							attention_mask=lowerCAmelCase	,							token_type_ids=lowerCAmelCase	,							labels=lowerCAmelCase	,							)
     self.parent.assertEqual(result.logits.shape	,							(self.batch_size, 2)   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Tuple	,							lowerCAmelCase			:   Any	,							lowerCAmelCase			:   Any	,							lowerCAmelCase			:   Optional[int]	,							lowerCAmelCase			:   Union[str, Any]	,							lowerCAmelCase			:   Tuple	,							lowerCAmelCase			:   List[Any]	,							lowerCAmelCase			:   List[Any]   )							->     Optional[Any]:






     """simple docstring"""







     __lowerCAmelCase  :      Any    =						MobileBertForPreTraining(config=lowerCAmelCase   )
     model.to(lowerCAmelCase   )
     model.eval()
     __lowerCAmelCase  :      Optional[int]    =						model(
         lowerCAmelCase	,							attention_mask=lowerCAmelCase	,							token_type_ids=lowerCAmelCase	,							labels=lowerCAmelCase	,							next_sentence_label=lowerCAmelCase	,							)
     self.parent.assertEqual(result.prediction_logits.shape	,							(self.batch_size, self.seq_length, self.vocab_size)   )
     self.parent.assertEqual(result.seq_relationship_logits.shape	,							(self.batch_size, 2)   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Dict	,							lowerCAmelCase			:   Tuple	,							lowerCAmelCase			:   int	,							lowerCAmelCase			:   Dict	,							lowerCAmelCase			:   Optional[int]	,							lowerCAmelCase			:   int	,							lowerCAmelCase			:   Union[str, Any]	,							lowerCAmelCase			:   Any   )							->     Any:






     """simple docstring"""







     __lowerCAmelCase  :      Optional[int]    =						MobileBertForQuestionAnswering(config=lowerCAmelCase   )
     model.to(lowerCAmelCase   )
     model.eval()
     __lowerCAmelCase  :      int    =						model(
         lowerCAmelCase	,							attention_mask=lowerCAmelCase	,							token_type_ids=lowerCAmelCase	,							start_positions=lowerCAmelCase	,							end_positions=lowerCAmelCase	,							)
     self.parent.assertEqual(result.start_logits.shape	,							(self.batch_size, self.seq_length)   )
     self.parent.assertEqual(result.end_logits.shape	,							(self.batch_size, self.seq_length)   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Dict	,							lowerCAmelCase			:   List[Any]	,							lowerCAmelCase			:   str	,							lowerCAmelCase			:   str	,							lowerCAmelCase			:   List[str]	,							lowerCAmelCase			:   int	,							lowerCAmelCase			:   Union[str, Any]	,							lowerCAmelCase			:   Union[str, Any]   )							->     Tuple:






     """simple docstring"""







     __lowerCAmelCase  :      List[str]    =						self.num_labels
     __lowerCAmelCase  :      int    =						MobileBertForSequenceClassification(lowerCAmelCase   )
     model.to(lowerCAmelCase   )
     model.eval()
     __lowerCAmelCase  :      Optional[int]    =						model(lowerCAmelCase	,							attention_mask=lowerCAmelCase	,							token_type_ids=lowerCAmelCase	,							labels=lowerCAmelCase   )
     self.parent.assertEqual(result.logits.shape	,							(self.batch_size, self.num_labels)   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Union[str, Any]	,							lowerCAmelCase			:   Optional[Any]	,							lowerCAmelCase			:   Union[str, Any]	,							lowerCAmelCase			:   Tuple	,							lowerCAmelCase			:   int	,							lowerCAmelCase			:   int	,							lowerCAmelCase			:   Any	,							lowerCAmelCase			:   Optional[int]   )							->     Optional[Any]:






     """simple docstring"""







     __lowerCAmelCase  :      List[Any]    =						self.num_labels
     __lowerCAmelCase  :      Dict    =						MobileBertForTokenClassification(config=lowerCAmelCase   )
     model.to(lowerCAmelCase   )
     model.eval()
     __lowerCAmelCase  :      Tuple    =						model(lowerCAmelCase	,							attention_mask=lowerCAmelCase	,							token_type_ids=lowerCAmelCase	,							labels=lowerCAmelCase   )
     self.parent.assertEqual(result.logits.shape	,							(self.batch_size, self.seq_length, self.num_labels)   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   List[str]	,							lowerCAmelCase			:   List[Any]	,							lowerCAmelCase			:   Dict	,							lowerCAmelCase			:   Union[str, Any]	,							lowerCAmelCase			:   Optional[Any]	,							lowerCAmelCase			:   Union[str, Any]	,							lowerCAmelCase			:   Any	,							lowerCAmelCase			:   List[Any]   )							->     Optional[Any]:






     """simple docstring"""







     __lowerCAmelCase  :      Any    =						self.num_choices
     __lowerCAmelCase  :      List[Any]    =						MobileBertForMultipleChoice(config=lowerCAmelCase   )
     model.to(lowerCAmelCase   )
     model.eval()
     __lowerCAmelCase  :      Dict    =						input_ids.unsqueeze(1   ).expand(-1	,							self.num_choices	,							-1   ).contiguous()
     __lowerCAmelCase  :      Union[str, Any]    =						token_type_ids.unsqueeze(1   ).expand(-1	,							self.num_choices	,							-1   ).contiguous()
     __lowerCAmelCase  :      Optional[int]    =						input_mask.unsqueeze(1   ).expand(-1	,							self.num_choices	,							-1   ).contiguous()
     __lowerCAmelCase  :      Optional[int]    =						model(
         lowerCAmelCase	,							attention_mask=lowerCAmelCase	,							token_type_ids=lowerCAmelCase	,							labels=lowerCAmelCase	,							)
     self.parent.assertEqual(result.logits.shape	,							(self.batch_size, self.num_choices)   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   str   )							->     List[str]:






     """simple docstring"""







     __lowerCAmelCase  :      Tuple    =						self.prepare_config_and_inputs()
     (
         (
         __lowerCAmelCase
     )   ,(
         __lowerCAmelCase
     )   ,(
         __lowerCAmelCase
     )   ,(
         __lowerCAmelCase
     )   ,(
         __lowerCAmelCase
     )   ,(
         __lowerCAmelCase
     )   ,(
         __lowerCAmelCase
     )   ,
     )  :      List[Any]    =						config_and_inputs
     __lowerCAmelCase  :      List[str]    =						{"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
     return config, inputs_dict






@require_torch
class SCREAMING_SNAKE_CASE     (  a_       ,							a_       ,							unittest.TestCase			):
  """simple docstring"""






  lowerCamelCase   :      str										=(
      (
          MobileBertModel,
          MobileBertForMaskedLM,
          MobileBertForMultipleChoice,
          MobileBertForNextSentencePrediction,
          MobileBertForPreTraining,
          MobileBertForQuestionAnswering,
          MobileBertForSequenceClassification,
          MobileBertForTokenClassification,
      )
      if is_torch_available()
      else ()
  )
  lowerCamelCase   :      Optional[int]										=(
      {
          "feature-extraction": MobileBertModel,
          "fill-mask": MobileBertForMaskedLM,
          "question-answering": MobileBertForQuestionAnswering,
          "text-classification": MobileBertForSequenceClassification,
          "token-classification": MobileBertForTokenClassification,
          "zero-shot": MobileBertForSequenceClassification,
      }
      if is_torch_available()
      else {}
  )
  lowerCamelCase   :      Union[str, Any]										=True







  def 	SCREAMING_SNAKE_CASE   (      self			:   List[Any]	,							lowerCAmelCase			:   str	,							lowerCAmelCase			:   List[str]	,							lowerCAmelCase			:   List[Any]=False   )							->     List[str]:






     """simple docstring"""







     __lowerCAmelCase  :      Union[str, Any]    =						super()._prepare_for_class(lowerCAmelCase	,							lowerCAmelCase	,							return_labels=lowerCAmelCase   )

     if return_labels:
        if model_class in get_values(lowerCAmelCase   ):
           __lowerCAmelCase  :      Tuple    =						torch.zeros(
               (self.model_tester.batch_size, self.model_tester.seq_length)	,							dtype=torch.long	,							device=lowerCAmelCase   )
           __lowerCAmelCase  :      Union[str, Any]    =						torch.zeros(
               self.model_tester.batch_size	,							dtype=torch.long	,							device=lowerCAmelCase   )
     return inputs_dict







  def 	SCREAMING_SNAKE_CASE   (      self			:   int   )							->     Optional[Any]:






     """simple docstring"""







     __lowerCAmelCase  :      int    =						MobileBertModelTester(self   )
     __lowerCAmelCase  :      Optional[Any]    =						ConfigTester(self	,							config_class=lowerCAmelCase	,							hidden_size=37   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   str   )							->     Optional[Any]:






     """simple docstring"""







     self.config_tester.run_common_tests()







  def 	SCREAMING_SNAKE_CASE   (      self			:   int   )							->     Optional[Any]:






     """simple docstring"""







     __lowerCAmelCase  :      List[str]    =						self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_mobilebert_model(*lowerCAmelCase   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Dict   )							->     Tuple:






     """simple docstring"""







     __lowerCAmelCase  :      str    =						self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_mobilebert_for_masked_lm(*lowerCAmelCase   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Optional[int]   )							->     Dict:






     """simple docstring"""







     __lowerCAmelCase  :      str    =						self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_mobilebert_for_multiple_choice(*lowerCAmelCase   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Union[str, Any]   )							->     Dict:






     """simple docstring"""







     __lowerCAmelCase  :      Dict    =						self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_mobilebert_for_next_sequence_prediction(*lowerCAmelCase   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Dict   )							->     Optional[int]:






     """simple docstring"""







     __lowerCAmelCase  :      Tuple    =						self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_mobilebert_for_pretraining(*lowerCAmelCase   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Union[str, Any]   )							->     str:






     """simple docstring"""







     __lowerCAmelCase  :      Dict    =						self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_mobilebert_for_question_answering(*lowerCAmelCase   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   List[str]   )							->     Tuple:






     """simple docstring"""







     __lowerCAmelCase  :      Dict    =						self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_mobilebert_for_sequence_classification(*lowerCAmelCase   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Optional[int]   )							->     Tuple:






     """simple docstring"""







     __lowerCAmelCase  :      List[str]    =						self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_mobilebert_for_token_classification(*lowerCAmelCase   )






def      snake_case_				(__A					:						Any					)      ->     Optional[Any]:
   return torch.tensor(
       __A ,       dtype=torch.long ,       device=__A ,       )


__UpperCAmelCase								=			1e-3



@require_torch
@require_sentencepiece
@require_tokenizers
class SCREAMING_SNAKE_CASE     (  unittest.TestCase			):
  """simple docstring"""






  @slow
  def 	SCREAMING_SNAKE_CASE   (      self			:   Dict   )							->     Union[str, Any]:






     """simple docstring"""







     __lowerCAmelCase  :      Optional[Any]    =						MobileBertModel.from_pretrained("""google/mobilebert-uncased"""   ).to(lowerCAmelCase   )
     __lowerCAmelCase  :      int    =						_long_tensor([[1_01, 71_10, 10_05, 10_56, 20_23, 1_13_33, 1_74_13, 10_29, 1_02]]   )
     with torch.no_grad():
        __lowerCAmelCase  :      List[str]    =						model(lowerCAmelCase   )[0]
     __lowerCAmelCase  :      List[Any]    =						torch.Size((1, 9, 5_12)   )
     self.assertEqual(output.shape	,							lowerCAmelCase   )
     __lowerCAmelCase  :      int    =						torch.tensor(
         [
             [
                 [-2.473_6526e07, 8.269_1656e04, 1.652_1838e05],
                 [-5.754_1704e-01, 3.905_6022e00, 4.401_1507e00],
                 [2.604_7359e00, 1.567_7652e00, -1.732_4188e-01],
             ]
         ]	,							device=lowerCAmelCase	,							)

     # MobileBERT results range from 10e0 to 10e8. Even a 0.0000001% difference with a value of 10e8 results in a
     # ~1 difference, it's therefore not a good idea to measure using addition.
     # Here, we instead divide the expected result with the result in order to obtain ~1. We then check that the
     # result is held between bounds: 1 - TOLERANCE < expected_result / result < 1 + TOLERANCE
     __lowerCAmelCase  :      Union[str, Any]    =						torch.all((expected_slice / output[..., :3, :3]) >= 1 - TOLERANCE   )
     __lowerCAmelCase  :      Union[str, Any]    =						torch.all((expected_slice / output[..., :3, :3]) <= 1 + TOLERANCE   )

     self.assertTrue(lower_bound and upper_bound   )


