


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_squeezebert import SqueezeBertTokenizer


UpperCAmelCase_				:			Union[str, Any]   =	logging.get_logger(__name__)

UpperCAmelCase_				:			Optional[int]   =	{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

UpperCAmelCase_				:			List[Any]   =	{
    'vocab_file': {
        'squeezebert/squeezebert-uncased': (
            'https://huggingface.co/squeezebert/squeezebert-uncased/resolve/main/vocab.txt'
        ),
        'squeezebert/squeezebert-mnli': 'https://huggingface.co/squeezebert/squeezebert-mnli/resolve/main/vocab.txt',
        'squeezebert/squeezebert-mnli-headless': (
            'https://huggingface.co/squeezebert/squeezebert-mnli-headless/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'squeezebert/squeezebert-uncased': (
            'https://huggingface.co/squeezebert/squeezebert-uncased/resolve/main/tokenizer.json'
        ),
        'squeezebert/squeezebert-mnli': (
            'https://huggingface.co/squeezebert/squeezebert-mnli/resolve/main/tokenizer.json'
        ),
        'squeezebert/squeezebert-mnli-headless': (
            'https://huggingface.co/squeezebert/squeezebert-mnli-headless/resolve/main/tokenizer.json'
        ),
    },
}

UpperCAmelCase_				:			Optional[int]   =	{
    'squeezebert/squeezebert-uncased': 512,
    'squeezebert/squeezebert-mnli': 512,
    'squeezebert/squeezebert-mnli-headless': 512,
}


UpperCAmelCase_				:			List[Any]   =	{
    'squeezebert/squeezebert-uncased': {'do_lower_case': True},
    'squeezebert/squeezebert-mnli': {'do_lower_case': True},
    'squeezebert/squeezebert-mnli-headless': {'do_lower_case': True},
}
class 			SCREAMING_SNAKE_CASE__  (    lowercase__      ):
      snake_case__    :					Optional[int]					       =						VOCAB_FILES_NAMES
      snake_case__    :					Dict					       =						PRETRAINED_VOCAB_FILES_MAP
      snake_case__    :					Union[str, Any]					       =						PRETRAINED_INIT_CONFIGURATION
      snake_case__    :					Dict					       =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      snake_case__    :					Union[str, Any]					       =						SqueezeBertTokenizer
      def __init__(	self    :	Optional[int]					,							SCREAMING_SNAKE_CASE__    :	Union[str, Any]=None					,							SCREAMING_SNAKE_CASE__    :	str=None					,							SCREAMING_SNAKE_CASE__    :	Dict=True					,							SCREAMING_SNAKE_CASE__    :	Union[str, Any]="[UNK]"					,							SCREAMING_SNAKE_CASE__    :	Optional[int]="[SEP]"					,							SCREAMING_SNAKE_CASE__    :	int="[PAD]"					,							SCREAMING_SNAKE_CASE__    :	Union[str, Any]="[CLS]"					,							SCREAMING_SNAKE_CASE__    :	Union[str, Any]="[MASK]"					,							SCREAMING_SNAKE_CASE__    :	str=True					,							SCREAMING_SNAKE_CASE__    :	Any=None					,							**SCREAMING_SNAKE_CASE__    :	Tuple					,							)		->      List[Any]:
             super().__init__(
                 SCREAMING_SNAKE_CASE__					,							tokenizer_file=SCREAMING_SNAKE_CASE__					,							do_lower_case=SCREAMING_SNAKE_CASE__					,							unk_token=SCREAMING_SNAKE_CASE__					,							sep_token=SCREAMING_SNAKE_CASE__					,							pad_token=SCREAMING_SNAKE_CASE__					,							cls_token=SCREAMING_SNAKE_CASE__					,							mask_token=SCREAMING_SNAKE_CASE__					,							tokenize_chinese_chars=SCREAMING_SNAKE_CASE__					,							strip_accents=SCREAMING_SNAKE_CASE__					,							**SCREAMING_SNAKE_CASE__					,							)

             a_ :		List[str]							=  json.loads(self.backend_tokenizer.normalizer.__getstate__()					)
             if (
                 normalizer_state.get('lowercase'					,							SCREAMING_SNAKE_CASE__					) != do_lower_case
                 or normalizer_state.get('strip_accents'					,							SCREAMING_SNAKE_CASE__					) != strip_accents
                 or normalizer_state.get('handle_chinese_chars'					,							SCREAMING_SNAKE_CASE__					) != tokenize_chinese_chars
             ):
                    a_ :		Dict							=  getattr(SCREAMING_SNAKE_CASE__					,							normalizer_state.pop('type'					)					)
                    a_ :		List[Any]							=  do_lower_case
                    a_ :		int							=  strip_accents
                    a_ :		Tuple							=  tokenize_chinese_chars
                    a_ :		List[str]							=  normalizer_class(**SCREAMING_SNAKE_CASE__					)

             a_ :		Dict							=  do_lower_case
      def 	SCREAMING_SNAKE_CASE							(	self    :	Optional[int]					,							SCREAMING_SNAKE_CASE__    :	Union[str, Any]					,							SCREAMING_SNAKE_CASE__    :	Optional[Any]=None					)		->      List[Any]:
             a_ :		List[str]							=  [self.cls_token_id] + token_ids_a + [self.sep_token_id]

             if token_ids_a:
                    output += token_ids_a + [self.sep_token_id]

             return output
      def 	SCREAMING_SNAKE_CASE							(	self    :	Dict					,							SCREAMING_SNAKE_CASE__    :	List[int]					,							SCREAMING_SNAKE_CASE__    :	Optional[List[int]] = None					)		->      List[int]:
             a_ :		Tuple							=  [self.sep_token_id]
             a_ :		Dict							=  [self.cls_token_id]
             if token_ids_a is None:
                    return len(cls + token_ids_a + sep					) * [0]
             return len(cls + token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1]

      def 	SCREAMING_SNAKE_CASE							(	self    :	Union[str, Any]					,							SCREAMING_SNAKE_CASE__    :	str					,							SCREAMING_SNAKE_CASE__    :	Optional[str] = None					)		->      Tuple[str]:
             a_ :		int							=  self._tokenizer.model.save(SCREAMING_SNAKE_CASE__					,							name=SCREAMING_SNAKE_CASE__					)
             return tuple(SCREAMING_SNAKE_CASE__					)



from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


UpperCAmelCase_				:			Any   =	{
    'configuration_mobilebert': [
        'MOBILEBERT_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'MobileBertConfig',
        'MobileBertOnnxConfig',
    ],
    'tokenization_mobilebert': ['MobileBertTokenizer'],
}

try:
   if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCAmelCase_				:			Union[str, Any]   =	['MobileBertTokenizerFast']

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCAmelCase_				:			List[str]   =	[
       'MOBILEBERT_PRETRAINED_MODEL_ARCHIVE_LIST',
       'MobileBertForMaskedLM',
       'MobileBertForMultipleChoice',
       'MobileBertForNextSentencePrediction',
       'MobileBertForPreTraining',
       'MobileBertForQuestionAnswering',
       'MobileBertForSequenceClassification',
       'MobileBertForTokenClassification',
       'MobileBertLayer',
       'MobileBertModel',
       'MobileBertPreTrainedModel',
       'load_tf_weights_in_mobilebert',
   ]

try:
   if not is_tf_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCAmelCase_				:			Tuple   =	[
       'TF_MOBILEBERT_PRETRAINED_MODEL_ARCHIVE_LIST',
       'TFMobileBertForMaskedLM',
       'TFMobileBertForMultipleChoice',
       'TFMobileBertForNextSentencePrediction',
       'TFMobileBertForPreTraining',
       'TFMobileBertForQuestionAnswering',
       'TFMobileBertForSequenceClassification',
       'TFMobileBertForTokenClassification',
       'TFMobileBertMainLayer',
       'TFMobileBertModel',
       'TFMobileBertPreTrainedModel',
   ]


if TYPE_CHECKING:
   from .configuration_mobilebert import (
       MOBILEBERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
       MobileBertConfig,
       MobileBertOnnxConfig,
   )
   from .tokenization_mobilebert import MobileBertTokenizer

   try:
      if not is_tokenizers_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .tokenization_mobilebert_fast import MobileBertTokenizerFast

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_mobilebert import (
          MOBILEBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
          MobileBertForMaskedLM,
          MobileBertForMultipleChoice,
          MobileBertForNextSentencePrediction,
          MobileBertForPreTraining,
          MobileBertForQuestionAnswering,
          MobileBertForSequenceClassification,
          MobileBertForTokenClassification,
          MobileBertLayer,
          MobileBertModel,
          MobileBertPreTrainedModel,
          load_tf_weights_in_mobilebert,
      )

   try:
      if not is_tf_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_tf_mobilebert import (
          TF_MOBILEBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
          TFMobileBertForMaskedLM,
          TFMobileBertForMultipleChoice,
          TFMobileBertForNextSentencePrediction,
          TFMobileBertForPreTraining,
          TFMobileBertForQuestionAnswering,
          TFMobileBertForSequenceClassification,
          TFMobileBertForTokenClassification,
          TFMobileBertMainLayer,
          TFMobileBertModel,
          TFMobileBertPreTrainedModel,
      )

else:
   import sys

   UpperCAmelCase_				:			Tuple   =	_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)


