






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ....tokenization_utils_fast import PreTrainedTokenizerFast
from ....utils import logging
from .tokenization_retribert import RetriBertTokenizer


UpperCamelCase			    =     logging.get_logger(__name__)

UpperCamelCase			    =     {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

UpperCamelCase			    =     {
    '''vocab_file''': {
        '''yjernite/retribert-base-uncased''': (
            '''https://huggingface.co/yjernite/retribert-base-uncased/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''yjernite/retribert-base-uncased''': (
            '''https://huggingface.co/yjernite/retribert-base-uncased/resolve/main/tokenizer.json'''
        ),
    },
}

UpperCamelCase			    =     {
    '''yjernite/retribert-base-uncased''': 512,
}


UpperCamelCase			    =     {
    '''yjernite/retribert-base-uncased''': {'''do_lower_case''': True},
}



class 				__UpperCAmelCase		(_UpperCAmelCase   ):
			__snake_case    :				Any									=							VOCAB_FILES_NAMES
			__snake_case    :				List[Any]									=							PRETRAINED_VOCAB_FILES_MAP
			__snake_case    :				List[str]									=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			__snake_case    :				List[str]									=							PRETRAINED_INIT_CONFIGURATION
			__snake_case    :				Optional[int]									=							RetriBertTokenizer
			__snake_case    :				str									=							["input_ids", "attention_mask"]







			def __init__(     self:  Dict				,     UpperCAmelCase_:  Optional[int]=None				,     UpperCAmelCase_:  str=None				,     UpperCAmelCase_:  List[str]=True				,     UpperCAmelCase_:  str="[UNK]"				,     UpperCAmelCase_:  str="[SEP]"				,     UpperCAmelCase_:  Tuple="[PAD]"				,     UpperCAmelCase_:  Union[str, Any]="[CLS]"				,     UpperCAmelCase_:  List[Any]="[MASK]"				,     UpperCAmelCase_:  List[Any]=True				,     UpperCAmelCase_:  Optional[int]=None				,     **UpperCAmelCase_:  Any				,     ):







									'''simple docstring'''




									super().__init__(
									    UpperCAmelCase_				,     tokenizer_file=UpperCAmelCase_				,     do_lower_case=UpperCAmelCase_				,     unk_token=UpperCAmelCase_				,     sep_token=UpperCAmelCase_				,     pad_token=UpperCAmelCase_				,     cls_token=UpperCAmelCase_				,     mask_token=UpperCAmelCase_				,     tokenize_chinese_chars=UpperCAmelCase_				,     strip_accents=UpperCAmelCase_				,     **UpperCAmelCase_				,     )

									_SCREAMING_SNAKE_CASE     				=  json.loads(self.backend_tokenizer.normalizer.__getstate__()					)
									if (
									    normalizer_state.get("""lowercase"""				,     UpperCAmelCase_					) != do_lower_case
									    or normalizer_state.get("""strip_accents"""				,     UpperCAmelCase_					) != strip_accents
									    or normalizer_state.get("""handle_chinese_chars"""				,     UpperCAmelCase_					) != tokenize_chinese_chars
									):
															_SCREAMING_SNAKE_CASE     				=  getattr(UpperCAmelCase_				,     normalizer_state.pop("""type"""					)					)
															_SCREAMING_SNAKE_CASE     				=  do_lower_case
															_SCREAMING_SNAKE_CASE     				=  strip_accents
															_SCREAMING_SNAKE_CASE     				=  tokenize_chinese_chars
															_SCREAMING_SNAKE_CASE     				=  normalizer_class(**UpperCAmelCase_					)

									_SCREAMING_SNAKE_CASE     				=  do_lower_case







			def 							UpperCamelCase				(     self:  Union[str, Any]				,     UpperCAmelCase_:  Any				,     UpperCAmelCase_:  str=None					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  [self.cls_token_id] + token_ids_a + [self.sep_token_id]

									if token_ids_a:
															output += token_ids_a + [self.sep_token_id]

									return output







			def 							UpperCamelCase				(     self:  Optional[Any]				,     UpperCAmelCase_:  List[int]				,     UpperCAmelCase_:  Optional[List[int]] = None					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  [self.sep_token_id]
									_SCREAMING_SNAKE_CASE     				=  [self.cls_token_id]
									if token_ids_a is None:
															return len(cls + token_ids_a + sep					) * [0]
									return len(cls + token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1]







			def 							UpperCamelCase				(     self:  Any				,     UpperCAmelCase_:  str				,     UpperCAmelCase_:  Optional[str] = None					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  self._tokenizer.model.save(UpperCAmelCase_				,     name=UpperCAmelCase_					)
									return tuple(UpperCAmelCase_					)









from typing import TYPE_CHECKING

from ....utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


UpperCamelCase			    =     {'''configuration_mmbt''': ['''MMBTConfig''']}

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		UpperCamelCase			    =     ['''MMBTForClassification''', '''MMBTModel''', '''ModalEmbeddings''']


if TYPE_CHECKING:
		from .configuration_mmbt import MMBTConfig

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_mmbt import MMBTForClassification, MMBTModel, ModalEmbeddings

else:
		import sys

		UpperCamelCase			    =     _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)

