





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ....tokenization_utils_fast import PreTrainedTokenizerFast
from ....utils import logging
from .tokenization_retribert import RetriBertTokenizer


_SCREAMING_SNAKE_CASE 		=						logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE 		=						{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

_SCREAMING_SNAKE_CASE 		=						{
    """vocab_file""": {
        """yjernite/retribert-base-uncased""": (
            """https://huggingface.co/yjernite/retribert-base-uncased/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """yjernite/retribert-base-uncased""": (
            """https://huggingface.co/yjernite/retribert-base-uncased/resolve/main/tokenizer.json"""
        ),
    },
}

_SCREAMING_SNAKE_CASE 		=						{
    """yjernite/retribert-base-uncased""": 5_12,
}


_SCREAMING_SNAKE_CASE 		=						{
    """yjernite/retribert-base-uncased""": {"""do_lower_case""": True},
}
class SCREAMING_SNAKE_CASE_  (   snake_case_   ):
						__magic_name__:     str	 =    VOCAB_FILES_NAMES
						__magic_name__:     Optional[int]	 =    PRETRAINED_VOCAB_FILES_MAP
						__magic_name__:     str	 =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						__magic_name__:     str	 =    PRETRAINED_INIT_CONFIGURATION
						__magic_name__:     List[Any]	 =    RetriBertTokenizer
						__magic_name__:     Optional[Any]	 =    ["input_ids", "attention_mask"]






						def __init__( self : Any						,				_A : Any=None						,				_A : Tuple=None						,				_A : int=True						,				_A : Optional[int]="[UNK]"						,				_A : List[Any]="[SEP]"						,				_A : Optional[Any]="[PAD]"						,				_A : List[str]="[CLS]"						,				_A : Dict="[MASK]"						,				_A : int=True						,				_A : Dict=None						,				**_A : List[Any]						,				)							->      Optional[int]:







							"""simple docstring"""


							super().__init__(
							    _A						,				tokenizer_file=_A						,				do_lower_case=_A						,				unk_token=_A						,				sep_token=_A						,				pad_token=_A						,				cls_token=_A						,				mask_token=_A						,				tokenize_chinese_chars=_A						,				strip_accents=_A						,				**_A						,				)

							snake_case_						:     List[str]     =		json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
							if (
							    normalizer_state.get('lowercase'						,				_A		) != do_lower_case
							    or normalizer_state.get('strip_accents'						,				_A		) != strip_accents
							    or normalizer_state.get('handle_chinese_chars'						,				_A		) != tokenize_chinese_chars
							):
								snake_case_						:     Any     =		getattr(_A						,				normalizer_state.pop('type'		)		)
								snake_case_						:     List[str]     =		do_lower_case
								snake_case_						:     Optional[Any]     =		strip_accents
								snake_case_						:     Optional[Any]     =		tokenize_chinese_chars
								snake_case_						:     List[Any]     =		normalizer_class(**_A		)

							snake_case_						:     Tuple     =		do_lower_case






						def UpperCAmelCase_			( self : Optional[int]						,				_A : Any						,				_A : List[Any]=None		)							->      Union[str, Any]:







							"""simple docstring"""


							snake_case_						:     Tuple     =		[self.cls_token_id] + token_ids_a + [self.sep_token_id]

							if token_ids_a:
								output += token_ids_a + [self.sep_token_id]

							return output






						def UpperCAmelCase_			( self : int						,				_A : List[int]						,				_A : Optional[List[int]] = None		)							->      List[int]:







							"""simple docstring"""


							snake_case_						:     Dict     =		[self.sep_token_id]
							snake_case_						:     int     =		[self.cls_token_id]
							if token_ids_a is None:
								return len(cls + token_ids_a + sep		) * [0]
							return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]






						def UpperCAmelCase_			( self : List[Any]						,				_A : str						,				_A : Optional[str] = None		)							->      Tuple[str]:







							"""simple docstring"""


							snake_case_						:     Any     =		self._tokenizer.model.save(_A						,				name=_A		)
							return tuple(_A		)








from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


_SCREAMING_SNAKE_CASE 		=						{
    """configuration_xlm""": ["""XLM_PRETRAINED_CONFIG_ARCHIVE_MAP""", """XLMConfig""", """XLMOnnxConfig"""],
    """tokenization_xlm""": ["""XLMTokenizer"""],
}

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						_SCREAMING_SNAKE_CASE 		=						[
						    """XLM_PRETRAINED_MODEL_ARCHIVE_LIST""",
						    """XLMForMultipleChoice""",
						    """XLMForQuestionAnswering""",
						    """XLMForQuestionAnsweringSimple""",
						    """XLMForSequenceClassification""",
						    """XLMForTokenClassification""",
						    """XLMModel""",
						    """XLMPreTrainedModel""",
						    """XLMWithLMHeadModel""",
						]

try:
						if not is_tf_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						_SCREAMING_SNAKE_CASE 		=						[
						    """TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST""",
						    """TFXLMForMultipleChoice""",
						    """TFXLMForQuestionAnsweringSimple""",
						    """TFXLMForSequenceClassification""",
						    """TFXLMForTokenClassification""",
						    """TFXLMMainLayer""",
						    """TFXLMModel""",
						    """TFXLMPreTrainedModel""",
						    """TFXLMWithLMHeadModel""",
						]


if TYPE_CHECKING:
						from .configuration_xlm import XLM_PRETRAINED_CONFIG_ARCHIVE_MAP, XLMConfig, XLMOnnxConfig
						from .tokenization_xlm import XLMTokenizer

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_xlm import (
												    XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
												    XLMForMultipleChoice,
												    XLMForQuestionAnswering,
												    XLMForQuestionAnsweringSimple,
												    XLMForSequenceClassification,
												    XLMForTokenClassification,
												    XLMModel,
												    XLMPreTrainedModel,
												    XLMWithLMHeadModel,
												)

						try:
												if not is_tf_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_tf_xlm import (
												    TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
												    TFXLMForMultipleChoice,
												    TFXLMForQuestionAnsweringSimple,
												    TFXLMForSequenceClassification,
												    TFXLMForTokenClassification,
												    TFXLMMainLayer,
												    TFXLMModel,
												    TFXLMPreTrainedModel,
												    TFXLMWithLMHeadModel,
												)

else:
						import sys

						_SCREAMING_SNAKE_CASE 		=						_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)

