


'''simple docstring'''



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_squeezebert import SqueezeBertTokenizer


_lowercase      :      Dict            =					logging.get_logger(__name__)

_lowercase      :      Optional[Any]            =					{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

_lowercase      :      List[str]            =					{
    """vocab_file""": {
        """squeezebert/squeezebert-uncased""": (
            """https://huggingface.co/squeezebert/squeezebert-uncased/resolve/main/vocab.txt"""
        ),
        """squeezebert/squeezebert-mnli""": """https://huggingface.co/squeezebert/squeezebert-mnli/resolve/main/vocab.txt""",
        """squeezebert/squeezebert-mnli-headless""": (
            """https://huggingface.co/squeezebert/squeezebert-mnli-headless/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """squeezebert/squeezebert-uncased""": (
            """https://huggingface.co/squeezebert/squeezebert-uncased/resolve/main/tokenizer.json"""
        ),
        """squeezebert/squeezebert-mnli""": (
            """https://huggingface.co/squeezebert/squeezebert-mnli/resolve/main/tokenizer.json"""
        ),
        """squeezebert/squeezebert-mnli-headless""": (
            """https://huggingface.co/squeezebert/squeezebert-mnli-headless/resolve/main/tokenizer.json"""
        ),
    },
}

_lowercase      :      Optional[Any]            =					{
    """squeezebert/squeezebert-uncased""": 512,
    """squeezebert/squeezebert-mnli""": 512,
    """squeezebert/squeezebert-mnli-headless""": 512,
}


_lowercase      :      Optional[int]            =					{
    """squeezebert/squeezebert-uncased""": {"""do_lower_case""": True},
    """squeezebert/squeezebert-mnli""": {"""do_lower_case""": True},
    """squeezebert/squeezebert-mnli-headless""": {"""do_lower_case""": True},
}






class 	UpperCamelCase__(       lowerCAmelCase	):
				__magic_name__    :				Tuple  		=	VOCAB_FILES_NAMES
				__magic_name__    :				Optional[Any]  		=	PRETRAINED_VOCAB_FILES_MAP
				__magic_name__    :				Tuple  		=	PRETRAINED_INIT_CONFIGURATION
				__magic_name__    :				str  		=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__magic_name__    :				str  		=	SqueezeBertTokenizer




				def __init__(		self      :			List[Any]				,   lowerCAmelCase      :			Tuple=None				,   lowerCAmelCase      :			int=None				,   lowerCAmelCase      :			List[str]=True				,   lowerCAmelCase      :			Any="[UNK]"				,   lowerCAmelCase      :			Any="[SEP]"				,   lowerCAmelCase      :			Any="[PAD]"				,   lowerCAmelCase      :			Any="[CLS]"				,   lowerCAmelCase      :			List[Any]="[MASK]"				,   lowerCAmelCase      :			int=True				,   lowerCAmelCase      :			Any=None				,   **lowerCAmelCase      :			Optional[int]				,   )->					Optional[int]:

										"""simple docstring"""




										super().__init__(
										    lowerCAmelCase				,   tokenizer_file=lowerCAmelCase				,   do_lower_case=lowerCAmelCase				,   unk_token=lowerCAmelCase				,   sep_token=lowerCAmelCase				,   pad_token=lowerCAmelCase				,   cls_token=lowerCAmelCase				,   mask_token=lowerCAmelCase				,   tokenize_chinese_chars=lowerCAmelCase				,   strip_accents=lowerCAmelCase				,   **lowerCAmelCase				,   )

										UpperCAmelCase					    =     json.loads(self.backend_tokenizer.normalizer.__getstate__()				)
										if (
										    normalizer_state.get('''lowercase'''				,   lowerCAmelCase				) != do_lower_case
										    or normalizer_state.get('''strip_accents'''				,   lowerCAmelCase				) != strip_accents
										    or normalizer_state.get('''handle_chinese_chars'''				,   lowerCAmelCase				) != tokenize_chinese_chars
										):
																UpperCAmelCase					    =     getattr(lowerCAmelCase				,   normalizer_state.pop('''type'''				)				)
																UpperCAmelCase					    =     do_lower_case
																UpperCAmelCase					    =     strip_accents
																UpperCAmelCase					    =     tokenize_chinese_chars
																UpperCAmelCase					    =     normalizer_class(**lowerCAmelCase				)

										UpperCAmelCase					    =     do_lower_case




				def 					a__(		self      :			Tuple				,   lowerCAmelCase      :			List[Any]				,   lowerCAmelCase      :			Optional[int]=None				)->					str:

										"""simple docstring"""




										UpperCAmelCase					    =     [self.cls_token_id] + token_ids_a + [self.sep_token_id]

										if token_ids_a:
																output += token_ids_a + [self.sep_token_id]

										return output




				def 					a__(		self      :			Optional[int]				,   lowerCAmelCase      :			List[int]				,   lowerCAmelCase      :			Optional[List[int]] = None				)->					List[int]:

										"""simple docstring"""




										UpperCAmelCase					    =     [self.sep_token_id]
										UpperCAmelCase					    =     [self.cls_token_id]
										if token_ids_a is None:
																return len(cls + token_ids_a + sep				) * [0]
										return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]





				def 					a__(		self      :			List[str]				,   lowerCAmelCase      :			str				,   lowerCAmelCase      :			Optional[str] = None				)->					Tuple[str]:

										"""simple docstring"""




										UpperCAmelCase					    =     self._tokenizer.model.save(lowerCAmelCase				,   name=lowerCAmelCase				)
										return tuple(lowerCAmelCase				)









'''simple docstring'''



from .data_collator import (
    DataCollatorForLanguageModeling,
    DataCollatorForPermutationLanguageModeling,
    DataCollatorForSeqaSeq,
    DataCollatorForSOP,
    DataCollatorForTokenClassification,
    DataCollatorForWholeWordMask,
    DataCollatorWithPadding,
    DefaultDataCollator,
    default_data_collator,
)
from .metrics import glue_compute_metrics, xnli_compute_metrics
from .processors import (
    DataProcessor,
    InputExample,
    InputFeatures,
    SingleSentenceClassificationProcessor,
    SquadExample,
    SquadFeatures,
    SquadVaProcessor,
    SquadVaProcessor,
    glue_convert_examples_to_features,
    glue_output_modes,
    glue_processors,
    glue_tasks_num_labels,
    squad_convert_examples_to_features,
    xnli_output_modes,
    xnli_processors,
    xnli_tasks_num_labels,
)





