


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


A	:		List[str]    =       logging.get_logger(__name__)

A	:		Any    =       {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

A	:		Tuple    =       {
    'vocab_file': {
        'distilbert-base-uncased': 'https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt',
        'distilbert-base-uncased-distilled-squad': (
            'https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt'
        ),
        'distilbert-base-cased': 'https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt',
        'distilbert-base-cased-distilled-squad': (
            'https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt'
        ),
        'distilbert-base-german-cased': 'https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt',
        'distilbert-base-multilingual-cased': (
            'https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'distilbert-base-uncased': 'https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json',
        'distilbert-base-uncased-distilled-squad': (
            'https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json'
        ),
        'distilbert-base-cased': 'https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json',
        'distilbert-base-cased-distilled-squad': (
            'https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json'
        ),
        'distilbert-base-german-cased': (
            'https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json'
        ),
        'distilbert-base-multilingual-cased': (
            'https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json'
        ),
    },
}

A	:		Optional[Any]    =       {
    'distilbert-base-uncased': 5_1_2,
    'distilbert-base-uncased-distilled-squad': 5_1_2,
    'distilbert-base-cased': 5_1_2,
    'distilbert-base-cased-distilled-squad': 5_1_2,
    'distilbert-base-german-cased': 5_1_2,
    'distilbert-base-multilingual-cased': 5_1_2,
}


A	:		str    =       {
    'distilbert-base-uncased': {'do_lower_case': True},
    'distilbert-base-uncased-distilled-squad': {'do_lower_case': True},
    'distilbert-base-cased': {'do_lower_case': False},
    'distilbert-base-cased-distilled-squad': {'do_lower_case': False},
    'distilbert-base-german-cased': {'do_lower_case': False},
    'distilbert-base-multilingual-cased': {'do_lower_case': False},
}



class 							A   (	UpperCAmelCase__							):
					'''simple docstring'''






					A__      =			VOCAB_FILES_NAMES
					A__      =			PRETRAINED_VOCAB_FILES_MAP
					A__      =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					A__      =			PRETRAINED_INIT_CONFIGURATION
					A__      =			['''input_ids''', '''attention_mask''']
					A__      =			DistilBertTokenizer

					def __init__(self   :						Optional[int]				, _UpperCAmelCase   :						Union[str, Any]=None				, _UpperCAmelCase   :						Optional[int]=None				, _UpperCAmelCase   :						int=True				, _UpperCAmelCase   :						Optional[int]="[UNK]"				, _UpperCAmelCase   :						Tuple="[SEP]"				, _UpperCAmelCase   :						List[Any]="[PAD]"				, _UpperCAmelCase   :						Union[str, Any]="[CLS]"				, _UpperCAmelCase   :						str="[MASK]"				, _UpperCAmelCase   :						Tuple=True				, _UpperCAmelCase   :						Dict=None				, **_UpperCAmelCase   :						Optional[int]				, )						->       Union[str, Any]:




							"""simple docstring"""


							super().__init__(
							    _UpperCAmelCase				, tokenizer_file=_UpperCAmelCase				, do_lower_case=_UpperCAmelCase				, unk_token=_UpperCAmelCase				, sep_token=_UpperCAmelCase				, pad_token=_UpperCAmelCase				, cls_token=_UpperCAmelCase				, mask_token=_UpperCAmelCase				, tokenize_chinese_chars=_UpperCAmelCase				, strip_accents=_UpperCAmelCase				, **_UpperCAmelCase				, )

							lowercase__					      =		json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
							if (
							    normalizer_state.get("""lowercase"""				, _UpperCAmelCase		) != do_lower_case
							    or normalizer_state.get("""strip_accents"""				, _UpperCAmelCase		) != strip_accents
							    or normalizer_state.get("""handle_chinese_chars"""				, _UpperCAmelCase		) != tokenize_chinese_chars
							):
									lowercase__					      =		getattr(_UpperCAmelCase				, normalizer_state.pop("""type"""		)		)
									lowercase__					      =		do_lower_case
									lowercase__					      =		strip_accents
									lowercase__					      =		tokenize_chinese_chars
									lowercase__					      =		normalizer_class(**_UpperCAmelCase		)

							lowercase__					      =		do_lower_case

					def   lowerCamelCase__					(self   :						Any				, _UpperCAmelCase   :						str				, _UpperCAmelCase   :						int=None		)						->       Dict:




							"""simple docstring"""


							lowercase__					      =		[self.cls_token_id] + token_ids_a + [self.sep_token_id]

							if token_ids_a:
									output += token_ids_a + [self.sep_token_id]

							return output

					def   lowerCamelCase__					(self   :						Any				, _UpperCAmelCase   :						List[int]				, _UpperCAmelCase   :						Optional[List[int]] = None		)						->       List[int]:




							"""simple docstring"""


							lowercase__					      =		[self.sep_token_id]
							lowercase__					      =		[self.cls_token_id]
							if token_ids_a is None:
									return len(cls + token_ids_a + sep		) * [0]
							return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]


					def   lowerCamelCase__					(self   :						int				, _UpperCAmelCase   :						str				, _UpperCAmelCase   :						Optional[str] = None		)						->       Tuple[str]:




							"""simple docstring"""


							lowercase__					      =		self._tokenizer.model.save(_UpperCAmelCase				, name=_UpperCAmelCase		)
							return tuple(_UpperCAmelCase		)





class 							A   :
					'''simple docstring'''






					def __init__(self   :						int				, _UpperCAmelCase   :						list		)						->       None:




							"""simple docstring"""


							lowercase__					      =		set_counts
							lowercase__					      =		max(_UpperCAmelCase		)
							lowercase__					      =		len(_UpperCAmelCase		)
							lowercase__					      =		[1] * num_sets
							lowercase__					      =		list(range(_UpperCAmelCase		)		)

					def   lowerCamelCase__					(self   :						List[Any]				, _UpperCAmelCase   :						int				, _UpperCAmelCase   :						int		)						->       bool:




							"""simple docstring"""


							lowercase__					      =		self.get_parent(_UpperCAmelCase		)
							lowercase__					      =		self.get_parent(_UpperCAmelCase		)

							if src_parent == dst_parent:
									return False

							if self.ranks[dst_parent] >= self.ranks[src_parent]:
									self.set_counts[dst_parent] += self.set_counts[src_parent]
									lowercase__					      =		0
									lowercase__					      =		dst_parent
									if self.ranks[dst_parent] == self.ranks[src_parent]:
											self.ranks[dst_parent] += 1
									lowercase__					      =		self.set_counts[dst_parent]
							else:
									self.set_counts[src_parent] += self.set_counts[dst_parent]
									lowercase__					      =		0
									lowercase__					      =		src_parent
									lowercase__					      =		self.set_counts[src_parent]

							lowercase__					      =		max(self.max_set				, _UpperCAmelCase		)
							return True


					def   lowerCamelCase__					(self   :						List[Any]				, _UpperCAmelCase   :						int		)						->       int:




							"""simple docstring"""


							if self.parents[disj_set] == disj_set:
									return disj_set
							lowercase__					      =		self.get_parent(self.parents[disj_set]		)
							return self.parents[disj_set]

