
import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


_A   =			logging.get_logger(__name__)

_A   =			{"vocab_file": "spiece.model"}

_A   =			{
    "vocab_file": {
        "xlnet-base-cased": "https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model",
        "xlnet-large-cased": "https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model",
    }
}

_A   =			{
    "xlnet-base-cased": None,
    "xlnet-large-cased": None,
}

# Segments (not really needed)
_A   =			0
_A   =			1
_A   =			2
_A   =			3
_A   =			4







class 					lowerCamelCase		(  A_  ):
     UpperCAmelCase__    :							Any									=		VOCAB_FILES_NAMES
     UpperCAmelCase__    :							Optional[int]									=		PRETRAINED_VOCAB_FILES_MAP
     UpperCAmelCase__    :							Optional[Any]									=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     UpperCAmelCase__    :							Optional[Any]									=		"left"





     def __init__(self				:   str    ,   _A				:   Tuple    ,   _A				:   str=False    ,   _A				:   Optional[Any]=True    ,   _A				:   Any=False    ,   _A				:   Dict="<s>"    ,   _A				:   List[str]="</s>"    ,   _A				:   Union[str, Any]="<unk>"    ,   _A				:   int="<sep>"    ,   _A				:   Union[str, Any]="<pad>"    ,   _A				:   Optional[Any]="<cls>"    ,   _A				:   Optional[Any]="<mask>"    ,   _A				:   Any=["<eop>", "<eod>"]    ,   _A				:   Optional[Dict[str, Any]] = None    ,   **_A				:   Optional[int]    ,   )						->						None:
         # Mask token behave like a normal word, i.e. include the space before it
         snake_case								= AddedToken(_A    ,   lstrip=_A    ,   rstrip=_A       ) if isinstance(_A    ,   _A       ) else mask_token

         snake_case								= {} if sp_model_kwargs is None else sp_model_kwargs

         super().__init__(
             do_lower_case=_A    ,   remove_space=_A    ,   keep_accents=_A    ,   bos_token=_A    ,   eos_token=_A    ,   unk_token=_A    ,   sep_token=_A    ,   pad_token=_A    ,   cls_token=_A    ,   mask_token=_A    ,   additional_special_tokens=_A    ,   sp_model_kwargs=self.sp_model_kwargs    ,   **_A    ,   )

         snake_case								= 3

         snake_case								= do_lower_case
         snake_case								= remove_space
         snake_case								= keep_accents
         snake_case								= vocab_file

         snake_case								= spm.SentencePieceProcessor(**self.sp_model_kwargs       )
         self.sp_model.Load(_A       )





     @property
     def 		UpperCAmelCase(self				:   Any       )						->						Optional[Any]:
         return len(self.sp_model       )





     def 		UpperCAmelCase(self				:   Optional[int]       )						->						Dict:
         snake_case								= {self.convert_ids_to_tokens(_A       ): i for i in range(self.vocab_size       )}
         vocab.update(self.added_tokens_encoder       )
         return vocab





     def __getstate__(self				:   str       )						->						Optional[Any]:
         snake_case								= self.__dict__.copy()
         snake_case								= None
         return state





     def __setstate__(self				:   List[str]    ,   _A				:   Any       )						->						Optional[Any]:
         snake_case								= d

         # for backward compatibility
         if not hasattr(self    ,   "sp_model_kwargs"       ):
             snake_case								= {}

         snake_case								= spm.SentencePieceProcessor(**self.sp_model_kwargs       )
         self.sp_model.Load(self.vocab_file       )





     def 		UpperCAmelCase(self				:   Any    ,   _A				:   Optional[int]       )						->						List[Any]:
         if self.remove_space:
             snake_case								= " ".join(inputs.strip().split()       )
         else:
             snake_case								= inputs
         snake_case								= outputs.replace("``"    ,   "\""       ).replace("''"    ,   "\""       )

         if not self.keep_accents:
             snake_case								= unicodedata.normalize("NFKD"    ,   _A       )
             snake_case								= "".join([c for c in outputs if not unicodedata.combining(_A       )]       )
         if self.do_lower_case:
             snake_case								= outputs.lower()

         return outputs





     def 		UpperCAmelCase(self				:   Dict    ,   _A				:   str       )						->						List[str]:
         snake_case								= self.preprocess_text(_A       )
         snake_case								= self.sp_model.encode(_A    ,   out_type=_A       )
         snake_case								= []
         for piece in pieces:
             if len(_A       ) > 1 and piece[-1] == str(","       ) and piece[-2].isdigit():
                 snake_case								= self.sp_model.EncodeAsPieces(piece[:-1].replace(_A    ,   ""       )       )
                 if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                     if len(cur_pieces[0]       ) == 1:
                         snake_case								= cur_pieces[1:]
                     else:
                         snake_case								= cur_pieces[0][1:]
                 cur_pieces.append(piece[-1]       )
                 new_pieces.extend(_A       )
             else:
                 new_pieces.append(_A       )

         return new_pieces





     def 		UpperCAmelCase(self				:   int    ,   _A				:   List[str]       )						->						Any:
         return self.sp_model.PieceToId(_A       )





     def 		UpperCAmelCase(self				:   List[str]    ,   _A				:   Union[str, Any]       )						->						List[str]:
         return self.sp_model.IdToPiece(_A       )





     def 		UpperCAmelCase(self				:   List[Any]    ,   _A				:   List[Any]       )						->						List[Any]:
         snake_case								= "".join(_A       ).replace(_A    ,   " "       ).strip()
         return out_string





     def 		UpperCAmelCase(self				:   Tuple    ,   _A				:   List[int]    ,   _A				:   bool = False    ,   _A				:   bool = None    ,   _A				:   bool = True    ,   **_A				:   int    ,   )						->						str:
         snake_case								= kwargs.pop("use_source_tokenizer"    ,   _A       )

         snake_case								= self.convert_ids_to_tokens(_A    ,   skip_special_tokens=_A       )

         # To avoid mixing byte-level and unicode for byte-level BPT
         # we need to build string separately for added tokens and byte-level tokens
         # cf. https://github.com/huggingface/transformers/issues/1133
         snake_case								= []
         snake_case								= []
         for token in filtered_tokens:
             if skip_special_tokens and token in self.all_special_ids:
                 continue
             if token in self.added_tokens_encoder:
                 if current_sub_text:
                     sub_texts.append(self.convert_tokens_to_string(_A       )       )
                     snake_case								= []
                 sub_texts.append(_A       )
             else:
                 current_sub_text.append(_A       )
         if current_sub_text:
             sub_texts.append(self.convert_tokens_to_string(_A       )       )

         # Mimic the behavior of the Rust tokenizer:
         # By default, there are no spaces between special tokens
         snake_case								= "".join(_A       )

         snake_case								= (
             clean_up_tokenization_spaces
             if clean_up_tokenization_spaces is not None
             else self.clean_up_tokenization_spaces
         )
         if clean_up_tokenization_spaces:
             snake_case								= self.clean_up_tokenization(_A       )
             return clean_text
         else:
             return text





     def 		UpperCAmelCase(self				:   str    ,   _A				:   List[int]    ,   _A				:   Optional[List[int]] = None       )						->						List[int]:
         snake_case								= [self.sep_token_id]
         snake_case								= [self.cls_token_id]
         if token_ids_a is None:
             return token_ids_a + sep + cls
         return token_ids_a + sep + token_ids_a + sep + cls





     def 		UpperCAmelCase(self				:   Optional[Any]    ,   _A				:   List[int]    ,   _A				:   Optional[List[int]] = None    ,   _A				:   bool = False       )						->						List[int]:

         if already_has_special_tokens:
             return super().get_special_tokens_mask(
                 token_ids_a=_A    ,   token_ids_a=_A    ,   already_has_special_tokens=_A       )

         if token_ids_a is not None:
             return ([0] * len(_A       )) + [1] + ([0] * len(_A       )) + [1, 1]
         return ([0] * len(_A       )) + [1, 1]





     def 		UpperCAmelCase(self				:   List[str]    ,   _A				:   List[int]    ,   _A				:   Optional[List[int]] = None       )						->						List[int]:
         snake_case								= [self.sep_token_id]
         snake_case								= [2]

         if token_ids_a is None:
             return len(token_ids_a + sep       ) * [0] + cls_segment_id
         return len(token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1] + cls_segment_id





     def 		UpperCAmelCase(self				:   List[str]    ,   _A				:   str    ,   _A				:   Optional[str] = None       )						->						Tuple[str]:
         if not os.path.isdir(_A       ):
             logger.error(f'Vocabulary path ({save_directory}) should be a directory'       )
             return
         snake_case								= os.path.join(
             _A    ,   (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]       )

         if os.path.abspath(self.vocab_file       ) != os.path.abspath(_A       ) and os.path.isfile(self.vocab_file       ):
             copyfile(self.vocab_file    ,   _A       )
         elif not os.path.isfile(self.vocab_file       ):
             with open(_A    ,   "wb"       ) as fi:
                 snake_case								= self.sp_model.serialized_model_proto()
                 fi.write(_A       )

         return (out_vocab_file,)



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


_A   =			logging.get_logger(__name__)

_A   =			{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

_A   =			{
    "vocab_file": {
        "bert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt",
        "bert-large-uncased": "https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt",
        "bert-base-cased": "https://huggingface.co/bert-base-cased/resolve/main/vocab.txt",
        "bert-large-cased": "https://huggingface.co/bert-large-cased/resolve/main/vocab.txt",
        "bert-base-multilingual-uncased": (
            "https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt"
        ),
        "bert-base-multilingual-cased": "https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt",
        "bert-base-chinese": "https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt",
        "bert-base-german-cased": "https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt",
        "bert-large-uncased-whole-word-masking": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt"
        ),
        "bert-large-cased-whole-word-masking": (
            "https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt"
        ),
        "bert-large-uncased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt"
        ),
        "bert-large-cased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt"
        ),
        "bert-base-cased-finetuned-mrpc": (
            "https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt"
        ),
        "bert-base-german-dbmdz-cased": "https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt",
        "bert-base-german-dbmdz-uncased": (
            "https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt"
        ),
        "TurkuNLP/bert-base-finnish-cased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt"
        ),
        "TurkuNLP/bert-base-finnish-uncased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt"
        ),
        "wietsedv/bert-base-dutch-cased": (
            "https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "bert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json",
        "bert-large-uncased": "https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json",
        "bert-base-cased": "https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json",
        "bert-large-cased": "https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json",
        "bert-base-multilingual-uncased": (
            "https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json"
        ),
        "bert-base-multilingual-cased": (
            "https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json"
        ),
        "bert-base-chinese": "https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json",
        "bert-base-german-cased": "https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json",
        "bert-large-uncased-whole-word-masking": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json"
        ),
        "bert-large-cased-whole-word-masking": (
            "https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json"
        ),
        "bert-large-uncased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json"
        ),
        "bert-large-cased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json"
        ),
        "bert-base-cased-finetuned-mrpc": (
            "https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json"
        ),
        "bert-base-german-dbmdz-cased": (
            "https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json"
        ),
        "bert-base-german-dbmdz-uncased": (
            "https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json"
        ),
        "TurkuNLP/bert-base-finnish-cased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json"
        ),
        "TurkuNLP/bert-base-finnish-uncased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json"
        ),
        "wietsedv/bert-base-dutch-cased": (
            "https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json"
        ),
    },
}

_A   =			{
    "bert-base-uncased": 5_12,
    "bert-large-uncased": 5_12,
    "bert-base-cased": 5_12,
    "bert-large-cased": 5_12,
    "bert-base-multilingual-uncased": 5_12,
    "bert-base-multilingual-cased": 5_12,
    "bert-base-chinese": 5_12,
    "bert-base-german-cased": 5_12,
    "bert-large-uncased-whole-word-masking": 5_12,
    "bert-large-cased-whole-word-masking": 5_12,
    "bert-large-uncased-whole-word-masking-finetuned-squad": 5_12,
    "bert-large-cased-whole-word-masking-finetuned-squad": 5_12,
    "bert-base-cased-finetuned-mrpc": 5_12,
    "bert-base-german-dbmdz-cased": 5_12,
    "bert-base-german-dbmdz-uncased": 5_12,
    "TurkuNLP/bert-base-finnish-cased-v1": 5_12,
    "TurkuNLP/bert-base-finnish-uncased-v1": 5_12,
    "wietsedv/bert-base-dutch-cased": 5_12,
}

_A   =			{
    "bert-base-uncased": {"do_lower_case": True},
    "bert-large-uncased": {"do_lower_case": True},
    "bert-base-cased": {"do_lower_case": False},
    "bert-large-cased": {"do_lower_case": False},
    "bert-base-multilingual-uncased": {"do_lower_case": True},
    "bert-base-multilingual-cased": {"do_lower_case": False},
    "bert-base-chinese": {"do_lower_case": False},
    "bert-base-german-cased": {"do_lower_case": False},
    "bert-large-uncased-whole-word-masking": {"do_lower_case": True},
    "bert-large-cased-whole-word-masking": {"do_lower_case": False},
    "bert-large-uncased-whole-word-masking-finetuned-squad": {"do_lower_case": True},
    "bert-large-cased-whole-word-masking-finetuned-squad": {"do_lower_case": False},
    "bert-base-cased-finetuned-mrpc": {"do_lower_case": False},
    "bert-base-german-dbmdz-cased": {"do_lower_case": False},
    "bert-base-german-dbmdz-uncased": {"do_lower_case": True},
    "TurkuNLP/bert-base-finnish-cased-v1": {"do_lower_case": False},
    "TurkuNLP/bert-base-finnish-uncased-v1": {"do_lower_case": True},
    "wietsedv/bert-base-dutch-cased": {"do_lower_case": False},
}







class 					lowerCamelCase		(  A_  ):
     UpperCAmelCase__    :							Optional[Any]									=		VOCAB_FILES_NAMES
     UpperCAmelCase__    :							Union[str, Any]									=		PRETRAINED_VOCAB_FILES_MAP
     UpperCAmelCase__    :							Union[str, Any]									=		PRETRAINED_INIT_CONFIGURATION
     UpperCAmelCase__    :							Any									=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     UpperCAmelCase__    :							int									=		BertTokenizer





     def __init__(self				:   Union[str, Any]    ,   _A				:   Union[str, Any]=None    ,   _A				:   Optional[int]=None    ,   _A				:   List[str]=True    ,   _A				:   List[Any]="[UNK]"    ,   _A				:   Union[str, Any]="[SEP]"    ,   _A				:   List[Any]="[PAD]"    ,   _A				:   List[Any]="[CLS]"    ,   _A				:   Union[str, Any]="[MASK]"    ,   _A				:   int=True    ,   _A				:   Tuple=None    ,   **_A				:   Optional[int]    ,   )						->						int:
         super().__init__(
             _A    ,   tokenizer_file=_A    ,   do_lower_case=_A    ,   unk_token=_A    ,   sep_token=_A    ,   pad_token=_A    ,   cls_token=_A    ,   mask_token=_A    ,   tokenize_chinese_chars=_A    ,   strip_accents=_A    ,   **_A    ,   )

         snake_case								= json.loads(self.backend_tokenizer.normalizer.__getstate__()       )
         if (
             normalizer_state.get("lowercase"    ,   _A       ) != do_lower_case
             or normalizer_state.get("strip_accents"    ,   _A       ) != strip_accents
             or normalizer_state.get("handle_chinese_chars"    ,   _A       ) != tokenize_chinese_chars
         ):
             snake_case								= getattr(_A    ,   normalizer_state.pop("type"       )       )
             snake_case								= do_lower_case
             snake_case								= strip_accents
             snake_case								= tokenize_chinese_chars
             snake_case								= normalizer_class(**_A       )

         snake_case								= do_lower_case





     def 		UpperCAmelCase(self				:   str    ,   _A				:   Union[str, Any]    ,   _A				:   int=None       )						->						Any:
         snake_case								= [self.cls_token_id] + token_ids_a + [self.sep_token_id]

         if token_ids_a:
             output += token_ids_a + [self.sep_token_id]

         return output





     def 		UpperCAmelCase(self				:   Optional[Any]    ,   _A				:   List[int]    ,   _A				:   Optional[List[int]] = None       )						->						List[int]:
         snake_case								= [self.sep_token_id]
         snake_case								= [self.cls_token_id]
         if token_ids_a is None:
             return len(cls + token_ids_a + sep       ) * [0]
         return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]





     def 		UpperCAmelCase(self				:   Union[str, Any]    ,   _A				:   str    ,   _A				:   Optional[str] = None       )						->						Tuple[str]:
         snake_case								= self._tokenizer.model.save(_A    ,   name=_A       )
         return tuple(_A       )

