




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_convbert import ConvBertTokenizer


lowerCAmelCase_						=			logging.get_logger(__name__)

lowerCAmelCase_						=			{"vocab_file": "vocab.txt"}

lowerCAmelCase_						=			{
    "vocab_file": {
        "YituTech/conv-bert-base": "https://huggingface.co/YituTech/conv-bert-base/resolve/main/vocab.txt",
        "YituTech/conv-bert-medium-small": (
            "https://huggingface.co/YituTech/conv-bert-medium-small/resolve/main/vocab.txt"
        ),
        "YituTech/conv-bert-small": "https://huggingface.co/YituTech/conv-bert-small/resolve/main/vocab.txt",
    }
}

lowerCAmelCase_						=			{
    "YituTech/conv-bert-base": 5_12,
    "YituTech/conv-bert-medium-small": 5_12,
    "YituTech/conv-bert-small": 5_12,
}


lowerCAmelCase_						=			{
    "YituTech/conv-bert-base": {"do_lower_case": True},
    "YituTech/conv-bert-medium-small": {"do_lower_case": True},
    "YituTech/conv-bert-small": {"do_lower_case": True},
}
class 						_A (  SCREAMING_SNAKE_CASE__				):
  _UpperCamelCase       :      Tuple			=      VOCAB_FILES_NAMES
  _UpperCamelCase       :      Any			=      PRETRAINED_VOCAB_FILES_MAP
  _UpperCamelCase       :      List[str]			=      PRETRAINED_INIT_CONFIGURATION
  _UpperCamelCase       :      Any			=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  _UpperCamelCase       :      int			=      ConvBertTokenizer






  def __init__(  self		:     Any				,						_A		:     Dict=None				,						_A		:     Optional[int]=None				,						_A		:     Optional[int]=True				,						_A		:     List[str]="[UNK]"				,						_A		:     Tuple="[SEP]"				,						_A		:     str="[PAD]"				,						_A		:     Optional[Any]="[CLS]"				,						_A		:     int="[MASK]"				,						_A		:     Tuple=True				,						_A		:     Optional[Any]=None				,						**_A		:     Optional[int]				,						)		->   List[str]:






       """simple docstring"""







       super().__init__(
           A__				,						tokenizer_file=A__				,						do_lower_case=A__				,						unk_token=A__				,						sep_token=A__				,						pad_token=A__				,						cls_token=A__				,						mask_token=A__				,						tokenize_chinese_chars=A__				,						strip_accents=A__				,						**A__				,						)

       lowercase :       str       =			json.loads(self.backend_tokenizer.normalizer.__getstate__()				)
       if (
           normalizer_state.get('''lowercase'''				,						A__				) != do_lower_case
           or normalizer_state.get('''strip_accents'''				,						A__				) != strip_accents
           or normalizer_state.get('''handle_chinese_chars'''				,						A__				) != tokenize_chinese_chars
       ):
            lowercase :       List[str]       =			getattr(A__				,						normalizer_state.pop('''type'''				)				)
            lowercase :       List[Any]       =			do_lower_case
            lowercase :       Optional[Any]       =			strip_accents
            lowercase :       str       =			tokenize_chinese_chars
            lowercase :       Optional[Any]       =			normalizer_class(**A__				)

       lowercase :       int       =			do_lower_case






  def  __a      (  self		:     str				,						_A		:     Any				,						_A		:     str=None				)		->   Union[str, Any]:






       """simple docstring"""







       lowercase :       int       =			[self.cls_token_id] + token_ids_a + [self.sep_token_id]

       if token_ids_a:
            output += token_ids_a + [self.sep_token_id]

       return output






  def  __a      (  self		:     Optional[int]				,						_A		:     Union[str, Any]				,						_A		:     Dict = None				)		->   Tuple:






       """simple docstring"""







       lowercase :       Dict       =			[self.sep_token_id]
       lowercase :       Optional[int]       =			[self.cls_token_id]
       if token_ids_a is None:
            return len(cls + token_ids_a + sep				) * [0]
       return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]






  def  __a      (  self		:     int				,						_A		:     List[str]				,						_A		:     Tuple = None				)		->   List[Any]:






       """simple docstring"""







       lowercase :       Tuple       =			self._tokenizer.model.save(A__				,						name=A__				)
       return tuple(A__				)





def  snake_case(    __magic_name__			,		__magic_name__   )			->							bool:





     '''simple docstring'''


     lowercase :       List[Any]       =			len(__magic_name__   )
     lowercase :       str       =			[[False] * (required_sum + 1) for _ in range(arr_len + 1   )]

     # for each arr value, a sum of zero(0) can be formed by not taking any element
     # hence True/1
     for i in range(arr_len + 1   ):
          lowercase :       List[str]       =			True

     # sum is not zero and set is empty then false
     for i in range(1			,		required_sum + 1   ):
          lowercase :       str       =			False

     for i in range(1			,		arr_len + 1   ):
          for j in range(1			,		required_sum + 1   ):
               if arr[i - 1] > j:
                    lowercase :       Optional[Any]       =			subset[i - 1][j]
               if arr[i - 1] <= j:
                    lowercase :       Dict       =			subset[i - 1][j] or subset[i - 1][j - arr[i - 1]]

     return subset[arr_len][required_sum]


if __name__ == "__main__":
 import doctest

 doctest.testmod()