




import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase_						=			logging.get_logger(__name__)
lowerCAmelCase_						=			{'vocab_file': 'spiece.model'}

lowerCAmelCase_						=			{
    'vocab_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/spiece.model',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/spiece.model',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/spiece.model',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/spiece.model',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model',
    }
}

lowerCAmelCase_						=			{
    'albert-base-v1': 5_12,
    'albert-large-v1': 5_12,
    'albert-xlarge-v1': 5_12,
    'albert-xxlarge-v1': 5_12,
    'albert-base-v2': 5_12,
    'albert-large-v2': 5_12,
    'albert-xlarge-v2': 5_12,
    'albert-xxlarge-v2': 5_12,
}

lowerCAmelCase_						=			'▁'
class 						_A (  _lowerCamelCase				):
  _UpperCamelCase       :      int			=      VOCAB_FILES_NAMES
  _UpperCamelCase       :      List[str]			=      PRETRAINED_VOCAB_FILES_MAP
  _UpperCamelCase       :      Optional[Any]			=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES






  def __init__(  self		:     List[str]				,						_A		:     Tuple				,						_A		:     List[Any]=True				,						_A		:     List[str]=True				,						_A		:     List[str]=False				,						_A		:     Tuple="[CLS]"				,						_A		:     int="[SEP]"				,						_A		:     str="<unk>"				,						_A		:     int="[SEP]"				,						_A		:     int="<pad>"				,						_A		:     Union[str, Any]="[CLS]"				,						_A		:     Optional[Any]="[MASK]"				,						_A		:     Optional[Dict[str, Any]] = None				,						**_A		:     str				,						)		->   None:






       """simple docstring"""







       lowercase :       Any       =			(
           AddedToken(_A				,						lstrip=_A				,						rstrip=_A				,						normalized=_A				)
           if isinstance(_A				,						_A				)
           else mask_token
       )

       lowercase :       Any       =			{} if sp_model_kwargs is None else sp_model_kwargs

       super().__init__(
           do_lower_case=_A				,						remove_space=_A				,						keep_accents=_A				,						bos_token=_A				,						eos_token=_A				,						unk_token=_A				,						sep_token=_A				,						pad_token=_A				,						cls_token=_A				,						mask_token=_A				,						sp_model_kwargs=self.sp_model_kwargs				,						**_A				,						)

       lowercase :       Dict       =			do_lower_case
       lowercase :       Optional[int]       =			remove_space
       lowercase :       int       =			keep_accents
       lowercase :       Any       =			vocab_file

       lowercase :       Union[str, Any]       =			spm.SentencePieceProcessor(**self.sp_model_kwargs				)
       self.sp_model.Load(_A				)






  @property
  def  __a      (  self		:     Any				)		->   str:






       """simple docstring"""







       return len(self.sp_model				)






  def  __a      (  self		:     List[str]				)		->   Tuple:






       """simple docstring"""







       lowercase :       Optional[Any]       =			{self.convert_ids_to_tokens(_A				): i for i in range(self.vocab_size				)}
       vocab.update(self.added_tokens_encoder				)
       return vocab






  def __getstate__(  self		:     List[str]				)		->   Any:






       """simple docstring"""







       lowercase :       Optional[int]       =			self.__dict__.copy()
       lowercase :       List[Any]       =			None
       return state






  def __setstate__(  self		:     List[str]				,						_A		:     Optional[int]				)		->   Any:






       """simple docstring"""







       lowercase :       Optional[Any]       =			d

       # for backward compatibility
       if not hasattr(self				,						'''sp_model_kwargs'''				):
            lowercase :       Dict       =			{}

       lowercase :       Dict       =			spm.SentencePieceProcessor(**self.sp_model_kwargs				)
       self.sp_model.Load(self.vocab_file				)






  def  __a      (  self		:     Optional[int]				,						_A		:     List[Any]				)		->   Tuple:






       """simple docstring"""







       if self.remove_space:
            lowercase :       List[str]       =			''' '''.join(inputs.strip().split()				)
       else:
            lowercase :       Optional[Any]       =			inputs
       lowercase :       Optional[int]       =			outputs.replace('''``'''				,						'''"'''				).replace('''\'\''''				,						'''"'''				)

       if not self.keep_accents:
            lowercase :       Union[str, Any]       =			unicodedata.normalize('''NFKD'''				,						_A				)
            lowercase :       List[Any]       =			''''''.join([c for c in outputs if not unicodedata.combining(_A				)]				)
       if self.do_lower_case:
            lowercase :       Union[str, Any]       =			outputs.lower()

       return outputs






  def  __a      (  self		:     Any				,						_A		:     str				)		->   List[str]:






       """simple docstring"""







       lowercase :       Dict       =			self.preprocess_text(_A				)
       lowercase :       Tuple       =			self.sp_model.encode(_A				,						out_type=_A				)
       lowercase :       Tuple       =			[]
       for piece in pieces:
            if len(_A				) > 1 and piece[-1] == str(''','''				) and piece[-2].isdigit():
                 lowercase :       str       =			self.sp_model.EncodeAsPieces(piece[:-1].replace(_A				,						''''''				)				)
                 if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                      if len(cur_pieces[0]				) == 1:
                           lowercase :       Tuple       =			cur_pieces[1:]
                      else:
                           lowercase :       int       =			cur_pieces[0][1:]
                 cur_pieces.append(piece[-1]				)
                 new_pieces.extend(_A				)
            else:
                 new_pieces.append(_A				)

       return new_pieces






  def  __a      (  self		:     Union[str, Any]				,						_A		:     List[str]				)		->   List[Any]:






       """simple docstring"""







       return self.sp_model.PieceToId(_A				)






  def  __a      (  self		:     Union[str, Any]				,						_A		:     str				)		->   Tuple:






       """simple docstring"""







       return self.sp_model.IdToPiece(_A				)






  def  __a      (  self		:     Tuple				,						_A		:     int				)		->   Any:






       """simple docstring"""







       lowercase :       str       =			[]
       lowercase :       Optional[Any]       =			''''''
       lowercase :       int       =			False
       for token in tokens:
            # make sure that special tokens are not decoded using sentencepiece model
            if token in self.all_special_tokens:
                 if not prev_is_special:
                      out_string += " "
                 out_string += self.sp_model.decode(_A				) + token
                 lowercase :       Tuple       =			True
                 lowercase :       Union[str, Any]       =			[]
            else:
                 current_sub_tokens.append(_A				)
                 lowercase :       str       =			False
       out_string += self.sp_model.decode(_A				)
       return out_string.strip()






  def  __a      (  self		:     Optional[Any]				,						_A		:     List[int]				,						_A		:     Optional[List[int]] = None				)		->   List[int]:






       """simple docstring"""







       lowercase :       Optional[int]       =			[self.sep_token_id]
       lowercase :       Dict       =			[self.cls_token_id]
       if token_ids_a is None:
            return cls + token_ids_a + sep
       return cls + token_ids_a + sep + token_ids_a + sep






  def  __a      (  self		:     Tuple				,						_A		:     List[int]				,						_A		:     Optional[List[int]] = None				,						_A		:     bool = False				)		->   List[int]:






       """simple docstring"""







       if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_a=_A				,						token_ids_a=_A				,						already_has_special_tokens=_A				)

       if token_ids_a is not None:
            return [1] + ([0] * len(_A				)) + [1] + ([0] * len(_A				)) + [1]
       return [1] + ([0] * len(_A				)) + [1]






  def  __a      (  self		:     Tuple				,						_A		:     List[int]				,						_A		:     Optional[List[int]] = None				)		->   List[int]:






       """simple docstring"""







       lowercase :       Optional[Any]       =			[self.sep_token_id]
       lowercase :       List[str]       =			[self.cls_token_id]

       if token_ids_a is None:
            return len(cls + token_ids_a + sep				) * [0]
       return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]






  def  __a      (  self		:     Dict				,						_A		:     str				,						_A		:     Optional[str] = None				)		->   Tuple[str]:






       """simple docstring"""







       if not os.path.isdir(_A				):
            logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""				)
            return
       lowercase :       Union[str, Any]       =			os.path.join(
           _A				,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']				)

       if os.path.abspath(self.vocab_file				) != os.path.abspath(_A				) and os.path.isfile(self.vocab_file				):
            copyfile(self.vocab_file				,						_A				)
       elif not os.path.isfile(self.vocab_file				):
            with open(_A				,						'''wb'''				) as fi:
                 lowercase :       Any       =			self.sp_model.serialized_model_proto()
                 fi.write(_A				)

       return (out_vocab_file,)






from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


lowerCAmelCase_						=			logging.get_logger(__name__)

lowerCAmelCase_						=			{
    'facebook/xmod-base': 'https://huggingface.co/facebook/xmod-base/resolve/main/config.json',
    'facebook/xmod-large-prenorm': 'https://huggingface.co/facebook/xmod-large-prenorm/resolve/main/config.json',
    'facebook/xmod-base-13-125k': 'https://huggingface.co/facebook/xmod-base-13-125k/resolve/main/config.json',
    'facebook/xmod-base-30-125k': 'https://huggingface.co/facebook/xmod-base-30-125k/resolve/main/config.json',
    'facebook/xmod-base-30-195k': 'https://huggingface.co/facebook/xmod-base-30-195k/resolve/main/config.json',
    'facebook/xmod-base-60-125k': 'https://huggingface.co/facebook/xmod-base-60-125k/resolve/main/config.json',
    'facebook/xmod-base-60-265k': 'https://huggingface.co/facebook/xmod-base-60-265k/resolve/main/config.json',
    'facebook/xmod-base-75-125k': 'https://huggingface.co/facebook/xmod-base-75-125k/resolve/main/config.json',
    'facebook/xmod-base-75-269k': 'https://huggingface.co/facebook/xmod-base-75-269k/resolve/main/config.json',
}
class 						_A (  _lowerCamelCase				):
  _UpperCamelCase       :      Tuple			=      '''xmod'''






  def __init__(  self		:     Optional[Any]				,						_A		:     Union[str, Any]=30_522				,						_A		:     List[Any]=768				,						_A		:     Optional[Any]=12				,						_A		:     Any=12				,						_A		:     Tuple=3_072				,						_A		:     Optional[int]="gelu"				,						_A		:     List[Any]=0.1				,						_A		:     str=0.1				,						_A		:     List[Any]=512				,						_A		:     List[str]=2				,						_A		:     str=0.02				,						_A		:     Any=1E-12				,						_A		:     Union[str, Any]=1				,						_A		:     List[Any]=0				,						_A		:     Dict=2				,						_A		:     int="absolute"				,						_A		:     Dict=True				,						_A		:     int=None				,						_A		:     List[str]=False				,						_A		:     Dict=2				,						_A		:     int=False				,						_A		:     Optional[int]=True				,						_A		:     Any=True				,						_A		:     Optional[int]=("en_XX",)				,						_A		:     Any=None				,						**_A		:     Optional[int]				,						)		->   Union[str, Any]:






       """simple docstring"""







       super().__init__(pad_token_id=_A				,						bos_token_id=_A				,						eos_token_id=_A				,						**_A				)

       lowercase :       Optional[Any]       =			vocab_size
       lowercase :       Union[str, Any]       =			hidden_size
       lowercase :       Optional[Any]       =			num_hidden_layers
       lowercase :       Dict       =			num_attention_heads
       lowercase :       Union[str, Any]       =			hidden_act
       lowercase :       Tuple       =			intermediate_size
       lowercase :       List[str]       =			hidden_dropout_prob
       lowercase :       Union[str, Any]       =			attention_probs_dropout_prob
       lowercase :       Dict       =			max_position_embeddings
       lowercase :       Any       =			type_vocab_size
       lowercase :       Optional[Any]       =			initializer_range
       lowercase :       str       =			layer_norm_eps
       lowercase :       Tuple       =			position_embedding_type
       lowercase :       Optional[Any]       =			use_cache
       lowercase :       int       =			classifier_dropout
       lowercase :       Optional[int]       =			pre_norm
       lowercase :       Any       =			adapter_reduction_factor
       lowercase :       Union[str, Any]       =			adapter_layer_norm
       lowercase :       Optional[int]       =			adapter_reuse_layer_norm
       lowercase :       Optional[Any]       =			ln_before_adapter
       lowercase :       Union[str, Any]       =			list(_A				)
       lowercase :       List[Any]       =			default_language


class 						_A (  _lowerCamelCase				):






  @property
  def  __a      (  self		:     Union[str, Any]				)		->   Mapping[str, Mapping[int, str]]:






       """simple docstring"""







       if self.task == "multiple-choice":
            lowercase :       Any       =			{0: '''batch''', 1: '''choice''', 2: '''sequence'''}
       else:
            lowercase :       Union[str, Any]       =			{0: '''batch''', 1: '''sequence'''}
       return OrderedDict(
           [
               ('''input_ids''', dynamic_axis),
               ('''attention_mask''', dynamic_axis),
           ]				)