


import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_A			=						logging.get_logger(__name__)
_A			=						{'vocab_file': 'spiece.model'}

_A			=						{
    'vocab_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/spiece.model',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/spiece.model',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/spiece.model',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/spiece.model',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model',
    }
}

_A			=						{
    'albert-base-v1': 512,
    'albert-large-v1': 512,
    'albert-xlarge-v1': 512,
    'albert-xxlarge-v1': 512,
    'albert-base-v2': 512,
    'albert-large-v2': 512,
    'albert-xlarge-v2': 512,
    'albert-xxlarge-v2': 512,
}

_A			=						'▁'






class    UpperCAmelCase__	( __lowerCAmelCase		):



    """simple docstring"""






    UpperCAmelCase__						:		List[str]              =  VOCAB_FILES_NAMES
    UpperCAmelCase__						:		List[Any]              =  PRETRAINED_VOCAB_FILES_MAP
    UpperCAmelCase__						:		str              =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES






    def __init__(      self ,       A_ ,       A_=True ,       A_=True ,       A_=False ,       A_="[CLS]" ,       A_="[SEP]" ,       A_="<unk>" ,       A_="[SEP]" ,       A_="<pad>" ,       A_="[CLS]" ,       A_="[MASK]" ,       A_ = None ,       **A_ ,       )  ->	None:
         # Mask token behave like a normal word, i.e. include the space before it and
         # is included in the raw text, there should be a match in a non-normalized sentence.
         __UpperCamelCase   							=(
             AddedToken(lowerCAmelCase_ ,       lstrip=lowerCAmelCase_ ,       rstrip=lowerCAmelCase_ ,       normalized=lowerCAmelCase_   )
             if isinstance(lowerCAmelCase_ ,       lowerCAmelCase_   )
             else mask_token
         )

         __UpperCamelCase   							={} if sp_model_kwargs is None else sp_model_kwargs

         super().__init__(
             do_lower_case=lowerCAmelCase_ ,       remove_space=lowerCAmelCase_ ,       keep_accents=lowerCAmelCase_ ,       bos_token=lowerCAmelCase_ ,       eos_token=lowerCAmelCase_ ,       unk_token=lowerCAmelCase_ ,       sep_token=lowerCAmelCase_ ,       pad_token=lowerCAmelCase_ ,       cls_token=lowerCAmelCase_ ,       mask_token=lowerCAmelCase_ ,       sp_model_kwargs=self.sp_model_kwargs ,       **lowerCAmelCase_ ,       )

         __UpperCamelCase   							=do_lower_case
         __UpperCamelCase   							=remove_space
         __UpperCamelCase   							=keep_accents
         __UpperCamelCase   							=vocab_file

         __UpperCamelCase   							=spm.SentencePieceProcessor(**self.sp_model_kwargs   )
         self.sp_model.Load(lowerCAmelCase_   )






    @property
    def _a (      self   )  ->	str:
         return len(self.sp_model   )






    def _a (      self   )  ->	Dict:
         __UpperCamelCase   							={self.convert_ids_to_tokens(lowerCAmelCase_   ): i for i in range(self.vocab_size   )}
         vocab.update(self.added_tokens_encoder   )
         return vocab






    def __getstate__(      self   )  ->	Tuple:
         __UpperCamelCase   							=self.__dict__.copy()
         __UpperCamelCase   							=None
         return state






    def __setstate__(      self ,       A_   )  ->	Optional[int]:
         __UpperCamelCase   							=d

         # for backward compatibility
         if not hasattr(self ,       'sp_model_kwargs'   ):
              __UpperCamelCase   							={}

         __UpperCamelCase   							=spm.SentencePieceProcessor(**self.sp_model_kwargs   )
         self.sp_model.Load(self.vocab_file   )






    def _a (      self ,       A_   )  ->	Union[str, Any]:
         if self.remove_space:
              __UpperCamelCase   							=' '.join(inputs.strip().split()   )
         else:
              __UpperCamelCase   							=inputs
         __UpperCamelCase   							=outputs.replace('``' ,       '\"'   ).replace('\'\'' ,       '\"'   )

         if not self.keep_accents:
              __UpperCamelCase   							=unicodedata.normalize('NFKD' ,       lowerCAmelCase_   )
              __UpperCamelCase   							=''.join([c for c in outputs if not unicodedata.combining(lowerCAmelCase_   )]   )
         if self.do_lower_case:
              __UpperCamelCase   							=outputs.lower()

         return outputs






    def _a (      self ,       A_   )  ->	List[str]:
         __UpperCamelCase   							=self.preprocess_text(lowerCAmelCase_   )
         __UpperCamelCase   							=self.sp_model.encode(lowerCAmelCase_ ,       out_type=lowerCAmelCase_   )
         __UpperCamelCase   							=[]
         for piece in pieces:
              if len(lowerCAmelCase_   ) > 1 and piece[-1] == str(','   ) and piece[-2].isdigit():
                   __UpperCamelCase   							=self.sp_model.EncodeAsPieces(piece[:-1].replace(lowerCAmelCase_ ,       ''   )   )
                   if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                        if len(cur_pieces[0]   ) == 1:
                             __UpperCamelCase   							=cur_pieces[1:]
                        else:
                             __UpperCamelCase   							=cur_pieces[0][1:]
                   cur_pieces.append(piece[-1]   )
                   new_pieces.extend(lowerCAmelCase_   )
              else:
                   new_pieces.append(lowerCAmelCase_   )

         return new_pieces






    def _a (      self ,       A_   )  ->	Any:
         return self.sp_model.PieceToId(lowerCAmelCase_   )






    def _a (      self ,       A_   )  ->	Optional[int]:
         return self.sp_model.IdToPiece(lowerCAmelCase_   )






    def _a (      self ,       A_   )  ->	List[str]:
         __UpperCamelCase   							=[]
         __UpperCamelCase   							=''
         __UpperCamelCase   							=False
         for token in tokens:
              # make sure that special tokens are not decoded using sentencepiece model
              if token in self.all_special_tokens:
                   if not prev_is_special:
                        out_string += " "
                   out_string += self.sp_model.decode(lowerCAmelCase_   ) + token
                   __UpperCamelCase   							=True
                   __UpperCamelCase   							=[]
              else:
                   current_sub_tokens.append(lowerCAmelCase_   )
                   __UpperCamelCase   							=False
         out_string += self.sp_model.decode(lowerCAmelCase_   )
         return out_string.strip()






    def _a (      self ,       A_ ,       A_ = None   )  ->	List[int]:
         __UpperCamelCase   							=[self.sep_token_id]
         __UpperCamelCase   							=[self.cls_token_id]
         if token_ids_a is None:
              return cls + token_ids_a + sep
         return cls + token_ids_a + sep + token_ids_a + sep






    def _a (      self ,       A_ ,       A_ = None ,       A_ = False   )  ->	List[int]:

         if already_has_special_tokens:
              return super().get_special_tokens_mask(
                  token_ids_a=lowerCAmelCase_ ,       token_ids_a=lowerCAmelCase_ ,       already_has_special_tokens=lowerCAmelCase_   )

         if token_ids_a is not None:
              return [1] + ([0] * len(lowerCAmelCase_   )) + [1] + ([0] * len(lowerCAmelCase_   )) + [1]
         return [1] + ([0] * len(lowerCAmelCase_   )) + [1]






    def _a (      self ,       A_ ,       A_ = None   )  ->	List[int]:
         __UpperCamelCase   							=[self.sep_token_id]
         __UpperCamelCase   							=[self.cls_token_id]

         if token_ids_a is None:
              return len(cls + token_ids_a + sep   ) * [0]
         return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]






    def _a (      self ,       A_ ,       A_ = None   )  ->	Tuple[str]:
         if not os.path.isdir(lowerCAmelCase_   ):
              logger.error(f'Vocabulary path ({save_directory}) should be a directory'   )
              return
         __UpperCamelCase   							=os.path.join(
             lowerCAmelCase_ ,       (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']   )

         if os.path.abspath(self.vocab_file   ) != os.path.abspath(lowerCAmelCase_   ) and os.path.isfile(self.vocab_file   ):
              copyfile(self.vocab_file ,       lowerCAmelCase_   )
         elif not os.path.isfile(self.vocab_file   ):
              with open(lowerCAmelCase_ ,       'wb'   ) as fi:
                   __UpperCamelCase   							=self.sp_model.serialized_model_proto()
                   fi.write(lowerCAmelCase_   )

         return (out_vocab_file,)



from ....utils import logging


_A			=						logging.get_logger(__name__)






class    UpperCAmelCase__	( A_		):



    """simple docstring"""






    def __init__(      self ,       A_ ,       A_=None ,       A_=2048   )  ->	Any:
         __UpperCamelCase   							=config.__dict__
         __UpperCamelCase   							=modal_hidden_size
         if num_labels:
              __UpperCamelCase   							=num_labels





