
import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowercase__          =   logging.get_logger(__name__)
lowercase__          =   {'vocab_file': 'spiece.model'}

lowercase__          =   {
    'vocab_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/spiece.model',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/spiece.model',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/spiece.model',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/spiece.model',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model',
    }
}

lowercase__          =   {
    'albert-base-v1': 512,
    'albert-large-v1': 512,
    'albert-xlarge-v1': 512,
    'albert-xxlarge-v1': 512,
    'albert-base-v2': 512,
    'albert-large-v2': 512,
    'albert-xlarge-v2': 512,
    'albert-xxlarge-v2': 512,
}

lowercase__          =   '▁'



class 		UpperCamelCase__  (	__lowercase   ):
   _SCREAMING_SNAKE_CASE				:							Dict     	=				VOCAB_FILES_NAMES
   _SCREAMING_SNAKE_CASE				:							Union[str, Any]     	=				PRETRAINED_VOCAB_FILES_MAP
   _SCREAMING_SNAKE_CASE				:							str     	=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   def __init__(self			: Dict ,    snake_case_			: Optional[int] ,    snake_case_			: Tuple=True ,    snake_case_			: Any=True ,    snake_case_			: List[Any]=False ,    snake_case_			: List[str]="[CLS]" ,    snake_case_			: List[Any]="[SEP]" ,    snake_case_			: List[str]="<unk>" ,    snake_case_			: Dict="[SEP]" ,    snake_case_			: Tuple="<pad>" ,    snake_case_			: str="[CLS]" ,    snake_case_			: List[str]="[MASK]" ,    snake_case_			: Optional[Dict[str, Any]] = None ,    **snake_case_			: Tuple ,    ):
    # Mask token behave like a normal word, i.e. include the space before it and
    # is included in the raw text, there should be a match in a non-normalized sentence.
    __a			:  List[Any]		    =  (
        AddedToken(snake_case_ ,    lstrip=snake_case_ ,    rstrip=snake_case_ ,    normalized=snake_case_       )
        if isinstance(snake_case_ ,    snake_case_       )
        else mask_token
    )

    __a			:  int		    =  {} if sp_model_kwargs is None else sp_model_kwargs

    super().__init__(
        do_lower_case=snake_case_ ,    remove_space=snake_case_ ,    keep_accents=snake_case_ ,    bos_token=snake_case_ ,    eos_token=snake_case_ ,    unk_token=snake_case_ ,    sep_token=snake_case_ ,    pad_token=snake_case_ ,    cls_token=snake_case_ ,    mask_token=snake_case_ ,    sp_model_kwargs=self.sp_model_kwargs ,    **snake_case_ ,    )

    __a			:  List[str]		    =  do_lower_case
    __a			:  Optional[int]		    =  remove_space
    __a			:  int		    =  keep_accents
    __a			:  Dict		    =  vocab_file

    __a			:  List[str]		    =  spm.SentencePieceProcessor(**self.sp_model_kwargs       )
    self.sp_model.Load(snake_case_       )
   @property
   def 				lowerCAmelCase      (self			: Optional[Any]       ):
    return len(self.sp_model       )
   def 				lowerCAmelCase      (self			: List[Any]       ):
    __a			:  Optional[Any]		    =  {self.convert_ids_to_tokens(snake_case_       ): i for i in range(self.vocab_size       )}
    vocab.update(self.added_tokens_encoder       )
    return vocab
   def __getstate__(self			: Union[str, Any]       ):
    __a			:  Union[str, Any]		    =  self.__dict__.copy()
    __a			:  Optional[int]		    =  None
    return state
   def __setstate__(self			: Any ,    snake_case_			: Optional[int]       ):
    __a			:  Tuple		    =  d

    # for backward compatibility
    if not hasattr(self ,    '''sp_model_kwargs'''       ):
     __a			:  Union[str, Any]		    =  {}

    __a			:  List[str]		    =  spm.SentencePieceProcessor(**self.sp_model_kwargs       )
    self.sp_model.Load(self.vocab_file       )
   def 				lowerCAmelCase      (self			: Optional[Any] ,    snake_case_			: Union[str, Any]       ):
    if self.remove_space:
     __a			:  int		    =  ''' '''.join(inputs.strip().split()       )
    else:
     __a			:  Union[str, Any]		    =  inputs
    __a			:  int		    =  outputs.replace('''``''' ,    '''"'''       ).replace('''\'\'''' ,    '''"'''       )

    if not self.keep_accents:
     __a			:  Dict		    =  unicodedata.normalize('''NFKD''' ,    snake_case_       )
     __a			:  str		    =  ''''''.join([c for c in outputs if not unicodedata.combining(snake_case_       )]       )
    if self.do_lower_case:
     __a			:  Optional[Any]		    =  outputs.lower()

    return outputs
   def 				lowerCAmelCase      (self			: Optional[int] ,    snake_case_			: str       ):
    __a			:  List[Any]		    =  self.preprocess_text(snake_case_       )
    __a			:  Any		    =  self.sp_model.encode(snake_case_ ,    out_type=snake_case_       )
    __a			:  str		    =  []
    for piece in pieces:
     if len(snake_case_       ) > 1 and piece[-1] == str(''','''       ) and piece[-2].isdigit():
      __a			:  Tuple		    =  self.sp_model.EncodeAsPieces(piece[:-1].replace(snake_case_ ,    ''''''       )       )
      if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
       if len(cur_pieces[0]       ) == 1:
        __a			:  int		    =  cur_pieces[1:]
       else:
        __a			:  str		    =  cur_pieces[0][1:]
      cur_pieces.append(piece[-1]       )
      new_pieces.extend(snake_case_       )
     else:
      new_pieces.append(snake_case_       )

    return new_pieces
   def 				lowerCAmelCase      (self			: List[Any] ,    snake_case_			: Dict       ):
    return self.sp_model.PieceToId(snake_case_       )
   def 				lowerCAmelCase      (self			: int ,    snake_case_			: Tuple       ):
    return self.sp_model.IdToPiece(snake_case_       )
   def 				lowerCAmelCase      (self			: Union[str, Any] ,    snake_case_			: Union[str, Any]       ):
    __a			:  Any		    =  []
    __a			:  Optional[Any]		    =  ''''''
    __a			:  Dict		    =  False
    for token in tokens:
     # make sure that special tokens are not decoded using sentencepiece model
     if token in self.all_special_tokens:
      if not prev_is_special:
       out_string += " "
      out_string += self.sp_model.decode(snake_case_       ) + token
      __a			:  Optional[int]		    =  True
      __a			:  int		    =  []
     else:
      current_sub_tokens.append(snake_case_       )
      __a			:  Any		    =  False
    out_string += self.sp_model.decode(snake_case_       )
    return out_string.strip()
   def 				lowerCAmelCase      (self			: List[Any] ,    snake_case_			: List[int] ,    snake_case_			: Optional[List[int]] = None       ):
    __a			:  Tuple		    =  [self.sep_token_id]
    __a			:  Any		    =  [self.cls_token_id]
    if token_ids_a is None:
     return cls + token_ids_a + sep
    return cls + token_ids_a + sep + token_ids_a + sep
   def 				lowerCAmelCase      (self			: str ,    snake_case_			: List[int] ,    snake_case_			: Optional[List[int]] = None ,    snake_case_			: bool = False       ):

    if already_has_special_tokens:
     return super().get_special_tokens_mask(
         token_ids_a=snake_case_ ,    token_ids_a=snake_case_ ,    already_has_special_tokens=snake_case_       )

    if token_ids_a is not None:
     return [1] + ([0] * len(snake_case_       )) + [1] + ([0] * len(snake_case_       )) + [1]
    return [1] + ([0] * len(snake_case_       )) + [1]
   def 				lowerCAmelCase      (self			: Union[str, Any] ,    snake_case_			: List[int] ,    snake_case_			: Optional[List[int]] = None       ):
    __a			:  Union[str, Any]		    =  [self.sep_token_id]
    __a			:  Optional[Any]		    =  [self.cls_token_id]

    if token_ids_a is None:
     return len(cls + token_ids_a + sep       ) * [0]
    return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]







   def 				lowerCAmelCase      (self			: str ,    snake_case_			: str ,    snake_case_			: Optional[str] = None       ):
    if not os.path.isdir(snake_case_       ):
     logger.error(f"Vocabulary path ({save_directory}) should be a directory"       )
     return
    __a			:  Optional[Any]		    =  os.path.join(
        snake_case_ ,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']       )

    if os.path.abspath(self.vocab_file       ) != os.path.abspath(snake_case_       ) and os.path.isfile(self.vocab_file       ):
     copyfile(self.vocab_file ,    snake_case_       )
    elif not os.path.isfile(self.vocab_file       ):
     with open(snake_case_ ,    '''wb'''       ) as fi:
      __a			:  int		    =  self.sp_model.serialized_model_proto()
      fi.write(snake_case_       )

    return (out_vocab_file,)


import os
import unittest

from transformers.models.phobert.tokenization_phobert import VOCAB_FILES_NAMES, PhobertTokenizer

from ...test_tokenization_common import TokenizerTesterMixin



class 		UpperCamelCase__  (	__lowercase   ,unittest.TestCase   ):
   _SCREAMING_SNAKE_CASE				:							Union[str, Any]     	=				PhobertTokenizer
   _SCREAMING_SNAKE_CASE				:							int     	=				False
   def 				lowerCAmelCase      (self			: Optional[int]       ):
    super().setUp()

    # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
    __a			:  Optional[int]		    =  ['''T@@''', '''i''', '''I''', '''R@@''', '''r''', '''e@@''']
    __a			:  Tuple		    =  dict(zip(snake_case_ ,    range(len(snake_case_       )       )       )       )
    __a			:  int		    =  ['''#version: 0.2''', '''l à</w>''']
    __a			:  List[Any]		    =  {'''unk_token''': '''<unk>'''}

    __a			:  Any		    =  os.path.join(self.tmpdirname ,    VOCAB_FILES_NAMES['''vocab_file''']       )
    __a			:  List[Any]		    =  os.path.join(self.tmpdirname ,    VOCAB_FILES_NAMES['''merges_file''']       )

    with open(self.vocab_file ,    '''w''' ,    encoding='''utf-8'''       ) as fp:
     for token in vocab_tokens:
      fp.write(f"{token} {vocab_tokens[token]}\n"       )
    with open(self.merges_file ,    '''w''' ,    encoding='''utf-8'''       ) as fp:
     fp.write('''\n'''.join(snake_case_       )       )
   def 				lowerCAmelCase      (self			: Union[str, Any] ,    **snake_case_			: List[str]       ):
    kwargs.update(self.special_tokens_map       )
    return PhobertTokenizer.from_pretrained(self.tmpdirname ,    **snake_case_       )
   def 				lowerCAmelCase      (self			: Any ,    snake_case_			: Dict       ):
    __a			:  Union[str, Any]		    =  '''Tôi là VinAI Research'''
    __a			:  int		    =  '''T<unk> i <unk> <unk> <unk> <unk> <unk> <unk> I Re<unk> e<unk> <unk> <unk> <unk>'''
    return input_text, output_text







   def 				lowerCAmelCase      (self			: Optional[Any]       ):
    __a			:  Optional[int]		    =  PhobertTokenizer(self.vocab_file ,    self.merges_file ,    **self.special_tokens_map       )
    __a			:  Any		    =  '''Tôi là VinAI Research'''
    __a			:  Union[str, Any]		    =  '''T@@ ô@@ i l@@ à V@@ i@@ n@@ A@@ I R@@ e@@ s@@ e@@ a@@ r@@ c@@ h'''.split()
    __a			:  List[str]		    =  tokenizer.tokenize(snake_case_       )
    print(snake_case_       )
    self.assertListEqual(snake_case_ ,    snake_case_       )

    __a			:  str		    =  tokens + [tokenizer.unk_token]

    __a			:  str		    =  [4, 3, 5, 3, 3, 3, 3, 3, 3, 6, 7, 9, 3, 9, 3, 3, 3, 3, 3]
    self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case_       ) ,    snake_case_       )
