





"""simple docstring"""




import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


UpperCamelCase_									=logging.get_logger(__name__)

UpperCamelCase_									={"""vocab_file""": """spiece.model"""}

UpperCamelCase_									={
    """vocab_file""": {
        """xlnet-base-cased""": """https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model""",
        """xlnet-large-cased""": """https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model""",
    }
}

UpperCamelCase_									={
    """xlnet-base-cased""": None,
    """xlnet-large-cased""": None,
}

# Segments (not really needed)
UpperCamelCase_									=0
UpperCamelCase_									=1
UpperCamelCase_									=2
UpperCamelCase_									=3
UpperCamelCase_									=4




class 			_a						(				__lowercase    ):
				UpperCamelCase								=   VOCAB_FILES_NAMES
				UpperCamelCase								=   PRETRAINED_VOCAB_FILES_MAP
				UpperCamelCase								=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				UpperCamelCase								=   '''left'''






				def __init__(			self		:   int,			lowerCAmelCase__		:   str,			lowerCAmelCase__		:   int=False,			lowerCAmelCase__		:   int=True,			lowerCAmelCase__		:   List[str]=False,			lowerCAmelCase__		:   Any="<s>",			lowerCAmelCase__		:   List[str]="</s>",			lowerCAmelCase__		:   int="<unk>",			lowerCAmelCase__		:   Dict="<sep>",			lowerCAmelCase__		:   str="<pad>",			lowerCAmelCase__		:   Tuple="<cls>",			lowerCAmelCase__		:   Optional[Any]="<mask>",			lowerCAmelCase__		:   List[str]=["<eop>", "<eod>"],			lowerCAmelCase__		:   Optional[Dict[str, Any]] = None,			**lowerCAmelCase__		:   Dict,			)     ->				None:




										'''simple docstring'''



										_UpperCamelCase      : Tuple         =   AddedToken(lowerCAmelCase__,			lstrip=lowerCAmelCase__,			rstrip=lowerCAmelCase__							) if isinstance(lowerCAmelCase__,			lowerCAmelCase__							) else mask_token

										_UpperCamelCase      : Union[str, Any]         =   {} if sp_model_kwargs is None else sp_model_kwargs

										super().__init__(
										    do_lower_case=lowerCAmelCase__,			remove_space=lowerCAmelCase__,			keep_accents=lowerCAmelCase__,			bos_token=lowerCAmelCase__,			eos_token=lowerCAmelCase__,			unk_token=lowerCAmelCase__,			sep_token=lowerCAmelCase__,			pad_token=lowerCAmelCase__,			cls_token=lowerCAmelCase__,			mask_token=lowerCAmelCase__,			additional_special_tokens=lowerCAmelCase__,			sp_model_kwargs=self.sp_model_kwargs,			**lowerCAmelCase__,			)

										_UpperCamelCase      : Tuple         =   3

										_UpperCamelCase      : int         =   do_lower_case
										_UpperCamelCase      : Optional[int]         =   remove_space
										_UpperCamelCase      : Optional[int]         =   keep_accents
										_UpperCamelCase      : List[Any]         =   vocab_file

										_UpperCamelCase      : Optional[Any]         =   spm.SentencePieceProcessor(**self.sp_model_kwargs							)
										self.sp_model.Load(lowerCAmelCase__							)






				@property
				def     snake_case     (			self		:   Tuple							)     ->				str:




										'''simple docstring'''



										return len(self.sp_model							)






				def     snake_case     (			self		:   List[Any]							)     ->				Any:




										'''simple docstring'''



										_UpperCamelCase      : Any         =   {self.convert_ids_to_tokens(lowerCAmelCase__							): i for i in range(self.vocab_size							)}
										vocab.update(self.added_tokens_encoder							)
										return vocab






				def __getstate__(			self		:   Tuple							)     ->				List[Any]:




										'''simple docstring'''



										_UpperCamelCase      : Optional[Any]         =   self.__dict__.copy()
										_UpperCamelCase      : List[Any]         =   None
										return state






				def __setstate__(			self		:   Union[str, Any],			lowerCAmelCase__		:   Optional[int]							)     ->				Union[str, Any]:




										'''simple docstring'''



										_UpperCamelCase      : Union[str, Any]         =   d

										# for backward compatibility
										if not hasattr(self,			'''sp_model_kwargs'''							):
																_UpperCamelCase      : int         =   {}

										_UpperCamelCase      : List[str]         =   spm.SentencePieceProcessor(**self.sp_model_kwargs							)
										self.sp_model.Load(self.vocab_file							)






				def     snake_case     (			self		:   Dict,			lowerCAmelCase__		:   str							)     ->				List[str]:




										'''simple docstring'''



										if self.remove_space:
																_UpperCamelCase      : Optional[int]         =   """ """.join(inputs.strip().split()							)
										else:
																_UpperCamelCase      : List[str]         =   inputs
										_UpperCamelCase      : Optional[Any]         =   outputs.replace('''``''',			'''\"'''							).replace('''\'\'''',			'''\"'''							)

										if not self.keep_accents:
																_UpperCamelCase      : int         =   unicodedata.normalize('''NFKD''',			lowerCAmelCase__							)
																_UpperCamelCase      : int         =   """""".join([c for c in outputs if not unicodedata.combining(lowerCAmelCase__							)]							)
										if self.do_lower_case:
																_UpperCamelCase      : Optional[Any]         =   outputs.lower()

										return outputs






				def     snake_case     (			self		:   List[Any],			lowerCAmelCase__		:   str							)     ->				List[str]:




										'''simple docstring'''



										_UpperCamelCase      : List[Any]         =   self.preprocess_text(lowerCAmelCase__							)
										_UpperCamelCase      : Dict         =   self.sp_model.encode(lowerCAmelCase__,			out_type=lowerCAmelCase__							)
										_UpperCamelCase      : Optional[Any]         =   []
										for piece in pieces:
																if len(lowerCAmelCase__							) > 1 and piece[-1] == str(''','''							) and piece[-2].isdigit():
																						_UpperCamelCase      : Any         =   self.sp_model.EncodeAsPieces(piece[:-1].replace(lowerCAmelCase__,			''''''							)							)
																						if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																												if len(cur_pieces[0]							) == 1:
																																		_UpperCamelCase      : List[str]         =   cur_pieces[1:]
																												else:
																																		_UpperCamelCase      : Any         =   cur_pieces[0][1:]
																						cur_pieces.append(piece[-1]							)
																						new_pieces.extend(lowerCAmelCase__							)
																else:
																						new_pieces.append(lowerCAmelCase__							)

										return new_pieces






				def     snake_case     (			self		:   Tuple,			lowerCAmelCase__		:   str							)     ->				Any:




										'''simple docstring'''



										return self.sp_model.PieceToId(lowerCAmelCase__							)






				def     snake_case     (			self		:   Any,			lowerCAmelCase__		:   Optional[int]							)     ->				int:




										'''simple docstring'''



										return self.sp_model.IdToPiece(lowerCAmelCase__							)






				def     snake_case     (			self		:   str,			lowerCAmelCase__		:   Optional[Any]							)     ->				Dict:




										'''simple docstring'''



										_UpperCamelCase      : Optional[int]         =   """""".join(lowerCAmelCase__							).replace(lowerCAmelCase__,			''' '''							).strip()
										return out_string






				def     snake_case     (			self		:   List[Any],			lowerCAmelCase__		:   List[int],			lowerCAmelCase__		:   bool = False,			lowerCAmelCase__		:   bool = None,			lowerCAmelCase__		:   bool = True,			**lowerCAmelCase__		:   Optional[int],			)     ->				str:




										'''simple docstring'''



										_UpperCamelCase      : Union[str, Any]         =   kwargs.pop('''use_source_tokenizer''',			lowerCAmelCase__							)

										_UpperCamelCase      : Any         =   self.convert_ids_to_tokens(lowerCAmelCase__,			skip_special_tokens=lowerCAmelCase__							)

										# To avoid mixing byte-level and unicode for byte-level BPT
										# we need to build string separately for added tokens and byte-level tokens
										# cf. https://github.com/huggingface/transformers/issues/1133
										_UpperCamelCase      : Dict         =   []
										_UpperCamelCase      : Any         =   []
										for token in filtered_tokens:
																if skip_special_tokens and token in self.all_special_ids:
																						continue
																if token in self.added_tokens_encoder:
																						if current_sub_text:
																												sub_texts.append(self.convert_tokens_to_string(lowerCAmelCase__							)							)
																												_UpperCamelCase      : Optional[int]         =   []
																						sub_texts.append(lowerCAmelCase__							)
																else:
																						current_sub_text.append(lowerCAmelCase__							)
										if current_sub_text:
																sub_texts.append(self.convert_tokens_to_string(lowerCAmelCase__							)							)

										# Mimic the behavior of the Rust tokenizer:
										# By default, there are no spaces between special tokens
										_UpperCamelCase      : Tuple         =   """""".join(lowerCAmelCase__							)

										_UpperCamelCase      : Optional[Any]         =   (
										    clean_up_tokenization_spaces
										    if clean_up_tokenization_spaces is not None
										    else self.clean_up_tokenization_spaces
										)
										if clean_up_tokenization_spaces:
																_UpperCamelCase      : Dict         =   self.clean_up_tokenization(lowerCAmelCase__							)
																return clean_text
										else:
																return text






				def     snake_case     (			self		:   List[Any],			lowerCAmelCase__		:   List[int],			lowerCAmelCase__		:   Optional[List[int]] = None							)     ->				List[int]:




										'''simple docstring'''



										_UpperCamelCase      : Union[str, Any]         =   [self.sep_token_id]
										_UpperCamelCase      : Union[str, Any]         =   [self.cls_token_id]
										if token_ids_a is None:
																return token_ids_a + sep + cls
										return token_ids_a + sep + token_ids_a + sep + cls






				def     snake_case     (			self		:   Union[str, Any],			lowerCAmelCase__		:   List[int],			lowerCAmelCase__		:   Optional[List[int]] = None,			lowerCAmelCase__		:   bool = False							)     ->				List[int]:




										'''simple docstring'''



										if already_has_special_tokens:
																return super().get_special_tokens_mask(
																    token_ids_a=lowerCAmelCase__,			token_ids_a=lowerCAmelCase__,			already_has_special_tokens=lowerCAmelCase__							)

										if token_ids_a is not None:
																return ([0] * len(lowerCAmelCase__							)) + [1] + ([0] * len(lowerCAmelCase__							)) + [1, 1]
										return ([0] * len(lowerCAmelCase__							)) + [1, 1]






				def     snake_case     (			self		:   int,			lowerCAmelCase__		:   List[int],			lowerCAmelCase__		:   Optional[List[int]] = None							)     ->				List[int]:




										'''simple docstring'''



										_UpperCamelCase      : Any         =   [self.sep_token_id]
										_UpperCamelCase      : Optional[int]         =   [2]

										if token_ids_a is None:
																return len(token_ids_a + sep							) * [0] + cls_segment_id
										return len(token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1] + cls_segment_id






				def     snake_case     (			self		:   Union[str, Any],			lowerCAmelCase__		:   str,			lowerCAmelCase__		:   Optional[str] = None							)     ->				Tuple[str]:




										'''simple docstring'''



										if not os.path.isdir(lowerCAmelCase__							):
																logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""							)
																return
										_UpperCamelCase      : int         =   os.path.join(
										    lowerCAmelCase__,			(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']							)

										if os.path.abspath(self.vocab_file							) != os.path.abspath(lowerCAmelCase__							) and os.path.isfile(self.vocab_file							):
																copyfile(self.vocab_file,			lowerCAmelCase__							)
										elif not os.path.isfile(self.vocab_file							):
																with open(lowerCAmelCase__,			'''wb'''							) as fi:
																						_UpperCamelCase      : Dict         =   self.sp_model.serialized_model_proto()
																						fi.write(lowerCAmelCase__							)

										return (out_vocab_file,)












"""simple docstring"""




from functools import lru_cache





@lru_cache
def 	a_    (							_lowercase       ):
						if num < 0:
												raise ValueError('''Number should not be negative.'''       )

						return 1 if num in (0, 1) else num * factorial(num - 1       )


if __name__ == "__main__":
						import doctest

						doctest.testmod()



