





"""simple docstring"""




import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCamelCase_									=logging.get_logger(__name__)

UpperCamelCase_									="""▁"""

UpperCamelCase_									={"""vocab_file""": """vocab.txt""", """sentencepiece_model_ckpt""": """sentencepiece.bpe.model"""}

UpperCamelCase_									={
    """sentencepiece_model_file""": """sentencepiece.bpe.model""",
    """vocab_file""": """vocab.txt""",
}

UpperCamelCase_									={
    """vocab_file""": {
        """ernie-m-base""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt""",
        """ernie-m-large""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt""",
    },
    """sentencepiece_model_file""": {
        """ernie-m-base""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model""",
        """ernie-m-large""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model""",
    },
}

UpperCamelCase_									={
    """ernie-m-base""": 514,
    """ernie-m-large""": 514,
}

UpperCamelCase_									={
    """ernie-m-base""": {"""do_lower_case""": False},
    """ernie-m-large""": {"""do_lower_case""": False},
}




class 			_a						(				_lowerCAmelCase    ):
				UpperCamelCase								=   ["input_ids"]

				UpperCamelCase								=   VOCAB_FILES_NAMES
				UpperCamelCase								=   PRETRAINED_INIT_CONFIGURATION
				UpperCamelCase								=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				UpperCamelCase								=   PRETRAINED_VOCAB_FILES_MAP
				UpperCamelCase								=   RESOURCE_FILES_NAMES






				def __init__(			self		:   Any,			lowerCAmelCase__		:   int,			lowerCAmelCase__		:   List[str]=None,			lowerCAmelCase__		:   List[Any]=False,			lowerCAmelCase__		:   Union[str, Any]="utf8",			lowerCAmelCase__		:   str="[UNK]",			lowerCAmelCase__		:   Optional[int]="[SEP]",			lowerCAmelCase__		:   str="[PAD]",			lowerCAmelCase__		:   str="[CLS]",			lowerCAmelCase__		:   str="[MASK]",			lowerCAmelCase__		:   Optional[Dict[str, Any]] = None,			**lowerCAmelCase__		:   Optional[Any],			)     ->				None:




										'''simple docstring'''



										_UpperCamelCase      : str         =   {} if sp_model_kwargs is None else sp_model_kwargs
										super().__init__(
										    do_lower_case=lowerCAmelCase__,			unk_token=lowerCAmelCase__,			sep_token=lowerCAmelCase__,			pad_token=lowerCAmelCase__,			cls_token=lowerCAmelCase__,			mask_token=lowerCAmelCase__,			vocab_file=lowerCAmelCase__,			encoding=lowerCAmelCase__,			sp_model_kwargs=self.sp_model_kwargs,			**lowerCAmelCase__,			)
										_UpperCamelCase      : Tuple         =   do_lower_case
										_UpperCamelCase      : Dict         =   sentencepiece_model_ckpt
										_UpperCamelCase      : str         =   spm.SentencePieceProcessor(**self.sp_model_kwargs							)
										self.sp_model.Load(lowerCAmelCase__							)

										# to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
										if vocab_file is not None:
																_UpperCamelCase      : List[Any]         =   self.load_vocab(filepath=lowerCAmelCase__							)
										else:
																_UpperCamelCase      : int         =   {self.sp_model.id_to_piece(lowerCAmelCase__							): id for id in range(self.sp_model.get_piece_size()							)}
										_UpperCamelCase      : List[str]         =   {v: k for k, v in self.vocab.items()}






				def     snake_case     (			self		:   Optional[int],			lowerCAmelCase__		:   Union[str, Any]							)     ->				str:




										'''simple docstring'''



										if text is None:
																return None

										_UpperCamelCase      : Optional[int]         =   self.tokenize(lowerCAmelCase__							)
										_UpperCamelCase       ,							_UpperCamelCase      : Optional[Any]         =   '''''', []

										for i, ch in enumerate(lowerCAmelCase__							):
																if ch in self.SP_CHAR_MAPPING:
																						_UpperCamelCase      : int         =   self.SP_CHAR_MAPPING.get(lowerCAmelCase__							)
																else:
																						_UpperCamelCase      : Optional[int]         =   unicodedata.normalize('''NFKC''',			lowerCAmelCase__							)
																if self.is_whitespace(lowerCAmelCase__							):
																						continue
																normalized_text += ch
																char_mapping.extend([i] * len(lowerCAmelCase__							)							)

										_UpperCamelCase       ,							_UpperCamelCase       ,							_UpperCamelCase      : Tuple         =   normalized_text, [], 0

										if self.do_lower_case:
																_UpperCamelCase      : Union[str, Any]         =   text.lower()

										for token in split_tokens:
																if token[:1] == "▁":
																						_UpperCamelCase      : List[str]         =   token[1:]
																_UpperCamelCase      : Tuple         =   text[offset:].index(lowerCAmelCase__							) + offset
																_UpperCamelCase      : List[Any]         =   start + len(lowerCAmelCase__							)

																token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)							)
																_UpperCamelCase      : List[Any]         =   end
										return token_mapping






				@property
				def     snake_case     (			self		:   Tuple							)     ->				Dict:




										'''simple docstring'''



										return len(self.vocab							)






				def     snake_case     (			self		:   Any							)     ->				List[str]:




										'''simple docstring'''



										return dict(self.vocab,			**self.added_tokens_encoder							)






				def __getstate__(			self		:   Any							)     ->				int:




										'''simple docstring'''



										_UpperCamelCase      : List[Any]         =   self.__dict__.copy()
										_UpperCamelCase      : List[str]         =   None
										return state






				def __setstate__(			self		:   Dict,			lowerCAmelCase__		:   int							)     ->				str:




										'''simple docstring'''



										_UpperCamelCase      : List[str]         =   d

										# for backward compatibility
										if not hasattr(self,			'''sp_model_kwargs'''							):
																_UpperCamelCase      : Optional[int]         =   {}

										_UpperCamelCase      : Union[str, Any]         =   spm.SentencePieceProcessor(**self.sp_model_kwargs							)
										self.sp_model.Load(self.sentencepiece_model_ckpt							)






				def     snake_case     (			self		:   Optional[int],			lowerCAmelCase__		:   List[str]							)     ->				Optional[Any]:




										'''simple docstring'''



										return "".join((self.SP_CHAR_MAPPING.get(lowerCAmelCase__,			lowerCAmelCase__							) for c in text)							)






				def     snake_case     (			self		:   str,			lowerCAmelCase__		:   Union[str, Any],			lowerCAmelCase__		:   str=False,			lowerCAmelCase__		:   Tuple=6_4,			lowerCAmelCase__		:   List[str]=0.1							)     ->				List[str]:




										'''simple docstring'''



										if self.sp_model_kwargs.get('''enable_sampling'''							) is True:
																_UpperCamelCase      : str         =   True
										if self.sp_model_kwargs.get('''alpha'''							) is not None:
																_UpperCamelCase      : Union[str, Any]         =   self.sp_model_kwargs.get('''alpha'''							)
										if self.sp_model_kwargs.get('''nbest_size'''							) is not None:
																_UpperCamelCase      : Any         =   self.sp_model_kwargs.get('''nbest_size'''							)

										if not enable_sampling:
																_UpperCamelCase      : List[str]         =   self.sp_model.EncodeAsPieces(lowerCAmelCase__							)
										else:
																_UpperCamelCase      : Optional[int]         =   self.sp_model.SampleEncodeAsPieces(lowerCAmelCase__,			lowerCAmelCase__,			lowerCAmelCase__							)
										_UpperCamelCase      : List[str]         =   []
										for pi, piece in enumerate(lowerCAmelCase__							):
																if piece == SPIECE_UNDERLINE:
																						if not pieces[pi + 1].startswith(lowerCAmelCase__							) and pi != 0:
																												new_pieces.append(lowerCAmelCase__							)
																												continue
																						else:
																												continue
																_UpperCamelCase      : List[str]         =   0
																for i, chunk in enumerate(lowerCAmelCase__							):
																						if chunk == SPIECE_UNDERLINE:
																												continue
																						if self.is_ch_char(lowerCAmelCase__							) or self.is_punct(lowerCAmelCase__							):
																												if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																																		new_pieces.append(piece[lst_i:i]							)
																												new_pieces.append(lowerCAmelCase__							)
																												_UpperCamelCase      : int         =   i + 1
																						elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
																												if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																																		new_pieces.append(piece[lst_i:i]							)
																												_UpperCamelCase      : Optional[Any]         =   i
																						elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
																												if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																																		new_pieces.append(piece[lst_i:i]							)
																												_UpperCamelCase      : Any         =   i
																if len(lowerCAmelCase__							) > lst_i:
																						new_pieces.append(piece[lst_i:]							)
										return new_pieces






				def     snake_case     (			self		:   Dict,			lowerCAmelCase__		:   int							)     ->				Optional[int]:




										'''simple docstring'''



										_UpperCamelCase      : str         =   ''''''.join(lowerCAmelCase__							).replace(lowerCAmelCase__,			''' '''							).strip()
										return out_string






				def     snake_case     (			self		:   Dict,			lowerCAmelCase__		:   List[str]							)     ->				Optional[int]:




										'''simple docstring'''



										_UpperCamelCase      : Dict         =   self.convert_ids_to_tokens(lowerCAmelCase__							)
										_UpperCamelCase      : str         =   ''''''.join(lowerCAmelCase__							).replace(lowerCAmelCase__,			''' '''							).strip()
										return out_string






				def     snake_case     (			self		:   str,			lowerCAmelCase__		:   Union[str, Any]							)     ->				str:




										'''simple docstring'''



										return self.vocab.get(lowerCAmelCase__,			self.vocab.get(self.unk_token							)							)






				def     snake_case     (			self		:   List[Any],			lowerCAmelCase__		:   Any							)     ->				Dict:




										'''simple docstring'''



										return self.reverse_vocab.get(lowerCAmelCase__,			self.unk_token							)






				def     snake_case     (			self		:   Any,			lowerCAmelCase__		:   List[Any],			lowerCAmelCase__		:   Optional[Any]=None							)     ->				Union[str, Any]:




										'''simple docstring'''



										if token_ids_a is None:
																return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
										_UpperCamelCase      : List[str]         =   [self.cls_token_id]
										_UpperCamelCase      : str         =   [self.sep_token_id]
										return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep






				def     snake_case     (			self		:   List[Any],			lowerCAmelCase__		:   Optional[Any],			lowerCAmelCase__		:   List[str]=None							)     ->				Tuple:




										'''simple docstring'''



										if offset_mapping_a is None:
																return [(0, 0)] + offset_mapping_a + [(0, 0)]

										return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]






				def     snake_case     (			self		:   Any,			lowerCAmelCase__		:   Any,			lowerCAmelCase__		:   List[str]=None,			lowerCAmelCase__		:   int=False							)     ->				Union[str, Any]:




										'''simple docstring'''



										if already_has_special_tokens:
																if token_ids_a is not None:
																						raise ValueError(
																						    '''You should not supply a second sequence if the provided sequence of '''
																						    '''ids is already formatted with special tokens for the model.'''							)
																return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

										if token_ids_a is not None:
																return [1] + ([0] * len(lowerCAmelCase__							)) + [1, 1] + ([0] * len(lowerCAmelCase__							)) + [1]
										return [1] + ([0] * len(lowerCAmelCase__							)) + [1]






				def     snake_case     (			self		:   int,			lowerCAmelCase__		:   List[int],			lowerCAmelCase__		:   Optional[List[int]] = None							)     ->				List[int]:




										'''simple docstring'''



										if token_ids_a is None:
																# [CLS] X [SEP]
																return (len(lowerCAmelCase__							) + 2) * [0]

										# [CLS] A [SEP] [SEP] B [SEP]
										return [0] * (len(lowerCAmelCase__							) + 1) + [1] * (len(lowerCAmelCase__							) + 3)






				def     snake_case     (			self		:   str,			lowerCAmelCase__		:   Optional[int]							)     ->				int:




										'''simple docstring'''



										if "\u4e00" <= char <= "\u9fff":
																return True
										return False






				def     snake_case     (			self		:   Tuple,			lowerCAmelCase__		:   Tuple							)     ->				List[str]:




										'''simple docstring'''



										if ("a" <= char <= "z") or ("A" <= char <= "Z"):
																return True
										return False






				def     snake_case     (			self		:   Any,			lowerCAmelCase__		:   Optional[Any]							)     ->				Union[str, Any]:




										'''simple docstring'''



										if char in ",;:.?!~，；：。？！《》【】":
																return True
										return False






				def     snake_case     (			self		:   Any,			lowerCAmelCase__		:   int							)     ->				str:




										'''simple docstring'''



										if char == " " or char == "\t" or char == "\n" or char == "\r":
																return True
										if len(lowerCAmelCase__							) == 1:
																_UpperCamelCase      : Tuple         =   unicodedata.category(lowerCAmelCase__							)
																if cat == "Zs":
																						return True
										return False






				def     snake_case     (			self		:   Tuple,			lowerCAmelCase__		:   int							)     ->				List[str]:




										'''simple docstring'''



										_UpperCamelCase      : Dict         =   {}
										with io.open(lowerCAmelCase__,			'''r''',			encoding='''utf-8'''							) as f:
																for index, line in enumerate(lowerCAmelCase__							):
																						_UpperCamelCase      : Union[str, Any]         =   line.rstrip('''\n'''							)
																						_UpperCamelCase      : Optional[Any]         =   int(lowerCAmelCase__							)

										return token_to_idx






				def     snake_case     (			self		:   Optional[Any],			lowerCAmelCase__		:   str,			lowerCAmelCase__		:   Optional[str] = None							)     ->				Tuple[str]:




										'''simple docstring'''



										_UpperCamelCase      : Any         =   0
										if os.path.isdir(lowerCAmelCase__							):
																_UpperCamelCase      : Optional[Any]         =   os.path.join(
																    lowerCAmelCase__,			(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']							)
										else:
																_UpperCamelCase      : Union[str, Any]         =   (filename_prefix + '''-''' if filename_prefix else '''''') + save_directory
										with open(lowerCAmelCase__,			'''w''',			encoding='''utf-8'''							) as writer:
																for token, token_index in sorted(self.vocab.items(),			key=lambda lowerCAmelCase__							: kv[1]							):
																						if index != token_index:
																												logger.warning(
																												    f"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
																												    ''' Please check that the vocabulary is not corrupted!'''							)
																												_UpperCamelCase      : Union[str, Any]         =   token_index
																						writer.write(token + '''\n'''							)
																						index += 1

										_UpperCamelCase      : List[str]         =   os.path.join(lowerCAmelCase__,			'''sentencepiece.bpe.model'''							)
										with open(lowerCAmelCase__,			'''wb'''							) as fi:
																_UpperCamelCase      : str         =   self.sp_model.serialized_model_proto()
																fi.write(lowerCAmelCase__							)

										return (vocab_file,)










"""simple docstring"""




from __future__ import annotations

import math





def 	a_    (							_lowercase	,						_lowercase	,						_lowercase	,						_lowercase	,						_lowercase       ):

						if depth < 0:
												raise ValueError('''Depth cannot be less than 0'''       )

						if len(_lowercase       ) == 0:
												raise ValueError('''Scores cannot be empty'''       )

						if depth == height:
												return scores[node_index]

						if is_max:
												return max(
												    minimax(depth + 1	,						node_index * 2	,						_lowercase	,						_lowercase	,						_lowercase       )	,						minimax(depth + 1	,						node_index * 2 + 1	,						_lowercase	,						_lowercase	,						_lowercase       )	,						)

						return min(
						    minimax(depth + 1	,						node_index * 2	,						_lowercase	,						_lowercase	,						_lowercase       )	,						minimax(depth + 1	,						node_index * 2 + 1	,						_lowercase	,						_lowercase	,						_lowercase       )	,						)





def 	a_    (							):
						_UpperCamelCase      : Dict         =   [90, 23, 6, 33, 21, 65, 123, 3_4423]
						_UpperCamelCase      : int         =   math.log(len(_lowercase       )	,						2       )
						print('''Optimal value : '''	,						end=''''''       )
						print(minimax(0	,						0	,						_lowercase	,						_lowercase	,						_lowercase       )       )


if __name__ == "__main__":
						import doctest

						doctest.testmod()
						main()



