






import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCamelCase			    =     logging.get_logger(__name__)

UpperCamelCase			    =     '''▁'''

UpperCamelCase			    =     {'''vocab_file''': '''vocab.txt''', '''sentencepiece_model_ckpt''': '''sentencepiece.bpe.model'''}

UpperCamelCase			    =     {
    '''sentencepiece_model_file''': '''sentencepiece.bpe.model''',
    '''vocab_file''': '''vocab.txt''',
}

UpperCamelCase			    =     {
    '''vocab_file''': {
        '''ernie-m-base''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt''',
        '''ernie-m-large''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt''',
    },
    '''sentencepiece_model_file''': {
        '''ernie-m-base''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model''',
        '''ernie-m-large''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model''',
    },
}

UpperCamelCase			    =     {
    '''ernie-m-base''': 514,
    '''ernie-m-large''': 514,
}

UpperCamelCase			    =     {
    '''ernie-m-base''': {'''do_lower_case''': False},
    '''ernie-m-large''': {'''do_lower_case''': False},
}



class 				__UpperCAmelCase		(_UpperCAmelCase   ):
			__snake_case    :				List[str]									=							["input_ids"]

			__snake_case    :				Tuple									=							VOCAB_FILES_NAMES
			__snake_case    :				int									=							PRETRAINED_INIT_CONFIGURATION
			__snake_case    :				int									=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			__snake_case    :				List[str]									=							PRETRAINED_VOCAB_FILES_MAP
			__snake_case    :				List[Any]									=							RESOURCE_FILES_NAMES







			def __init__(     self:  List[Any]				,     UpperCAmelCase_:  Optional[Any]				,     UpperCAmelCase_:  Optional[int]=None				,     UpperCAmelCase_:  Union[str, Any]=False				,     UpperCAmelCase_:  Any="utf8"				,     UpperCAmelCase_:  str="[UNK]"				,     UpperCAmelCase_:  Any="[SEP]"				,     UpperCAmelCase_:  Optional[int]="[PAD]"				,     UpperCAmelCase_:  List[str]="[CLS]"				,     UpperCAmelCase_:  int="[MASK]"				,     UpperCAmelCase_:  Optional[Dict[str, Any]] = None				,     **UpperCAmelCase_:  Union[str, Any]				,     ):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  {} if sp_model_kwargs is None else sp_model_kwargs
									super().__init__(
									    do_lower_case=UpperCAmelCase_				,     unk_token=UpperCAmelCase_				,     sep_token=UpperCAmelCase_				,     pad_token=UpperCAmelCase_				,     cls_token=UpperCAmelCase_				,     mask_token=UpperCAmelCase_				,     vocab_file=UpperCAmelCase_				,     encoding=UpperCAmelCase_				,     sp_model_kwargs=self.sp_model_kwargs				,     **UpperCAmelCase_				,     )
									_SCREAMING_SNAKE_CASE     				=  do_lower_case
									_SCREAMING_SNAKE_CASE     				=  sentencepiece_model_ckpt
									_SCREAMING_SNAKE_CASE     				=  spm.SentencePieceProcessor(**self.sp_model_kwargs					)
									self.sp_model.Load(UpperCAmelCase_					)

									# to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
									if vocab_file is not None:
															_SCREAMING_SNAKE_CASE     				=  self.load_vocab(filepath=UpperCAmelCase_					)
									else:
															_SCREAMING_SNAKE_CASE     				=  {self.sp_model.id_to_piece(UpperCAmelCase_					): id for id in range(self.sp_model.get_piece_size()					)}
									_SCREAMING_SNAKE_CASE     				=  {v: k for k, v in self.vocab.items()}







			def 							UpperCamelCase				(     self:  str				,     UpperCAmelCase_:  Any					):







									'''simple docstring'''




									if text is None:
															return None

									_SCREAMING_SNAKE_CASE     				=  self.tokenize(UpperCAmelCase_					)
									_SCREAMING_SNAKE_CASE				,					_SCREAMING_SNAKE_CASE     				=  """""", []

									for i, ch in enumerate(UpperCAmelCase_					):
															if ch in self.SP_CHAR_MAPPING:
																					_SCREAMING_SNAKE_CASE     				=  self.SP_CHAR_MAPPING.get(UpperCAmelCase_					)
															else:
																					_SCREAMING_SNAKE_CASE     				=  unicodedata.normalize("""NFKC"""				,     UpperCAmelCase_					)
															if self.is_whitespace(UpperCAmelCase_					):
																					continue
															normalized_text += ch
															char_mapping.extend([i] * len(UpperCAmelCase_					)					)

									_SCREAMING_SNAKE_CASE				,					_SCREAMING_SNAKE_CASE				,					_SCREAMING_SNAKE_CASE     				=  normalized_text, [], 0

									if self.do_lower_case:
															_SCREAMING_SNAKE_CASE     				=  text.lower()

									for token in split_tokens:
															if token[:1] == "▁":
																					_SCREAMING_SNAKE_CASE     				=  token[1:]
															_SCREAMING_SNAKE_CASE     				=  text[offset:].index(UpperCAmelCase_					) + offset
															_SCREAMING_SNAKE_CASE     				=  start + len(UpperCAmelCase_					)

															token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)					)
															_SCREAMING_SNAKE_CASE     				=  end
									return token_mapping







			@property
			def 							UpperCamelCase				(     self:  List[str]					):







									'''simple docstring'''




									return len(self.vocab					)







			def 							UpperCamelCase				(     self:  Any					):







									'''simple docstring'''




									return dict(self.vocab				,     **self.added_tokens_encoder					)







			def __getstate__(     self:  Union[str, Any]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  self.__dict__.copy()
									_SCREAMING_SNAKE_CASE     				=  None
									return state







			def __setstate__(     self:  Any				,     UpperCAmelCase_:  List[Any]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  d

									# for backward compatibility
									if not hasattr(self				,     """sp_model_kwargs"""					):
															_SCREAMING_SNAKE_CASE     				=  {}

									_SCREAMING_SNAKE_CASE     				=  spm.SentencePieceProcessor(**self.sp_model_kwargs					)
									self.sp_model.Load(self.sentencepiece_model_ckpt					)







			def 							UpperCamelCase				(     self:  int				,     UpperCAmelCase_:  List[str]					):







									'''simple docstring'''




									return "".join((self.SP_CHAR_MAPPING.get(UpperCAmelCase_				,     UpperCAmelCase_					) for c in text)					)







			def 							UpperCamelCase				(     self:  Optional[Any]				,     UpperCAmelCase_:  Union[str, Any]				,     UpperCAmelCase_:  Union[str, Any]=False				,     UpperCAmelCase_:  Any=64				,     UpperCAmelCase_:  List[Any]=0.1					):







									'''simple docstring'''




									if self.sp_model_kwargs.get("""enable_sampling"""					) is True:
															_SCREAMING_SNAKE_CASE     				=  True
									if self.sp_model_kwargs.get("""alpha"""					) is not None:
															_SCREAMING_SNAKE_CASE     				=  self.sp_model_kwargs.get("""alpha"""					)
									if self.sp_model_kwargs.get("""nbest_size"""					) is not None:
															_SCREAMING_SNAKE_CASE     				=  self.sp_model_kwargs.get("""nbest_size"""					)

									if not enable_sampling:
															_SCREAMING_SNAKE_CASE     				=  self.sp_model.EncodeAsPieces(UpperCAmelCase_					)
									else:
															_SCREAMING_SNAKE_CASE     				=  self.sp_model.SampleEncodeAsPieces(UpperCAmelCase_				,     UpperCAmelCase_				,     UpperCAmelCase_					)
									_SCREAMING_SNAKE_CASE     				=  []
									for pi, piece in enumerate(UpperCAmelCase_					):
															if piece == SPIECE_UNDERLINE:
																					if not pieces[pi + 1].startswith(UpperCAmelCase_					) and pi != 0:
																											new_pieces.append(UpperCAmelCase_					)
																											continue
																					else:
																											continue
															_SCREAMING_SNAKE_CASE     				=  0
															for i, chunk in enumerate(UpperCAmelCase_					):
																					if chunk == SPIECE_UNDERLINE:
																											continue
																					if self.is_ch_char(UpperCAmelCase_					) or self.is_punct(UpperCAmelCase_					):
																											if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																																	new_pieces.append(piece[lst_i:i]					)
																											new_pieces.append(UpperCAmelCase_					)
																											_SCREAMING_SNAKE_CASE     				=  i + 1
																					elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
																											if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																																	new_pieces.append(piece[lst_i:i]					)
																											_SCREAMING_SNAKE_CASE     				=  i
																					elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
																											if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																																	new_pieces.append(piece[lst_i:i]					)
																											_SCREAMING_SNAKE_CASE     				=  i
															if len(UpperCAmelCase_					) > lst_i:
																					new_pieces.append(piece[lst_i:]					)
									return new_pieces







			def 							UpperCamelCase				(     self:  Union[str, Any]				,     UpperCAmelCase_:  List[Any]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  """""".join(UpperCAmelCase_					).replace(UpperCAmelCase_				,     """ """					).strip()
									return out_string







			def 							UpperCamelCase				(     self:  Tuple				,     UpperCAmelCase_:  str					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  self.convert_ids_to_tokens(UpperCAmelCase_					)
									_SCREAMING_SNAKE_CASE     				=  """""".join(UpperCAmelCase_					).replace(UpperCAmelCase_				,     """ """					).strip()
									return out_string







			def 							UpperCamelCase				(     self:  Optional[Any]				,     UpperCAmelCase_:  str					):







									'''simple docstring'''




									return self.vocab.get(UpperCAmelCase_				,     self.vocab.get(self.unk_token					)					)







			def 							UpperCamelCase				(     self:  Optional[int]				,     UpperCAmelCase_:  Optional[int]					):







									'''simple docstring'''




									return self.reverse_vocab.get(UpperCAmelCase_				,     self.unk_token					)







			def 							UpperCamelCase				(     self:  Union[str, Any]				,     UpperCAmelCase_:  str				,     UpperCAmelCase_:  Dict=None					):







									'''simple docstring'''




									if token_ids_a is None:
															return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
									_SCREAMING_SNAKE_CASE     				=  [self.cls_token_id]
									_SCREAMING_SNAKE_CASE     				=  [self.sep_token_id]
									return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep







			def 							UpperCamelCase				(     self:  List[str]				,     UpperCAmelCase_:  Union[str, Any]				,     UpperCAmelCase_:  Dict=None					):







									'''simple docstring'''




									if offset_mapping_a is None:
															return [(0, 0)] + offset_mapping_a + [(0, 0)]

									return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]







			def 							UpperCamelCase				(     self:  Tuple				,     UpperCAmelCase_:  List[Any]				,     UpperCAmelCase_:  Dict=None				,     UpperCAmelCase_:  Tuple=False					):







									'''simple docstring'''




									if already_has_special_tokens:
															if token_ids_a is not None:
																					raise ValueError(
																					    """You should not supply a second sequence if the provided sequence of """
																					    """ids is already formatted with special tokens for the model."""					)
															return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

									if token_ids_a is not None:
															return [1] + ([0] * len(UpperCAmelCase_					)) + [1, 1] + ([0] * len(UpperCAmelCase_					)) + [1]
									return [1] + ([0] * len(UpperCAmelCase_					)) + [1]







			def 							UpperCamelCase				(     self:  str				,     UpperCAmelCase_:  List[int]				,     UpperCAmelCase_:  Optional[List[int]] = None					):







									'''simple docstring'''




									if token_ids_a is None:
															# [CLS] X [SEP]
															return (len(UpperCAmelCase_					) + 2) * [0]

									# [CLS] A [SEP] [SEP] B [SEP]
									return [0] * (len(UpperCAmelCase_					) + 1) + [1] * (len(UpperCAmelCase_					) + 3)







			def 							UpperCamelCase				(     self:  int				,     UpperCAmelCase_:  str					):







									'''simple docstring'''




									if "\u4e00" <= char <= "\u9fff":
															return True
									return False







			def 							UpperCamelCase				(     self:  Optional[Any]				,     UpperCAmelCase_:  Optional[int]					):







									'''simple docstring'''




									if ("a" <= char <= "z") or ("A" <= char <= "Z"):
															return True
									return False







			def 							UpperCamelCase				(     self:  List[str]				,     UpperCAmelCase_:  str					):







									'''simple docstring'''




									if char in ",;:.?!~，；：。？！《》【】":
															return True
									return False







			def 							UpperCamelCase				(     self:  Union[str, Any]				,     UpperCAmelCase_:  str					):







									'''simple docstring'''




									if char == " " or char == "\t" or char == "\n" or char == "\r":
															return True
									if len(UpperCAmelCase_					) == 1:
															_SCREAMING_SNAKE_CASE     				=  unicodedata.category(UpperCAmelCase_					)
															if cat == "Zs":
																					return True
									return False







			def 							UpperCamelCase				(     self:  Optional[Any]				,     UpperCAmelCase_:  Tuple					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  {}
									with io.open(UpperCAmelCase_				,     """r"""				,     encoding="""utf-8"""					) as f:
															for index, line in enumerate(UpperCAmelCase_					):
																					_SCREAMING_SNAKE_CASE     				=  line.rstrip("""\n"""					)
																					_SCREAMING_SNAKE_CASE     				=  int(UpperCAmelCase_					)

									return token_to_idx







			def 							UpperCamelCase				(     self:  Union[str, Any]				,     UpperCAmelCase_:  str				,     UpperCAmelCase_:  Optional[str] = None					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  0
									if os.path.isdir(UpperCAmelCase_					):
															_SCREAMING_SNAKE_CASE     				=  os.path.join(
															    UpperCAmelCase_				,     (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]					)
									else:
															_SCREAMING_SNAKE_CASE     				=  (filename_prefix + """-""" if filename_prefix else """""") + save_directory
									with open(UpperCAmelCase_				,     """w"""				,     encoding="""utf-8"""					) as writer:
															for token, token_index in sorted(self.vocab.items()				,     key=lambda UpperCAmelCase_					: kv[1]					):
																					if index != token_index:
																											logger.warning(
																											    F'Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'
																											    """ Please check that the vocabulary is not corrupted!"""					)
																											_SCREAMING_SNAKE_CASE     				=  token_index
																					writer.write(token + """\n"""					)
																					index += 1

									_SCREAMING_SNAKE_CASE     				=  os.path.join(UpperCAmelCase_				,     """sentencepiece.bpe.model"""					)
									with open(UpperCAmelCase_				,     """wb"""					) as fi:
															_SCREAMING_SNAKE_CASE     				=  self.sp_model.serialized_model_proto()
															fi.write(UpperCAmelCase_					)

									return (vocab_file,)









from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
    is_vision_available,
)


UpperCamelCase			    =     {
    '''configuration_layoutlmv3''': [
        '''LAYOUTLMV3_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''LayoutLMv3Config''',
        '''LayoutLMv3OnnxConfig''',
    ],
    '''processing_layoutlmv3''': ['''LayoutLMv3Processor'''],
    '''tokenization_layoutlmv3''': ['''LayoutLMv3Tokenizer'''],
}

try:
		if not is_tokenizers_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		UpperCamelCase			    =     ['''LayoutLMv3TokenizerFast''']

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		UpperCamelCase			    =     [
		    '''LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST''',
		    '''LayoutLMv3ForQuestionAnswering''',
		    '''LayoutLMv3ForSequenceClassification''',
		    '''LayoutLMv3ForTokenClassification''',
		    '''LayoutLMv3Model''',
		    '''LayoutLMv3PreTrainedModel''',
		]

try:
		if not is_tf_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		UpperCamelCase			    =     [
		    '''TF_LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST''',
		    '''TFLayoutLMv3ForQuestionAnswering''',
		    '''TFLayoutLMv3ForSequenceClassification''',
		    '''TFLayoutLMv3ForTokenClassification''',
		    '''TFLayoutLMv3Model''',
		    '''TFLayoutLMv3PreTrainedModel''',
		]

try:
		if not is_vision_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		UpperCamelCase			    =     ['''LayoutLMv3FeatureExtractor''']
		UpperCamelCase			    =     ['''LayoutLMv3ImageProcessor''']


if TYPE_CHECKING:
		from .configuration_layoutlmva import (
		    LAYOUTLMV3_PRETRAINED_CONFIG_ARCHIVE_MAP,
		    LayoutLMvaConfig,
		    LayoutLMvaOnnxConfig,
		)
		from .processing_layoutlmva import LayoutLMvaProcessor
		from .tokenization_layoutlmva import LayoutLMvaTokenizer

		try:
				if not is_tokenizers_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .tokenization_layoutlmva_fast import LayoutLMvaTokenizerFast

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_layoutlmva import (
				    LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST,
				    LayoutLMvaForQuestionAnswering,
				    LayoutLMvaForSequenceClassification,
				    LayoutLMvaForTokenClassification,
				    LayoutLMvaModel,
				    LayoutLMvaPreTrainedModel,
				)

		try:
				if not is_tf_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_tf_layoutlmva import (
				    TF_LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST,
				    TFLayoutLMvaForQuestionAnswering,
				    TFLayoutLMvaForSequenceClassification,
				    TFLayoutLMvaForTokenClassification,
				    TFLayoutLMvaModel,
				    TFLayoutLMvaPreTrainedModel,
				)

		try:
				if not is_vision_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .feature_extraction_layoutlmva import LayoutLMvaFeatureExtractor
				from .image_processing_layoutlmva import LayoutLMvaImageProcessor

else:
		import sys

		UpperCamelCase			    =     _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)

