






"""simple docstring"""




import collections
import os
from typing import List, Optional, Tuple

from transformers.utils import is_jieba_available, requires_backends


if is_jieba_available():
		import jieba

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__UpperCAmelCase     							=						logging.get_logger(__name__)

__UpperCAmelCase     							=						{'vocab_file': 'vocab.txt'}

__UpperCAmelCase     							=						{
    'vocab_file': {
        'openbmb/cpm-ant-10b': 'https://huggingface.co/openbmb/cpm-ant-10b/blob/main/vocab.txt',
    },
}

__UpperCAmelCase     							=						{
    'openbmb/cpm-ant-10b': 1024,
}




def 					lowercase__				(					__snake_case   :							int  ):


					'''simple docstring'''


					UpperCAmelCase_   :				int						=						collections.OrderedDict()
					with open(__snake_case					,	'r'					,	encoding='utf-8'  ) as reader:
										UpperCAmelCase_   :				List[Any]						=						reader.readlines()
					for index, token in enumerate(__snake_case  ):
										UpperCAmelCase_   :				List[Any]						=						token.rstrip('\n'  )
										UpperCAmelCase_   :				Union[str, Any]						=						index
					return vocab



class   lowerCamelCase				(_snake_case      ):







		'''simple docstring'''



		def __init__(							self    ,		_UpperCamelCase    ,		_UpperCamelCase="<unk>"    ,		_UpperCamelCase=2_0_0       )     ->							Tuple:
							UpperCAmelCase_   :				Any						=						vocab
							UpperCAmelCase_   :				int						=						unk_token
							UpperCAmelCase_   :				Optional[int]						=						max_input_chars_per_word



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase       )     ->							Dict:
							UpperCAmelCase_   :				Tuple						=						list(_UpperCamelCase       )
							if len(_UpperCamelCase       ) > self.max_input_chars_per_word:
												return [self.unk_token]

							UpperCAmelCase_   :				Dict						=						0
							UpperCAmelCase_   :				Dict						=						[]
							while start < len(_UpperCamelCase       ):
												UpperCAmelCase_   :				List[str]						=						len(_UpperCamelCase       )
												UpperCAmelCase_   :				int						=						None
												while start < end:
																	UpperCAmelCase_   :				Union[str, Any]						=						''.join(chars[start:end]       )
																	if substr in self.vocab:
																						UpperCAmelCase_   :				Any						=						substr
																						break
																	end -= 1
												if cur_substr is None:
																	sub_tokens.append(self.unk_token       )
																	start += 1
												else:
																	sub_tokens.append(_UpperCamelCase       )
																	UpperCAmelCase_   :				List[Any]						=						end

							return sub_tokens





class   lowerCamelCase				(_snake_case      ):







		'''simple docstring'''



		_snake_case   :   Dict							=  VOCAB_FILES_NAMES
		_snake_case   :   List[str]							=  PRETRAINED_VOCAB_FILES_MAP
		_snake_case   :   List[str]							=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		_snake_case   :   Optional[Any]							=  ['''input_ids''', '''attention_mask''']
		_snake_case   :   Any							=  False



		def __init__(							self    ,		_UpperCamelCase    ,		_UpperCamelCase="<d>"    ,		_UpperCamelCase="</d>"    ,		_UpperCamelCase="<s>"    ,		_UpperCamelCase="</s>"    ,		_UpperCamelCase="<pad>"    ,		_UpperCamelCase="<unk>"    ,		_UpperCamelCase="</n>"    ,		_UpperCamelCase="</_>"    ,		_UpperCamelCase="left"    ,		**_UpperCamelCase    ,		)     ->							List[Any]:
							requires_backends(self    ,		['jieba']       )
							super().__init__(
							    bod_token=_UpperCamelCase    ,		eod_token=_UpperCamelCase    ,		bos_token=_UpperCamelCase    ,		eos_token=_UpperCamelCase    ,		pad_token=_UpperCamelCase    ,		unk_token=_UpperCamelCase    ,		line_token=_UpperCamelCase    ,		space_token=_UpperCamelCase    ,		padding_side=_UpperCamelCase    ,		**_UpperCamelCase    ,		)
							UpperCAmelCase_   :				Union[str, Any]						=						bod_token
							UpperCAmelCase_   :				Optional[Any]						=						eod_token
							UpperCAmelCase_   :				List[str]						=						load_vocab(_UpperCamelCase       )
							UpperCAmelCase_   :				Union[str, Any]						=						self.encoder[space_token]
							UpperCAmelCase_   :				Union[str, Any]						=						self.encoder[line_token]

							del self.encoder[space_token]
							del self.encoder[line_token]

							UpperCAmelCase_   :				Dict						=						collections.OrderedDict(sorted(self.encoder.items()    ,		key=lambda _UpperCamelCase       : x[1]       )       )
							UpperCAmelCase_   :				List[str]						=						{v: k for k, v in self.encoder.items()}

							UpperCAmelCase_   :				int						=						WordpieceTokenizer(vocab=self.encoder    ,		unk_token=self.unk_token       )



		@property
		def  __UpperCAmelCase      (							self       )     ->							int:
							return self.encoder[self.bod_token]



		@property
		def  __UpperCAmelCase      (							self       )     ->							Dict:
							return self.encoder[self.eod_token]



		@property
		def  __UpperCAmelCase      (							self       )     ->							Union[str, Any]:
							return self.encoder["\n"]



		@property
		def  __UpperCAmelCase      (							self       )     ->							int:
							return len(self.encoder       )



		def  __UpperCAmelCase      (							self       )     ->							int:
							return dict(self.encoder    ,		**self.added_tokens_encoder       )



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase       )     ->							int:
							UpperCAmelCase_   :				Optional[Any]						=						[]
							for x in jieba.cut(_UpperCamelCase    ,		cut_all=_UpperCamelCase       ):
												output_tokens.extend(self.wordpiece_tokenizer.tokenize(_UpperCamelCase       )       )
							return output_tokens



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase    ,		**_UpperCamelCase       )     ->							Optional[int]:
							UpperCAmelCase_   :				Dict						=						[i for i in token_ids if i >= 0]
							UpperCAmelCase_   :				str						=						[
							    x for x in token_ids if x != self.pad_token_id and x != self.eos_token_id and x != self.bos_token_id
							]
							return super()._decode(_UpperCamelCase    ,		**_UpperCamelCase       )



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase       )     ->							Optional[int]:
							return token in self.encoder



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase       )     ->							str:
							return "".join(_UpperCamelCase       )



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase       )     ->							str:
							return self.encoder.get(_UpperCamelCase    ,		self.encoder.get(self.unk_token       )       )



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase       )     ->							List[Any]:
							return self.decoder.get(_UpperCamelCase    ,		self.unk_token       )



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase    ,		_UpperCamelCase = None       )     ->							Tuple[str]:
							if os.path.isdir(_UpperCamelCase       ):
												UpperCAmelCase_   :				Any						=						os.path.join(
												    _UpperCamelCase    ,		(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']       )
							else:
												UpperCAmelCase_   :				Dict						=						(filename_prefix + '-' if filename_prefix else '') + save_directory
							UpperCAmelCase_   :				int						=						0
							if " " in self.encoder:
												UpperCAmelCase_   :				Union[str, Any]						=						self.encoder[' ']
												del self.encoder[" "]
							if "\n" in self.encoder:
												UpperCAmelCase_   :				Optional[int]						=						self.encoder['\n']
												del self.encoder["\n"]
							UpperCAmelCase_   :				Any						=						collections.OrderedDict(sorted(self.encoder.items()    ,		key=lambda _UpperCamelCase       : x[1]       )       )
							with open(_UpperCamelCase    ,		'w'    ,		encoding='utf-8'       ) as writer:
												for token, token_index in self.encoder.items():
																	if index != token_index:
																						logger.warning(
																						    f"Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."
																						    ' Please check that the vocabulary is not corrupted!'       )
																						UpperCAmelCase_   :				Optional[int]						=						token_index
																	writer.write(token + '\n'       )
																	index += 1
							return (vocab_file,)



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase    ,		_UpperCamelCase = None       )     ->							List[int]:
							if token_ids_a is None:
												return [self.bos_token_id] + token_ids_a
							return [self.bos_token_id] + token_ids_a + [self.bos_token_id] + token_ids_a



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase    ,		_UpperCamelCase = None    ,		_UpperCamelCase = False       )     ->							List[int]:

							if already_has_special_tokens:
												return super().get_special_tokens_mask(
												    token_ids_a=_UpperCamelCase    ,		token_ids_a=_UpperCamelCase    ,		already_has_special_tokens=_UpperCamelCase       )

							if token_ids_a is not None:
												return [1] + ([0] * len(_UpperCamelCase       )) + [1] + ([0] * len(_UpperCamelCase       ))
							return [1] + ([0] * len(_UpperCamelCase       ))














from collections import OrderedDict
from typing import TYPE_CHECKING, Any, Mapping, Optional

from packaging import version

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import logging


if TYPE_CHECKING:
		from ...processing_utils import ProcessorMixin
		from ...utils import TensorType


__UpperCAmelCase     							=						logging.get_logger(__name__)

__UpperCAmelCase     							=						{
    'microsoft/layoutlmv3-base': 'https://huggingface.co/microsoft/layoutlmv3-base/resolve/main/config.json',
}



class   lowerCamelCase				(_snake_case      ):







		'''simple docstring'''



		_snake_case   :   Dict							=  '''layoutlmv3'''



		def __init__(							self    ,		_UpperCamelCase=5_0_2_6_5    ,		_UpperCamelCase=7_6_8    ,		_UpperCamelCase=1_2    ,		_UpperCamelCase=1_2    ,		_UpperCamelCase=3_0_7_2    ,		_UpperCamelCase="gelu"    ,		_UpperCamelCase=0.1    ,		_UpperCamelCase=0.1    ,		_UpperCamelCase=5_1_2    ,		_UpperCamelCase=2    ,		_UpperCamelCase=0.02    ,		_UpperCamelCase=1E-5    ,		_UpperCamelCase=1    ,		_UpperCamelCase=0    ,		_UpperCamelCase=2    ,		_UpperCamelCase=1_0_2_4    ,		_UpperCamelCase=1_2_8    ,		_UpperCamelCase=1_2_8    ,		_UpperCamelCase=True    ,		_UpperCamelCase=3_2    ,		_UpperCamelCase=1_2_8    ,		_UpperCamelCase=6_4    ,		_UpperCamelCase=2_5_6    ,		_UpperCamelCase=True    ,		_UpperCamelCase=True    ,		_UpperCamelCase=True    ,		_UpperCamelCase=2_2_4    ,		_UpperCamelCase=3    ,		_UpperCamelCase=1_6    ,		_UpperCamelCase=None    ,		**_UpperCamelCase    ,		)     ->							Optional[Any]:
							super().__init__(
							    vocab_size=_UpperCamelCase    ,		hidden_size=_UpperCamelCase    ,		num_hidden_layers=_UpperCamelCase    ,		num_attention_heads=_UpperCamelCase    ,		intermediate_size=_UpperCamelCase    ,		hidden_act=_UpperCamelCase    ,		hidden_dropout_prob=_UpperCamelCase    ,		attention_probs_dropout_prob=_UpperCamelCase    ,		max_position_embeddings=_UpperCamelCase    ,		type_vocab_size=_UpperCamelCase    ,		initializer_range=_UpperCamelCase    ,		layer_norm_eps=_UpperCamelCase    ,		pad_token_id=_UpperCamelCase    ,		bos_token_id=_UpperCamelCase    ,		eos_token_id=_UpperCamelCase    ,		**_UpperCamelCase    ,		)
							UpperCAmelCase_   :				str						=						max_ad_position_embeddings
							UpperCAmelCase_   :				Union[str, Any]						=						coordinate_size
							UpperCAmelCase_   :				Union[str, Any]						=						shape_size
							UpperCAmelCase_   :				str						=						has_relative_attention_bias
							UpperCAmelCase_   :				Tuple						=						rel_pos_bins
							UpperCAmelCase_   :				Dict						=						max_rel_pos
							UpperCAmelCase_   :				Any						=						has_spatial_attention_bias
							UpperCAmelCase_   :				Optional[Any]						=						rel_ad_pos_bins
							UpperCAmelCase_   :				List[str]						=						max_rel_ad_pos
							UpperCAmelCase_   :				List[str]						=						text_embed
							UpperCAmelCase_   :				Dict						=						visual_embed
							UpperCAmelCase_   :				Optional[int]						=						input_size
							UpperCAmelCase_   :				List[Any]						=						num_channels
							UpperCAmelCase_   :				List[Any]						=						patch_size
							UpperCAmelCase_   :				Union[str, Any]						=						classifier_dropout





class   lowerCamelCase				(_snake_case      ):







		'''simple docstring'''



		_snake_case   :   Tuple							=  version.parse('''1.12'''      )



		@property
		def  __UpperCAmelCase      (							self       )     ->							Mapping[str, Mapping[int, str]]:
							# The order of inputs is different for question answering and sequence classification
							if self.task in ["question-answering", "sequence-classification"]:
												return OrderedDict(
												    [
												        ('input_ids', {0: 'batch', 1: 'sequence'}),
												        ('attention_mask', {0: 'batch', 1: 'sequence'}),
												        ('bbox', {0: 'batch', 1: 'sequence'}),
												        ('pixel_values', {0: 'batch', 1: 'num_channels', 2: 'height', 3: 'width'}),
												    ]       )
							else:
												return OrderedDict(
												    [
												        ('input_ids', {0: 'batch', 1: 'sequence'}),
												        ('bbox', {0: 'batch', 1: 'sequence'}),
												        ('attention_mask', {0: 'batch', 1: 'sequence'}),
												        ('pixel_values', {0: 'batch', 1: 'num_channels'}),
												    ]       )



		@property
		def  __UpperCAmelCase      (							self       )     ->							float:
							return 1E-5



		@property
		def  __UpperCAmelCase      (							self       )     ->							int:
							return 1_2



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase    ,		_UpperCamelCase = -1    ,		_UpperCamelCase = -1    ,		_UpperCamelCase = False    ,		_UpperCamelCase = None    ,		_UpperCamelCase = 3    ,		_UpperCamelCase = 4_0    ,		_UpperCamelCase = 4_0    ,		)     ->							Mapping[str, Any]:
							setattr(processor.image_processor    ,		'apply_ocr'    ,		_UpperCamelCase       )

							# If dynamic axis (-1) we forward with a fixed dimension of 2 samples to avoid optimizations made by ONNX
							UpperCAmelCase_   :				Optional[int]						=						compute_effective_axis_dimension(
							    _UpperCamelCase    ,		fixed_dimension=OnnxConfig.default_fixed_batch    ,		num_token_to_add=0       )
							# If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
							UpperCAmelCase_   :				Any						=						processor.tokenizer.num_special_tokens_to_add(_UpperCamelCase       )
							UpperCAmelCase_   :				Any						=						compute_effective_axis_dimension(
							    _UpperCamelCase    ,		fixed_dimension=OnnxConfig.default_fixed_sequence    ,		num_token_to_add=_UpperCamelCase       )
							# Generate dummy inputs according to compute batch and sequence
							UpperCAmelCase_   :				Tuple						=						[[' '.join([processor.tokenizer.unk_token]       ) * seq_length]] * batch_size

							# Generate dummy bounding boxes
							UpperCAmelCase_   :				List[str]						=						[[[4_8, 8_4, 7_3, 1_2_8]]] * batch_size

							# If dynamic axis (-1) we forward with a fixed dimension of 2 samples to avoid optimizations made by ONNX
							# batch_size = compute_effective_axis_dimension(batch_size, fixed_dimension=OnnxConfig.default_fixed_batch)
							UpperCAmelCase_   :				str						=						self._generate_dummy_images(_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase       )

							UpperCAmelCase_   :				List[str]						=						dict(
							    processor(
							        _UpperCamelCase    ,		text=_UpperCamelCase    ,		boxes=_UpperCamelCase    ,		return_tensors=_UpperCamelCase    ,		)       )

							return inputs






