






import json
from typing import Iterator, List, Union

from tokenizers import AddedToken, Regex, Tokenizer, decoders, normalizers, pre_tokenizers, trainers
from tokenizers.implementations.base_tokenizer import BaseTokenizer
from tokenizers.models import Unigram
from tokenizers.processors import TemplateProcessing



class 				__UpperCAmelCase		(_UpperCAmelCase   ):







			def __init__(     self:  List[str]				,     UpperCAmelCase_:  str = "▁"				,     UpperCAmelCase_:  bool = True				,     UpperCAmelCase_:  Union[str, AddedToken] = "<unk>"				,     UpperCAmelCase_:  Union[str, AddedToken] = "</s>"				,     UpperCAmelCase_:  Union[str, AddedToken] = "<pad>"				,     ):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  {
									    """pad""": {"""id""": 0, """token""": pad_token},
									    """eos""": {"""id""": 1, """token""": eos_token},
									    """unk""": {"""id""": 2, """token""": unk_token},
									}

									_SCREAMING_SNAKE_CASE     				=  [None] * len(self.special_tokens					)
									for token_dict in self.special_tokens.values():
															_SCREAMING_SNAKE_CASE     				=  token_dict["""token"""]

									_SCREAMING_SNAKE_CASE     				=  Tokenizer(Unigram()					)

									_SCREAMING_SNAKE_CASE     				=  normalizers.Sequence(
									    [
									        normalizers.Nmt(),
									        normalizers.NFKC(),
									        normalizers.Replace(Regex(""" {2,}"""					)				,     """ """					),
									        normalizers.Lowercase(),
									    ]					)
									_SCREAMING_SNAKE_CASE     				=  pre_tokenizers.Sequence(
									    [
									        pre_tokenizers.Metaspace(replacement=UpperCAmelCase_				,     add_prefix_space=UpperCAmelCase_					),
									        pre_tokenizers.Digits(individual_digits=UpperCAmelCase_					),
									        pre_tokenizers.Punctuation(),
									    ]					)
									_SCREAMING_SNAKE_CASE     				=  decoders.Metaspace(replacement=UpperCAmelCase_				,     add_prefix_space=UpperCAmelCase_					)

									_SCREAMING_SNAKE_CASE     				=  TemplateProcessing(
									    single=F'$A {self.special_tokens["eos"]["token"]}'				,     special_tokens=[(self.special_tokens["""eos"""]["""token"""], self.special_tokens["""eos"""]["""id"""])]				,     )

									_SCREAMING_SNAKE_CASE     				=  {
									    """model""": """SentencePieceUnigram""",
									    """replacement""": replacement,
									    """add_prefix_space""": add_prefix_space,
									}

									super().__init__(UpperCAmelCase_				,     UpperCAmelCase_					)







			def 							UpperCamelCase				(     self:  int				,     UpperCAmelCase_:  Union[str, List[str]]				,     UpperCAmelCase_:  int = 8_000				,     UpperCAmelCase_:  bool = True				,     ):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  trainers.UnigramTrainer(
									    vocab_size=UpperCAmelCase_				,     special_tokens=self.special_tokens_list				,     show_progress=UpperCAmelCase_				,     )

									if isinstance(UpperCAmelCase_				,     UpperCAmelCase_					):
															_SCREAMING_SNAKE_CASE     				=  [files]
									self._tokenizer.train(UpperCAmelCase_				,     trainer=UpperCAmelCase_					)

									self.add_unk_id()







			def 							UpperCamelCase				(     self:  List[Any]				,     UpperCAmelCase_:  Union[Iterator[str], Iterator[Iterator[str]]]				,     UpperCAmelCase_:  int = 8_000				,     UpperCAmelCase_:  bool = True				,     ):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  trainers.UnigramTrainer(
									    vocab_size=UpperCAmelCase_				,     special_tokens=self.special_tokens_list				,     show_progress=UpperCAmelCase_				,     )

									self._tokenizer.train_from_iterator(UpperCAmelCase_				,     trainer=UpperCAmelCase_					)

									self.add_unk_id()







			def 							UpperCamelCase				(     self:  List[Any]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  json.loads(self._tokenizer.to_str()					)

									_SCREAMING_SNAKE_CASE     				=  self.special_tokens["""unk"""]["""id"""]

									_SCREAMING_SNAKE_CASE     				=  Tokenizer.from_str(json.dumps(UpperCAmelCase_					)					)









from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
    is_vision_available,
)


UpperCamelCase			    =     {
    '''configuration_layoutlmv3''': [
        '''LAYOUTLMV3_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''LayoutLMv3Config''',
        '''LayoutLMv3OnnxConfig''',
    ],
    '''processing_layoutlmv3''': ['''LayoutLMv3Processor'''],
    '''tokenization_layoutlmv3''': ['''LayoutLMv3Tokenizer'''],
}

try:
		if not is_tokenizers_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		UpperCamelCase			    =     ['''LayoutLMv3TokenizerFast''']

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		UpperCamelCase			    =     [
		    '''LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST''',
		    '''LayoutLMv3ForQuestionAnswering''',
		    '''LayoutLMv3ForSequenceClassification''',
		    '''LayoutLMv3ForTokenClassification''',
		    '''LayoutLMv3Model''',
		    '''LayoutLMv3PreTrainedModel''',
		]

try:
		if not is_tf_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		UpperCamelCase			    =     [
		    '''TF_LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST''',
		    '''TFLayoutLMv3ForQuestionAnswering''',
		    '''TFLayoutLMv3ForSequenceClassification''',
		    '''TFLayoutLMv3ForTokenClassification''',
		    '''TFLayoutLMv3Model''',
		    '''TFLayoutLMv3PreTrainedModel''',
		]

try:
		if not is_vision_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		UpperCamelCase			    =     ['''LayoutLMv3FeatureExtractor''']
		UpperCamelCase			    =     ['''LayoutLMv3ImageProcessor''']


if TYPE_CHECKING:
		from .configuration_layoutlmva import (
		    LAYOUTLMV3_PRETRAINED_CONFIG_ARCHIVE_MAP,
		    LayoutLMvaConfig,
		    LayoutLMvaOnnxConfig,
		)
		from .processing_layoutlmva import LayoutLMvaProcessor
		from .tokenization_layoutlmva import LayoutLMvaTokenizer

		try:
				if not is_tokenizers_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .tokenization_layoutlmva_fast import LayoutLMvaTokenizerFast

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_layoutlmva import (
				    LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST,
				    LayoutLMvaForQuestionAnswering,
				    LayoutLMvaForSequenceClassification,
				    LayoutLMvaForTokenClassification,
				    LayoutLMvaModel,
				    LayoutLMvaPreTrainedModel,
				)

		try:
				if not is_tf_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_tf_layoutlmva import (
				    TF_LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST,
				    TFLayoutLMvaForQuestionAnswering,
				    TFLayoutLMvaForSequenceClassification,
				    TFLayoutLMvaForTokenClassification,
				    TFLayoutLMvaModel,
				    TFLayoutLMvaPreTrainedModel,
				)

		try:
				if not is_vision_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .feature_extraction_layoutlmva import LayoutLMvaFeatureExtractor
				from .image_processing_layoutlmva import LayoutLMvaImageProcessor

else:
		import sys

		UpperCamelCase			    =     _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)

