




'''simple docstring'''



import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCamelCase_			=						logging.get_logger(__name__)
lowerCamelCase_			=						{'vocab_file': 'spiece.model'}

lowerCamelCase_			=						{
    'vocab_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/spiece.model',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/spiece.model',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/spiece.model',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/spiece.model',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model',
    }
}

lowerCamelCase_			=						{
    'albert-base-v1': 5_12,
    'albert-large-v1': 5_12,
    'albert-xlarge-v1': 5_12,
    'albert-xxlarge-v1': 5_12,
    'albert-base-v2': 5_12,
    'albert-large-v2': 5_12,
    'albert-xlarge-v2': 5_12,
    'albert-xxlarge-v2': 5_12,
}

lowerCamelCase_			=						'▁'







class 		lowercase_   (			A       ):


   """simple docstring"""

   lowerCamelCase_      =						VOCAB_FILES_NAMES
   lowerCamelCase_      =						PRETRAINED_VOCAB_FILES_MAP
   lowerCamelCase_      =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES



   def __init__(   self :     str  ,				__lowerCamelCase :     str  ,				__lowerCamelCase :     Union[str, Any]=True  ,				__lowerCamelCase :     Any=True  ,				__lowerCamelCase :     Optional[int]=False  ,				__lowerCamelCase :     Any="[CLS]"  ,				__lowerCamelCase :     Optional[int]="[SEP]"  ,				__lowerCamelCase :     List[str]="<unk>"  ,				__lowerCamelCase :     str="[SEP]"  ,				__lowerCamelCase :     Union[str, Any]="<pad>"  ,				__lowerCamelCase :     Dict="[CLS]"  ,				__lowerCamelCase :     Any="[MASK]"  ,				__lowerCamelCase :     Optional[Dict[str, Any]] = None  ,				**__lowerCamelCase :     Dict  ,				):

      """simple docstring"""
      # Mask token behave like a normal word, i.e. include the space before it and
      # is included in the raw text, there should be a match in a non-normalized sentence.
      _SCREAMING_SNAKE_CASE	=				(
          AddedToken(__lowerCamelCase  ,				lstrip=__lowerCamelCase  ,				rstrip=__lowerCamelCase  ,				normalized=__lowerCamelCase   )
          if isinstance(__lowerCamelCase  ,				__lowerCamelCase   )
          else mask_token
      )

      _SCREAMING_SNAKE_CASE	=				{} if sp_model_kwargs is None else sp_model_kwargs

      super().__init__(
          do_lower_case=__lowerCamelCase  ,				remove_space=__lowerCamelCase  ,				keep_accents=__lowerCamelCase  ,				bos_token=__lowerCamelCase  ,				eos_token=__lowerCamelCase  ,				unk_token=__lowerCamelCase  ,				sep_token=__lowerCamelCase  ,				pad_token=__lowerCamelCase  ,				cls_token=__lowerCamelCase  ,				mask_token=__lowerCamelCase  ,				sp_model_kwargs=self.sp_model_kwargs  ,				**__lowerCamelCase  ,				)

      _SCREAMING_SNAKE_CASE	=				do_lower_case
      _SCREAMING_SNAKE_CASE	=				remove_space
      _SCREAMING_SNAKE_CASE	=				keep_accents
      _SCREAMING_SNAKE_CASE	=				vocab_file

      _SCREAMING_SNAKE_CASE	=				spm.SentencePieceProcessor(**self.sp_model_kwargs   )
      self.sp_model.Load(__lowerCamelCase   )



   @property
   def 							lowerCAmelCase_      (   self :     List[Any]   ):

      """simple docstring"""
      return len(self.sp_model   )



   def 							lowerCAmelCase_      (   self :     Tuple   ):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE	=				{self.convert_ids_to_tokens(__lowerCamelCase   ): i for i in range(self.vocab_size   )}
      vocab.update(self.added_tokens_encoder   )
      return vocab



   def __getstate__(   self :     str   ):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE	=				self.__dict__.copy()
      _SCREAMING_SNAKE_CASE	=				None
      return state



   def __setstate__(   self :     List[str]  ,				__lowerCamelCase :     Optional[int]   ):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE	=				d

      # for backward compatibility
      if not hasattr(self  ,				"sp_model_kwargs"   ):
         _SCREAMING_SNAKE_CASE	=				{}

      _SCREAMING_SNAKE_CASE	=				spm.SentencePieceProcessor(**self.sp_model_kwargs   )
      self.sp_model.Load(self.vocab_file   )



   def 							lowerCAmelCase_      (   self :     Union[str, Any]  ,				__lowerCamelCase :     List[Any]   ):

      """simple docstring"""
      if self.remove_space:
         _SCREAMING_SNAKE_CASE	=				" ".join(inputs.strip().split()   )
      else:
         _SCREAMING_SNAKE_CASE	=				inputs
      _SCREAMING_SNAKE_CASE	=				outputs.replace("``"  ,				"\""   ).replace("''"  ,				"\""   )

      if not self.keep_accents:
         _SCREAMING_SNAKE_CASE	=				unicodedata.normalize("NFKD"  ,				__lowerCamelCase   )
         _SCREAMING_SNAKE_CASE	=				"".join([c for c in outputs if not unicodedata.combining(__lowerCamelCase   )]   )
      if self.do_lower_case:
         _SCREAMING_SNAKE_CASE	=				outputs.lower()

      return outputs



   def 							lowerCAmelCase_      (   self :     int  ,				__lowerCamelCase :     str   ):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE	=				self.preprocess_text(__lowerCamelCase   )
      _SCREAMING_SNAKE_CASE	=				self.sp_model.encode(__lowerCamelCase  ,				out_type=__lowerCamelCase   )
      _SCREAMING_SNAKE_CASE	=				[]
      for piece in pieces:
         if len(__lowerCamelCase   ) > 1 and piece[-1] == str(","   ) and piece[-2].isdigit():
            _SCREAMING_SNAKE_CASE	=				self.sp_model.EncodeAsPieces(piece[:-1].replace(__lowerCamelCase  ,				""   )   )
            if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
               if len(cur_pieces[0]   ) == 1:
                  _SCREAMING_SNAKE_CASE	=				cur_pieces[1:]
               else:
                  _SCREAMING_SNAKE_CASE	=				cur_pieces[0][1:]
            cur_pieces.append(piece[-1]   )
            new_pieces.extend(__lowerCamelCase   )
         else:
            new_pieces.append(__lowerCamelCase   )

      return new_pieces



   def 							lowerCAmelCase_      (   self :     Optional[Any]  ,				__lowerCamelCase :     Union[str, Any]   ):

      """simple docstring"""
      return self.sp_model.PieceToId(__lowerCamelCase   )



   def 							lowerCAmelCase_      (   self :     Union[str, Any]  ,				__lowerCamelCase :     Tuple   ):

      """simple docstring"""
      return self.sp_model.IdToPiece(__lowerCamelCase   )



   def 							lowerCAmelCase_      (   self :     Optional[Any]  ,				__lowerCamelCase :     Optional[Any]   ):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE	=				[]
      _SCREAMING_SNAKE_CASE	=				""
      _SCREAMING_SNAKE_CASE	=				False
      for token in tokens:
         # make sure that special tokens are not decoded using sentencepiece model
         if token in self.all_special_tokens:
            if not prev_is_special:
               out_string += " "
            out_string += self.sp_model.decode(__lowerCamelCase   ) + token
            _SCREAMING_SNAKE_CASE	=				True
            _SCREAMING_SNAKE_CASE	=				[]
         else:
            current_sub_tokens.append(__lowerCamelCase   )
            _SCREAMING_SNAKE_CASE	=				False
      out_string += self.sp_model.decode(__lowerCamelCase   )
      return out_string.strip()



   def 							lowerCAmelCase_      (   self :     Optional[int]  ,				__lowerCamelCase :     List[int]  ,				__lowerCamelCase :     Optional[List[int]] = None   ):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE	=				[self.sep_token_id]
      _SCREAMING_SNAKE_CASE	=				[self.cls_token_id]
      if token_ids_a is None:
         return cls + token_ids_a + sep
      return cls + token_ids_a + sep + token_ids_a + sep



   def 							lowerCAmelCase_      (   self :     Any  ,				__lowerCamelCase :     List[int]  ,				__lowerCamelCase :     Optional[List[int]] = None  ,				__lowerCamelCase :     bool = False   ):

      """simple docstring"""

      if already_has_special_tokens:
         return super().get_special_tokens_mask(
             token_ids_a=__lowerCamelCase  ,				token_ids_a=__lowerCamelCase  ,				already_has_special_tokens=__lowerCamelCase   )

      if token_ids_a is not None:
         return [1] + ([0] * len(__lowerCamelCase   )) + [1] + ([0] * len(__lowerCamelCase   )) + [1]
      return [1] + ([0] * len(__lowerCamelCase   )) + [1]



   def 							lowerCAmelCase_      (   self :     int  ,				__lowerCamelCase :     List[int]  ,				__lowerCamelCase :     Optional[List[int]] = None   ):

      """simple docstring"""
      _SCREAMING_SNAKE_CASE	=				[self.sep_token_id]
      _SCREAMING_SNAKE_CASE	=				[self.cls_token_id]

      if token_ids_a is None:
         return len(cls + token_ids_a + sep   ) * [0]
      return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]



   def 							lowerCAmelCase_      (   self :     Optional[int]  ,				__lowerCamelCase :     str  ,				__lowerCamelCase :     Optional[str] = None   ):

      """simple docstring"""
      if not os.path.isdir(__lowerCamelCase   ):
         logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""   )
         return
      _SCREAMING_SNAKE_CASE	=				os.path.join(
          __lowerCamelCase  ,				(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]   )

      if os.path.abspath(self.vocab_file   ) != os.path.abspath(__lowerCamelCase   ) and os.path.isfile(self.vocab_file   ):
         copyfile(self.vocab_file  ,				__lowerCamelCase   )
      elif not os.path.isfile(self.vocab_file   ):
         with open(__lowerCamelCase  ,				"wb"   ) as fi:
            _SCREAMING_SNAKE_CASE	=				self.sp_model.serialized_model_proto()
            fi.write(__lowerCamelCase   )

      return (out_vocab_file,)







'''simple docstring'''



from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tokenizers_available,
    is_torch_available,
    is_vision_available,
)


lowerCamelCase_			=						{
    'configuration_layoutlmv2': ['LAYOUTLMV2_PRETRAINED_CONFIG_ARCHIVE_MAP', 'LayoutLMv2Config'],
    'processing_layoutlmv2': ['LayoutLMv2Processor'],
    'tokenization_layoutlmv2': ['LayoutLMv2Tokenizer'],
}

try:
 if not is_tokenizers_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 lowerCamelCase_			=						['LayoutLMv2TokenizerFast']

try:
 if not is_vision_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 lowerCamelCase_			=						['LayoutLMv2FeatureExtractor']
 lowerCamelCase_			=						['LayoutLMv2ImageProcessor']

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 lowerCamelCase_			=						[
     'LAYOUTLMV2_PRETRAINED_MODEL_ARCHIVE_LIST',
     'LayoutLMv2ForQuestionAnswering',
     'LayoutLMv2ForSequenceClassification',
     'LayoutLMv2ForTokenClassification',
     'LayoutLMv2Layer',
     'LayoutLMv2Model',
     'LayoutLMv2PreTrainedModel',
 ]

if TYPE_CHECKING:
 from .configuration_layoutlmva import LAYOUTLMV2_PRETRAINED_CONFIG_ARCHIVE_MAP, LayoutLMvaConfig
 from .processing_layoutlmva import LayoutLMvaProcessor
 from .tokenization_layoutlmva import LayoutLMvaTokenizer

 try:
  if not is_tokenizers_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .tokenization_layoutlmva_fast import LayoutLMvaTokenizerFast

 try:
  if not is_vision_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .feature_extraction_layoutlmva import LayoutLMvaFeatureExtractor, LayoutLMvaImageProcessor

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_layoutlmva import (
      LAYOUTLMV2_PRETRAINED_MODEL_ARCHIVE_LIST,
      LayoutLMvaForQuestionAnswering,
      LayoutLMvaForSequenceClassification,
      LayoutLMvaForTokenClassification,
      LayoutLMvaLayer,
      LayoutLMvaModel,
      LayoutLMvaPreTrainedModel,
  )
else:
 import sys

 lowerCamelCase_			=						_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)

