

"""simple docstring"""


import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__A						       =      logging.get_logger(__name__)

__A						       =      "▁"

__A						       =      {"vocab_file": "vocab.txt", "sentencepiece_model_ckpt": "sentencepiece.bpe.model"}

__A						       =      {
    "sentencepiece_model_file": "sentencepiece.bpe.model",
    "vocab_file": "vocab.txt",
}

__A						       =      {
    "vocab_file": {
        "ernie-m-base": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt",
        "ernie-m-large": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt",
    },
    "sentencepiece_model_file": {
        "ernie-m-base": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model",
        "ernie-m-large": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model",
    },
}

__A						       =      {
    "ernie-m-base": 514,
    "ernie-m-large": 514,
}

__A						       =      {
    "ernie-m-base": {"do_lower_case": False},
    "ernie-m-large": {"do_lower_case": False},
}





class    snake_case     (    __snake_case  ):
				SCREAMING_SNAKE_CASE_       :				List[str]       				=  ["input_ids"]

				SCREAMING_SNAKE_CASE_       :				Dict       				=  VOCAB_FILES_NAMES
				SCREAMING_SNAKE_CASE_       :				List[str]       				=  PRETRAINED_INIT_CONFIGURATION
				SCREAMING_SNAKE_CASE_       :				str       				=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				SCREAMING_SNAKE_CASE_       :				str       				=  PRETRAINED_VOCAB_FILES_MAP
				SCREAMING_SNAKE_CASE_       :				Tuple       				=  RESOURCE_FILES_NAMES

				def __init__(     self							:    Optional[int]					,  UpperCamelCase__							:    Optional[int]					,  UpperCamelCase__							:    str=None					,  UpperCamelCase__							:    str=False					,  UpperCamelCase__							:    Tuple="utf8"					,  UpperCamelCase__							:    str="[UNK]"					,  UpperCamelCase__							:    int="[SEP]"					,  UpperCamelCase__							:    Optional[int]="[PAD]"					,  UpperCamelCase__							:    Tuple="[CLS]"					,  UpperCamelCase__							:    int="[MASK]"					,  UpperCamelCase__							:    Optional[Dict[str, Any]] = None					,  **UpperCamelCase__							:    Dict					,  )->       None:



						'''simple docstring'''


						__lowerCAmelCase:	Optional[Any]   				=      {} if sp_model_kwargs is None else sp_model_kwargs
						super().__init__(
						    do_lower_case=UpperCamelCase__					,  unk_token=UpperCamelCase__					,  sep_token=UpperCamelCase__					,  pad_token=UpperCamelCase__					,  cls_token=UpperCamelCase__					,  mask_token=UpperCamelCase__					,  vocab_file=UpperCamelCase__					,  encoding=UpperCamelCase__					,  sp_model_kwargs=self.sp_model_kwargs					,  **UpperCamelCase__					,  )
						__lowerCAmelCase:	Optional[int]   				=      do_lower_case
						__lowerCAmelCase:	Optional[Any]   				=      sentencepiece_model_ckpt
						__lowerCAmelCase:	str   				=      spm.SentencePieceProcessor(**self.sp_model_kwargs)
						self.sp_model.Load(UpperCamelCase__)

						# to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
						if vocab_file is not None:
								__lowerCAmelCase:	Any   				=      self.load_vocab(filepath=UpperCamelCase__)
						else:
								__lowerCAmelCase:	Union[str, Any]   				=      {self.sp_model.id_to_piece(UpperCamelCase__): id for id in range(self.sp_model.get_piece_size())}
						__lowerCAmelCase:	Tuple   				=      {v: k for k, v in self.vocab.items()}

				def 							lowercase_					(     self							:    Dict					,  UpperCamelCase__							:    int)->       Any:



						'''simple docstring'''


						if text is None:
								return None

						__lowerCAmelCase:	List[str]   				=      self.tokenize(UpperCamelCase__)
						__lowerCAmelCase				,			__lowerCAmelCase:	str   				=      "", []

						for i, ch in enumerate(UpperCamelCase__):
								if ch in self.SP_CHAR_MAPPING:
										__lowerCAmelCase:	int   				=      self.SP_CHAR_MAPPING.get(UpperCamelCase__)
								else:
										__lowerCAmelCase:	Dict   				=      unicodedata.normalize("NFKC"					,  UpperCamelCase__)
								if self.is_whitespace(UpperCamelCase__):
										continue
								normalized_text += ch
								char_mapping.extend([i] * len(UpperCamelCase__))

						__lowerCAmelCase				,			__lowerCAmelCase				,			__lowerCAmelCase:	Optional[int]   				=      normalized_text, [], 0

						if self.do_lower_case:
								__lowerCAmelCase:	str   				=      text.lower()

						for token in split_tokens:
								if token[:1] == "▁":
										__lowerCAmelCase:	int   				=      token[1:]
								__lowerCAmelCase:	str   				=      text[offset:].index(UpperCamelCase__) + offset
								__lowerCAmelCase:	Tuple   				=      start + len(UpperCamelCase__)

								token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1))
								__lowerCAmelCase:	List[str]   				=      end
						return token_mapping

				@property
				def 							lowercase_					(     self							:    Dict)->       Union[str, Any]:



						'''simple docstring'''


						return len(self.vocab)

				def 							lowercase_					(     self							:    List[Any])->       Dict:



						'''simple docstring'''


						return dict(self.vocab					,  **self.added_tokens_encoder)

				def __getstate__(     self							:    str)->       List[str]:



						'''simple docstring'''


						__lowerCAmelCase:	Union[str, Any]   				=      self.__dict__.copy()
						__lowerCAmelCase:	str   				=      None
						return state

				def __setstate__(     self							:    str					,  UpperCamelCase__							:    Any)->       Dict:



						'''simple docstring'''


						__lowerCAmelCase:	Any   				=      d

						# for backward compatibility
						if not hasattr(self					,  "sp_model_kwargs"):
								__lowerCAmelCase:	int   				=      {}

						__lowerCAmelCase:	int   				=      spm.SentencePieceProcessor(**self.sp_model_kwargs)
						self.sp_model.Load(self.sentencepiece_model_ckpt)

				def 							lowercase_					(     self							:    Dict					,  UpperCamelCase__							:    Optional[int])->       str:



						'''simple docstring'''


						return "".join((self.SP_CHAR_MAPPING.get(UpperCamelCase__					,  UpperCamelCase__) for c in text))

				def 							lowercase_					(     self							:    Dict					,  UpperCamelCase__							:    List[Any]					,  UpperCamelCase__							:    Optional[Any]=False					,  UpperCamelCase__							:    List[str]=6_4					,  UpperCamelCase__							:    Any=0.1)->       Any:



						'''simple docstring'''


						if self.sp_model_kwargs.get("enable_sampling") is True:
								__lowerCAmelCase:	Optional[Any]   				=      True
						if self.sp_model_kwargs.get("alpha") is not None:
								__lowerCAmelCase:	str   				=      self.sp_model_kwargs.get("alpha")
						if self.sp_model_kwargs.get("nbest_size") is not None:
								__lowerCAmelCase:	Optional[Any]   				=      self.sp_model_kwargs.get("nbest_size")

						if not enable_sampling:
								__lowerCAmelCase:	Any   				=      self.sp_model.EncodeAsPieces(UpperCamelCase__)
						else:
								__lowerCAmelCase:	str   				=      self.sp_model.SampleEncodeAsPieces(UpperCamelCase__					,  UpperCamelCase__					,  UpperCamelCase__)
						__lowerCAmelCase:	int   				=      []
						for pi, piece in enumerate(UpperCamelCase__):
								if piece == SPIECE_UNDERLINE:
										if not pieces[pi + 1].startswith(UpperCamelCase__) and pi != 0:
												new_pieces.append(UpperCamelCase__)
												continue
										else:
												continue
								__lowerCAmelCase:	Optional[int]   				=      0
								for i, chunk in enumerate(UpperCamelCase__):
										if chunk == SPIECE_UNDERLINE:
												continue
										if self.is_ch_char(UpperCamelCase__) or self.is_punct(UpperCamelCase__):
												if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
														new_pieces.append(piece[lst_i:i])
												new_pieces.append(UpperCamelCase__)
												__lowerCAmelCase:	Optional[int]   				=      i + 1
										elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
												if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
														new_pieces.append(piece[lst_i:i])
												__lowerCAmelCase:	Optional[int]   				=      i
										elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
												if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
														new_pieces.append(piece[lst_i:i])
												__lowerCAmelCase:	str   				=      i
								if len(UpperCamelCase__) > lst_i:
										new_pieces.append(piece[lst_i:])
						return new_pieces

				def 							lowercase_					(     self							:    Tuple					,  UpperCamelCase__							:    Any)->       Optional[int]:



						'''simple docstring'''


						__lowerCAmelCase:	Dict   				=      "".join(UpperCamelCase__).replace(UpperCamelCase__					,  " ").strip()
						return out_string

				def 							lowercase_					(     self							:    Any					,  UpperCamelCase__							:    Union[str, Any])->       Dict:



						'''simple docstring'''


						__lowerCAmelCase:	Any   				=      self.convert_ids_to_tokens(UpperCamelCase__)
						__lowerCAmelCase:	Union[str, Any]   				=      "".join(UpperCamelCase__).replace(UpperCamelCase__					,  " ").strip()
						return out_string

				def 							lowercase_					(     self							:    Any					,  UpperCamelCase__							:    int)->       Dict:



						'''simple docstring'''


						return self.vocab.get(UpperCamelCase__					,  self.vocab.get(self.unk_token))

				def 							lowercase_					(     self							:    List[Any]					,  UpperCamelCase__							:    Union[str, Any])->       Any:



						'''simple docstring'''


						return self.reverse_vocab.get(UpperCamelCase__					,  self.unk_token)

				def 							lowercase_					(     self							:    List[str]					,  UpperCamelCase__							:    str					,  UpperCamelCase__							:    int=None)->       int:



						'''simple docstring'''


						if token_ids_a is None:
								return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
						__lowerCAmelCase:	Tuple   				=      [self.cls_token_id]
						__lowerCAmelCase:	str   				=      [self.sep_token_id]
						return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep

				def 							lowercase_					(     self							:    Optional[int]					,  UpperCamelCase__							:    str					,  UpperCamelCase__							:    List[str]=None)->       Optional[int]:



						'''simple docstring'''


						if offset_mapping_a is None:
								return [(0, 0)] + offset_mapping_a + [(0, 0)]

						return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]

				def 							lowercase_					(     self							:    int					,  UpperCamelCase__							:    Dict					,  UpperCamelCase__							:    Any=None					,  UpperCamelCase__							:    List[Any]=False)->       Any:



						'''simple docstring'''


						if already_has_special_tokens:
								if token_ids_a is not None:
										raise ValueError(
										    "You should not supply a second sequence if the provided sequence of "
										    "ids is already formatted with special tokens for the model.")
								return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

						if token_ids_a is not None:
								return [1] + ([0] * len(UpperCamelCase__)) + [1, 1] + ([0] * len(UpperCamelCase__)) + [1]
						return [1] + ([0] * len(UpperCamelCase__)) + [1]

				def 							lowercase_					(     self							:    List[str]					,  UpperCamelCase__							:    List[int]					,  UpperCamelCase__							:    Optional[List[int]] = None)->       List[int]:



						'''simple docstring'''


						if token_ids_a is None:
								# [CLS] X [SEP]
								return (len(UpperCamelCase__) + 2) * [0]

						# [CLS] A [SEP] [SEP] B [SEP]
						return [0] * (len(UpperCamelCase__) + 1) + [1] * (len(UpperCamelCase__) + 3)

				def 							lowercase_					(     self							:    Union[str, Any]					,  UpperCamelCase__							:    int)->       Dict:



						'''simple docstring'''


						if "\u4e00" <= char <= "\u9fff":
								return True
						return False

				def 							lowercase_					(     self							:    List[Any]					,  UpperCamelCase__							:    int)->       List[Any]:



						'''simple docstring'''


						if ("a" <= char <= "z") or ("A" <= char <= "Z"):
								return True
						return False

				def 							lowercase_					(     self							:    Any					,  UpperCamelCase__							:    Any)->       List[Any]:



						'''simple docstring'''


						if char in ",;:.?!~，；：。？！《》【】":
								return True
						return False

				def 							lowercase_					(     self							:    List[Any]					,  UpperCamelCase__							:    Dict)->       int:



						'''simple docstring'''


						if char == " " or char == "\t" or char == "\n" or char == "\r":
								return True
						if len(UpperCamelCase__) == 1:
								__lowerCAmelCase:	Dict   				=      unicodedata.category(UpperCamelCase__)
								if cat == "Zs":
										return True
						return False

				def 							lowercase_					(     self							:    int					,  UpperCamelCase__							:    List[str])->       Dict:



						'''simple docstring'''


						__lowerCAmelCase:	List[Any]   				=      {}
						with io.open(UpperCamelCase__					,  "r"					,  encoding="utf-8") as f:
								for index, line in enumerate(UpperCamelCase__):
										__lowerCAmelCase:	Any   				=      line.rstrip("\n")
										__lowerCAmelCase:	Any   				=      int(UpperCamelCase__)

						return token_to_idx




				def 							lowercase_					(     self							:    List[Any]					,  UpperCamelCase__							:    str					,  UpperCamelCase__							:    Optional[str] = None)->       Tuple[str]:



						'''simple docstring'''


						__lowerCAmelCase:	Union[str, Any]   				=      0
						if os.path.isdir(UpperCamelCase__):
								__lowerCAmelCase:	int   				=      os.path.join(
								    UpperCamelCase__					,  (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"])
						else:
								__lowerCAmelCase:	Optional[int]   				=      (filename_prefix + "-" if filename_prefix else "") + save_directory
						with open(UpperCamelCase__					,  "w"					,  encoding="utf-8") as writer:
								for token, token_index in sorted(self.vocab.items()					,  key=lambda UpperCamelCase__: kv[1]):
										if index != token_index:
												logger.warning(
												    f"Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."
												    " Please check that the vocabulary is not corrupted!")
												__lowerCAmelCase:	Dict   				=      token_index
										writer.write(token + "\n")
										index += 1

						__lowerCAmelCase:	List[Any]   				=      os.path.join(UpperCamelCase__					,  "sentencepiece.bpe.model")
						with open(UpperCamelCase__					,  "wb") as fi:
								__lowerCAmelCase:	Union[str, Any]   				=      self.sp_model.serialized_model_proto()
								fi.write(UpperCamelCase__)

						return (vocab_file,)





"""simple docstring"""


from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


__A						       =      {
    "configuration_swinv2": ["SWINV2_PRETRAINED_CONFIG_ARCHIVE_MAP", "Swinv2Config"],
}


try:
			if not is_torch_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			__A						       =      [
			    "SWINV2_PRETRAINED_MODEL_ARCHIVE_LIST",
			    "Swinv2ForImageClassification",
			    "Swinv2ForMaskedImageModeling",
			    "Swinv2Model",
			    "Swinv2PreTrainedModel",
			]


if TYPE_CHECKING:
			from .configuration_swinva import SWINV2_PRETRAINED_CONFIG_ARCHIVE_MAP, SwinvaConfig

			try:
						if not is_torch_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_swinva import (
						    SWINV2_PRETRAINED_MODEL_ARCHIVE_LIST,
						    SwinvaForImageClassification,
						    SwinvaForMaskedImageModeling,
						    SwinvaModel,
						    SwinvaPreTrainedModel,
						)


else:
			import sys

			__A						       =      _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)


