




"""simple docstring"""




import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roberta import RobertaTokenizer


UpperCamelCase__         =      logging.get_logger(__name__)

UpperCamelCase__         =      {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

UpperCamelCase__         =      {
    """vocab_file""": {
        """roberta-base""": """https://huggingface.co/roberta-base/resolve/main/vocab.json""",
        """roberta-large""": """https://huggingface.co/roberta-large/resolve/main/vocab.json""",
        """roberta-large-mnli""": """https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json""",
        """distilroberta-base""": """https://huggingface.co/distilroberta-base/resolve/main/vocab.json""",
        """roberta-base-openai-detector""": """https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json""",
        """roberta-large-openai-detector""": (
            """https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """roberta-base""": """https://huggingface.co/roberta-base/resolve/main/merges.txt""",
        """roberta-large""": """https://huggingface.co/roberta-large/resolve/main/merges.txt""",
        """roberta-large-mnli""": """https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt""",
        """distilroberta-base""": """https://huggingface.co/distilroberta-base/resolve/main/merges.txt""",
        """roberta-base-openai-detector""": """https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt""",
        """roberta-large-openai-detector""": (
            """https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt"""
        ),
    },
    """tokenizer_file""": {
        """roberta-base""": """https://huggingface.co/roberta-base/resolve/main/tokenizer.json""",
        """roberta-large""": """https://huggingface.co/roberta-large/resolve/main/tokenizer.json""",
        """roberta-large-mnli""": """https://huggingface.co/roberta-large-mnli/resolve/main/tokenizer.json""",
        """distilroberta-base""": """https://huggingface.co/distilroberta-base/resolve/main/tokenizer.json""",
        """roberta-base-openai-detector""": (
            """https://huggingface.co/roberta-base-openai-detector/resolve/main/tokenizer.json"""
        ),
        """roberta-large-openai-detector""": (
            """https://huggingface.co/roberta-large-openai-detector/resolve/main/tokenizer.json"""
        ),
    },
}

UpperCamelCase__         =      {
    """roberta-base""": 512,
    """roberta-large""": 512,
    """roberta-large-mnli""": 512,
    """distilroberta-base""": 512,
    """roberta-base-openai-detector""": 512,
    """roberta-large-openai-detector""": 512,
}
class       a__   ( __lowercase				):
						_a							:   str =   VOCAB_FILES_NAMES
						_a							:   List[Any] =   PRETRAINED_VOCAB_FILES_MAP
						_a							:   List[str] =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						_a							:   Tuple =   ['''input_ids''', '''attention_mask''']
						_a							:   Optional[int] =   RobertaTokenizer






						def __init__(       self		,      _A=None		,      _A=None		,      _A=None		,      _A="replace"		,      _A="<s>"		,      _A="</s>"		,      _A="</s>"		,      _A="<s>"		,      _A="<unk>"		,      _A="<pad>"		,      _A="<mask>"		,      _A=False		,      _A=True		,      **_A		,      ):




											"""simple docstring"""



											super().__init__(
											    UpperCAmelCase__		,      UpperCAmelCase__		,      tokenizer_file=UpperCAmelCase__		,      errors=UpperCAmelCase__		,      bos_token=UpperCAmelCase__		,      eos_token=UpperCAmelCase__		,      sep_token=UpperCAmelCase__		,      cls_token=UpperCAmelCase__		,      unk_token=UpperCAmelCase__		,      pad_token=UpperCAmelCase__		,      mask_token=UpperCAmelCase__		,      add_prefix_space=UpperCAmelCase__		,      trim_offsets=UpperCAmelCase__		,      **UpperCAmelCase__		,      )

											__lowerCAmelCase           =	json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()      )
											if pre_tok_state.get("add_prefix_space"		,      UpperCAmelCase__      ) != add_prefix_space:
																__lowerCAmelCase           =	getattr(UpperCAmelCase__		,      pre_tok_state.pop("type"      )      )
																__lowerCAmelCase           =	add_prefix_space
																__lowerCAmelCase           =	pre_tok_class(**UpperCAmelCase__      )

											__lowerCAmelCase           =	add_prefix_space

											__lowerCAmelCase           =	"post_processor"
											__lowerCAmelCase           =	getattr(self.backend_tokenizer		,      UpperCAmelCase__		,      UpperCAmelCase__      )
											if tokenizer_component_instance:
																__lowerCAmelCase           =	json.loads(tokenizer_component_instance.__getstate__()      )

																# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
																if "sep" in state:
																					__lowerCAmelCase           =	tuple(state["sep"]      )
																if "cls" in state:
																					__lowerCAmelCase           =	tuple(state["cls"]      )

																__lowerCAmelCase           =	False

																if state.get("add_prefix_space"		,      UpperCAmelCase__      ) != add_prefix_space:
																					__lowerCAmelCase           =	add_prefix_space
																					__lowerCAmelCase           =	True

																if state.get("trim_offsets"		,      UpperCAmelCase__      ) != trim_offsets:
																					__lowerCAmelCase           =	trim_offsets
																					__lowerCAmelCase           =	True

																if changes_to_apply:
																					__lowerCAmelCase           =	getattr(UpperCAmelCase__		,      state.pop("type"      )      )
																					__lowerCAmelCase           =	component_class(**UpperCAmelCase__      )
																					setattr(self.backend_tokenizer		,      UpperCAmelCase__		,      UpperCAmelCase__      )






						@property
						def     __SCREAMING_SNAKE_CASE(       self      ):




											"""simple docstring"""



											if self._mask_token is None:
																if self.verbose:
																					logger.error("Using mask_token, but it is not set yet."      )
																return None
											return str(self._mask_token      )






						@mask_token.setter
						def     __SCREAMING_SNAKE_CASE(       self		,      _A      ):




											"""simple docstring"""



											__lowerCAmelCase           =	AddedToken(UpperCAmelCase__		,      lstrip=UpperCAmelCase__		,      rstrip=UpperCAmelCase__      ) if isinstance(UpperCAmelCase__		,      UpperCAmelCase__      ) else value
											__lowerCAmelCase           =	value






						def     __SCREAMING_SNAKE_CASE(       self		,      *_A		,      **_A      ):




											"""simple docstring"""



											__lowerCAmelCase           =	kwargs.get("is_split_into_words"		,      UpperCAmelCase__      )
											assert self.add_prefix_space or not is_split_into_words, (
											    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
											    "to use it with pretokenized inputs."
											)

											return super()._batch_encode_plus(*UpperCAmelCase__		,      **UpperCAmelCase__      )






						def     __SCREAMING_SNAKE_CASE(       self		,      *_A		,      **_A      ):




											"""simple docstring"""



											__lowerCAmelCase           =	kwargs.get("is_split_into_words"		,      UpperCAmelCase__      )

											assert self.add_prefix_space or not is_split_into_words, (
											    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
											    "to use it with pretokenized inputs."
											)

											return super()._encode_plus(*UpperCAmelCase__		,      **UpperCAmelCase__      )






						def     __SCREAMING_SNAKE_CASE(       self		,      _A		,      _A = None      ):




											"""simple docstring"""



											__lowerCAmelCase           =	self._tokenizer.model.save(UpperCAmelCase__		,      name=UpperCAmelCase__      )
											return tuple(UpperCAmelCase__      )






						def     __SCREAMING_SNAKE_CASE(       self		,      _A		,      _A=None      ):




											"""simple docstring"""



											__lowerCAmelCase           =	[self.bos_token_id] + token_ids_a + [self.eos_token_id]
											if token_ids_a is None:
																return output

											return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]






						def     __SCREAMING_SNAKE_CASE(       self		,      _A		,      _A = None      ):




											"""simple docstring"""



											__lowerCAmelCase           =	[self.sep_token_id]
											__lowerCAmelCase           =	[self.cls_token_id]

											if token_ids_a is None:
																return len(cls + token_ids_a + sep      ) * [0]
											return len(cls + token_ids_a + sep + sep + token_ids_a + sep      ) * [0]










from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCamelCase__         =      logging.get_logger(__name__)

UpperCamelCase__         =      {
    """alibaba-damo/mgp-str-base""": """https://huggingface.co/alibaba-damo/mgp-str-base/resolve/main/config.json""",
}
class       a__   ( snake_case__				):
						_a							:   Dict =   """mgp-str"""






						def __init__(       self		,      _A=[3_2, 1_2_8]		,      _A=4		,      _A=3		,      _A=2_7		,      _A=3_8		,      _A=5_0_2_5_7		,      _A=3_0_5_2_2		,      _A=7_6_8		,      _A=1_2		,      _A=1_2		,      _A=4.0		,      _A=True		,      _A=False		,      _A=1E-5		,      _A=0.0		,      _A=0.0		,      _A=0.0		,      _A=False		,      _A=0.02		,      **_A		,      ):




											"""simple docstring"""



											super().__init__(**_A      )

											__lowerCAmelCase           =	image_size
											__lowerCAmelCase           =	patch_size
											__lowerCAmelCase           =	num_channels
											__lowerCAmelCase           =	max_token_length
											__lowerCAmelCase           =	num_character_labels
											__lowerCAmelCase           =	num_bpe_labels
											__lowerCAmelCase           =	num_wordpiece_labels
											__lowerCAmelCase           =	hidden_size
											__lowerCAmelCase           =	num_hidden_layers
											__lowerCAmelCase           =	num_attention_heads
											__lowerCAmelCase           =	mlp_ratio
											__lowerCAmelCase           =	distilled
											__lowerCAmelCase           =	layer_norm_eps
											__lowerCAmelCase           =	drop_rate
											__lowerCAmelCase           =	qkv_bias
											__lowerCAmelCase           =	attn_drop_rate
											__lowerCAmelCase           =	drop_path_rate
											__lowerCAmelCase           =	output_aa_attentions
											__lowerCAmelCase           =	initializer_range




