"""simple docstring"""
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_herbert import HerbertTokenizer


SCREAMING_SNAKE_CASE__   =					logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__   =					{"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

SCREAMING_SNAKE_CASE__   =					{
    "vocab_file": {
        "allegro/herbert-base-cased": "https://huggingface.co/allegro/herbert-base-cased/resolve/main/vocab.json"
    },
    "merges_file": {
        "allegro/herbert-base-cased": "https://huggingface.co/allegro/herbert-base-cased/resolve/main/merges.txt"
    },
}

SCREAMING_SNAKE_CASE__   =					{"allegro/herbert-base-cased": 514}
SCREAMING_SNAKE_CASE__   =					{}



class 					lowerCAmelCase_      (     lowerCAmelCase   ):



				"""simple docstring"""
				_lowerCAmelCase       :      str     					=  VOCAB_FILES_NAMES
				_lowerCAmelCase       :      List[str]     					=  PRETRAINED_VOCAB_FILES_MAP
				_lowerCAmelCase       :      List[str]     					=  PRETRAINED_INIT_CONFIGURATION
				_lowerCAmelCase       :      Dict     					=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				_lowerCAmelCase       :      Optional[int]     					=  HerbertTokenizer

				def __init__(			self						,      lowerCAmelCase=None						,      lowerCAmelCase=None						,      lowerCAmelCase=None						,      lowerCAmelCase="<s>"						,      lowerCAmelCase="<unk>"						,      lowerCAmelCase="<pad>"						,      lowerCAmelCase="<mask>"						,      lowerCAmelCase="</s>"						,      **lowerCAmelCase						,      ):






										"""simple docstring"""

										super().__init__(
										    lowerCAmelCase						,      lowerCAmelCase						,      tokenizer_file=lowerCAmelCase						,      cls_token=lowerCAmelCase						,      unk_token=lowerCAmelCase						,      pad_token=lowerCAmelCase						,      mask_token=lowerCAmelCase						,      sep_token=lowerCAmelCase						,      **lowerCAmelCase						,      )

				def 						snake_case	(			self						,      lowerCAmelCase						,      lowerCAmelCase = None      ):






										"""simple docstring"""

										snake_case        =     [self.cls_token_id]
										snake_case        =     [self.sep_token_id]
										if token_ids_a is None:
																return cls + token_ids_a + sep

										return cls + token_ids_a + sep + token_ids_a + sep

				def 						snake_case	(			self						,      lowerCAmelCase						,      lowerCAmelCase = None						,      lowerCAmelCase = False      ):






										"""simple docstring"""

										if already_has_special_tokens:
																return super().get_special_tokens_mask(
																    token_ids_a=lowerCAmelCase						,      token_ids_a=lowerCAmelCase						,      already_has_special_tokens=lowerCAmelCase      )

										if token_ids_a is None:
																return [1] + ([0] * len(lowerCAmelCase      )) + [1]
										return [1] + ([0] * len(lowerCAmelCase      )) + [1] + ([0] * len(lowerCAmelCase      )) + [1]

				def 						snake_case	(			self						,      lowerCAmelCase						,      lowerCAmelCase = None      ):






										"""simple docstring"""

										snake_case        =     [self.sep_token_id]
										snake_case        =     [self.cls_token_id]

										if token_ids_a is None:
																return len(cls + token_ids_a + sep      ) * [0]
										return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]

				def 						snake_case	(			self						,      lowerCAmelCase						,      lowerCAmelCase = None      ):






										"""simple docstring"""

										snake_case        =     self._tokenizer.model.save(lowerCAmelCase						,      name=lowerCAmelCase      )
										return tuple(lowerCAmelCase      )



"""simple docstring"""
from ..utils import DummyObject, requires_backends



class 					lowerCAmelCase_      (     metaclass=lowerCAmelCase   ):



				"""simple docstring"""
				_lowerCAmelCase       :      Dict     					=  ["""sentencepiece"""]

				def __init__(			self						,      *lowerCAmelCase						,      **lowerCAmelCase      ):






										"""simple docstring"""

										requires_backends(self						,      ['sentencepiece']      )





class 					lowerCAmelCase_      (     metaclass=lowerCAmelCase   ):



				"""simple docstring"""
				_lowerCAmelCase       :      Optional[Any]     					=  ["""sentencepiece"""]

				def __init__(			self						,      *lowerCAmelCase						,      **lowerCAmelCase      ):






										"""simple docstring"""

										requires_backends(self						,      ['sentencepiece']      )





class 					lowerCAmelCase_      (     metaclass=lowerCAmelCase   ):



				"""simple docstring"""
				_lowerCAmelCase       :      Optional[Any]     					=  ["""sentencepiece"""]

				def __init__(			self						,      *lowerCAmelCase						,      **lowerCAmelCase      ):






										"""simple docstring"""

										requires_backends(self						,      ['sentencepiece']      )





class 					lowerCAmelCase_      (     metaclass=lowerCAmelCase   ):



				"""simple docstring"""
				_lowerCAmelCase       :      Union[str, Any]     					=  ["""sentencepiece"""]

				def __init__(			self						,      *lowerCAmelCase						,      **lowerCAmelCase      ):






										"""simple docstring"""

										requires_backends(self						,      ['sentencepiece']      )





class 					lowerCAmelCase_      (     metaclass=lowerCAmelCase   ):



				"""simple docstring"""
				_lowerCAmelCase       :      Optional[int]     					=  ["""sentencepiece"""]

				def __init__(			self						,      *lowerCAmelCase						,      **lowerCAmelCase      ):






										"""simple docstring"""

										requires_backends(self						,      ['sentencepiece']      )





class 					lowerCAmelCase_      (     metaclass=lowerCAmelCase   ):



				"""simple docstring"""
				_lowerCAmelCase       :      Dict     					=  ["""sentencepiece"""]

				def __init__(			self						,      *lowerCAmelCase						,      **lowerCAmelCase      ):






										"""simple docstring"""

										requires_backends(self						,      ['sentencepiece']      )





class 					lowerCAmelCase_      (     metaclass=lowerCAmelCase   ):



				"""simple docstring"""
				_lowerCAmelCase       :      Union[str, Any]     					=  ["""sentencepiece"""]

				def __init__(			self						,      *lowerCAmelCase						,      **lowerCAmelCase      ):






										"""simple docstring"""

										requires_backends(self						,      ['sentencepiece']      )





class 					lowerCAmelCase_      (     metaclass=lowerCAmelCase   ):



				"""simple docstring"""
				_lowerCAmelCase       :      Union[str, Any]     					=  ["""sentencepiece"""]

				def __init__(			self						,      *lowerCAmelCase						,      **lowerCAmelCase      ):






										"""simple docstring"""

										requires_backends(self						,      ['sentencepiece']      )





class 					lowerCAmelCase_      (     metaclass=lowerCAmelCase   ):



				"""simple docstring"""
				_lowerCAmelCase       :      Optional[int]     					=  ["""sentencepiece"""]

				def __init__(			self						,      *lowerCAmelCase						,      **lowerCAmelCase      ):






										"""simple docstring"""

										requires_backends(self						,      ['sentencepiece']      )





class 					lowerCAmelCase_      (     metaclass=lowerCAmelCase   ):



				"""simple docstring"""
				_lowerCAmelCase       :      Any     					=  ["""sentencepiece"""]

				def __init__(			self						,      *lowerCAmelCase						,      **lowerCAmelCase      ):






										"""simple docstring"""

										requires_backends(self						,      ['sentencepiece']      )





class 					lowerCAmelCase_      (     metaclass=lowerCAmelCase   ):



				"""simple docstring"""
				_lowerCAmelCase       :      Union[str, Any]     					=  ["""sentencepiece"""]

				def __init__(			self						,      *lowerCAmelCase						,      **lowerCAmelCase      ):






										"""simple docstring"""

										requires_backends(self						,      ['sentencepiece']      )





class 					lowerCAmelCase_      (     metaclass=lowerCAmelCase   ):



				"""simple docstring"""
				_lowerCAmelCase       :      List[Any]     					=  ["""sentencepiece"""]

				def __init__(			self						,      *lowerCAmelCase						,      **lowerCAmelCase      ):






										"""simple docstring"""

										requires_backends(self						,      ['sentencepiece']      )





class 					lowerCAmelCase_      (     metaclass=lowerCAmelCase   ):



				"""simple docstring"""
				_lowerCAmelCase       :      str     					=  ["""sentencepiece"""]

				def __init__(			self						,      *lowerCAmelCase						,      **lowerCAmelCase      ):






										"""simple docstring"""

										requires_backends(self						,      ['sentencepiece']      )





class 					lowerCAmelCase_      (     metaclass=lowerCAmelCase   ):



				"""simple docstring"""
				_lowerCAmelCase       :      Tuple     					=  ["""sentencepiece"""]

				def __init__(			self						,      *lowerCAmelCase						,      **lowerCAmelCase      ):






										"""simple docstring"""

										requires_backends(self						,      ['sentencepiece']      )





class 					lowerCAmelCase_      (     metaclass=lowerCAmelCase   ):



				"""simple docstring"""
				_lowerCAmelCase       :      Union[str, Any]     					=  ["""sentencepiece"""]

				def __init__(			self						,      *lowerCAmelCase						,      **lowerCAmelCase      ):






										"""simple docstring"""

										requires_backends(self						,      ['sentencepiece']      )





class 					lowerCAmelCase_      (     metaclass=lowerCAmelCase   ):



				"""simple docstring"""
				_lowerCAmelCase       :      Union[str, Any]     					=  ["""sentencepiece"""]

				def __init__(			self						,      *lowerCAmelCase						,      **lowerCAmelCase      ):






										"""simple docstring"""

										requires_backends(self						,      ['sentencepiece']      )





class 					lowerCAmelCase_      (     metaclass=lowerCAmelCase   ):



				"""simple docstring"""
				_lowerCAmelCase       :      str     					=  ["""sentencepiece"""]

				def __init__(			self						,      *lowerCAmelCase						,      **lowerCAmelCase      ):






										"""simple docstring"""

										requires_backends(self						,      ['sentencepiece']      )





class 					lowerCAmelCase_      (     metaclass=lowerCAmelCase   ):



				"""simple docstring"""
				_lowerCAmelCase       :      Tuple     					=  ["""sentencepiece"""]

				def __init__(			self						,      *lowerCAmelCase						,      **lowerCAmelCase      ):






										"""simple docstring"""

										requires_backends(self						,      ['sentencepiece']      )





class 					lowerCAmelCase_      (     metaclass=lowerCAmelCase   ):



				"""simple docstring"""
				_lowerCAmelCase       :      Any     					=  ["""sentencepiece"""]

				def __init__(			self						,      *lowerCAmelCase						,      **lowerCAmelCase      ):






										"""simple docstring"""

										requires_backends(self						,      ['sentencepiece']      )





class 					lowerCAmelCase_      (     metaclass=lowerCAmelCase   ):



				"""simple docstring"""
				_lowerCAmelCase       :      Tuple     					=  ["""sentencepiece"""]

				def __init__(			self						,      *lowerCAmelCase						,      **lowerCAmelCase      ):






										"""simple docstring"""

										requires_backends(self						,      ['sentencepiece']      )





class 					lowerCAmelCase_      (     metaclass=lowerCAmelCase   ):



				"""simple docstring"""
				_lowerCAmelCase       :      Tuple     					=  ["""sentencepiece"""]

				def __init__(			self						,      *lowerCAmelCase						,      **lowerCAmelCase      ):






										"""simple docstring"""

										requires_backends(self						,      ['sentencepiece']      )





class 					lowerCAmelCase_      (     metaclass=lowerCAmelCase   ):



				"""simple docstring"""
				_lowerCAmelCase       :      int     					=  ["""sentencepiece"""]

				def __init__(			self						,      *lowerCAmelCase						,      **lowerCAmelCase      ):






										"""simple docstring"""

										requires_backends(self						,      ['sentencepiece']      )





class 					lowerCAmelCase_      (     metaclass=lowerCAmelCase   ):



				"""simple docstring"""
				_lowerCAmelCase       :      Any     					=  ["""sentencepiece"""]

				def __init__(			self						,      *lowerCAmelCase						,      **lowerCAmelCase      ):






										"""simple docstring"""

										requires_backends(self						,      ['sentencepiece']      )





class 					lowerCAmelCase_      (     metaclass=lowerCAmelCase   ):



				"""simple docstring"""
				_lowerCAmelCase       :      Optional[int]     					=  ["""sentencepiece"""]

				def __init__(			self						,      *lowerCAmelCase						,      **lowerCAmelCase      ):






										"""simple docstring"""

										requires_backends(self						,      ['sentencepiece']      )





class 					lowerCAmelCase_      (     metaclass=lowerCAmelCase   ):



				"""simple docstring"""
				_lowerCAmelCase       :      int     					=  ["""sentencepiece"""]

				def __init__(			self						,      *lowerCAmelCase						,      **lowerCAmelCase      ):






										"""simple docstring"""

										requires_backends(self						,      ['sentencepiece']      )





class 					lowerCAmelCase_      (     metaclass=lowerCAmelCase   ):



				"""simple docstring"""
				_lowerCAmelCase       :      Dict     					=  ["""sentencepiece"""]

				def __init__(			self						,      *lowerCAmelCase						,      **lowerCAmelCase      ):






										"""simple docstring"""

										requires_backends(self						,      ['sentencepiece']      )





class 					lowerCAmelCase_      (     metaclass=lowerCAmelCase   ):



				"""simple docstring"""
				_lowerCAmelCase       :      Any     					=  ["""sentencepiece"""]

				def __init__(			self						,      *lowerCAmelCase						,      **lowerCAmelCase      ):






										"""simple docstring"""

										requires_backends(self						,      ['sentencepiece']      )





class 					lowerCAmelCase_      (     metaclass=lowerCAmelCase   ):



				"""simple docstring"""
				_lowerCAmelCase       :      Union[str, Any]     					=  ["""sentencepiece"""]

				def __init__(			self						,      *lowerCAmelCase						,      **lowerCAmelCase      ):






										"""simple docstring"""

										requires_backends(self						,      ['sentencepiece']      )





class 					lowerCAmelCase_      (     metaclass=lowerCAmelCase   ):



				"""simple docstring"""
				_lowerCAmelCase       :      str     					=  ["""sentencepiece"""]

				def __init__(			self						,      *lowerCAmelCase						,      **lowerCAmelCase      ):






										"""simple docstring"""

										requires_backends(self						,      ['sentencepiece']      )





class 					lowerCAmelCase_      (     metaclass=lowerCAmelCase   ):



				"""simple docstring"""
				_lowerCAmelCase       :      Dict     					=  ["""sentencepiece"""]

				def __init__(			self						,      *lowerCAmelCase						,      **lowerCAmelCase      ):






										"""simple docstring"""

										requires_backends(self						,      ['sentencepiece']      )





class 					lowerCAmelCase_      (     metaclass=lowerCAmelCase   ):



				"""simple docstring"""
				_lowerCAmelCase       :      Tuple     					=  ["""sentencepiece"""]

				def __init__(			self						,      *lowerCAmelCase						,      **lowerCAmelCase      ):






										"""simple docstring"""

										requires_backends(self						,      ['sentencepiece']      )


