



from typing import Dict, List, Optional

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__snake_case						:Any				=					logging.get_logger(__name__)


__snake_case						:Tuple				=					{
    'nielsr/canine-s': 2048,
}

# Unicode defines 1,114,112 total “codepoints”
__snake_case						:int				=					111_4112

# Below: Constants defining canonical codepoints for special, pseudo-characters.
# Copied from https://github.com/google-research/language/blob/master/language/canine/special_codepoints.py
__snake_case						:Union[str, Any]				=					0

__snake_case						:str				=					0xe000
__snake_case						:str				=					0xe001
__snake_case						:int				=					0xe002
__snake_case						:Optional[Any]				=					0xe003
__snake_case						:List[Any]				=					0xe004

# Maps special codepoints to human-readable names.
__snake_case						:Dict[int, str]				=					{
    # Special symbols are represented using codepoints values that are valid,
    # but designated as "Private Use", meaning that they will never be assigned
    # characters by the Unicode Consortium, and are thus safe for use here.
    #
    # NOTE: Do *NOT* add any sort of [UNK_CHAR] here. They are explicitly
    # excluded and should fail with a hard error.
    CLS: "[CLS]",
    SEP: "[SEP]",
    BOS: "[BOS]",
    MASK: "[MASK]",
    PAD: "[PAD]",
    RESERVED: "[RESERVED]",
}

# Maps special codepoint human-readable names to their codepoint values.
__snake_case						:Dict[str, int]				=					{name: codepoint for codepoint, name in SPECIAL_CODEPOINTS.items()}

class    _A	(     __UpperCAmelCase   ):
       UpperCamelCase__  :   int						 =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES



       def __init__(		self							:					Optional[Any]					,		__SCREAMING_SNAKE_CASE							:					Dict=chr(__SCREAMING_SNAKE_CASE)					,		__SCREAMING_SNAKE_CASE							:					str=chr(__SCREAMING_SNAKE_CASE)					,		__SCREAMING_SNAKE_CASE							:					Union[str, Any]=chr(__SCREAMING_SNAKE_CASE)					,		__SCREAMING_SNAKE_CASE							:					Tuple=chr(__SCREAMING_SNAKE_CASE)					,		__SCREAMING_SNAKE_CASE							:					Optional[Any]=chr(__SCREAMING_SNAKE_CASE)					,		__SCREAMING_SNAKE_CASE							:					str=chr(__SCREAMING_SNAKE_CASE)					,		__SCREAMING_SNAKE_CASE							:					str=False					,		__SCREAMING_SNAKE_CASE							:					Union[str, Any]=2_048					,		**__SCREAMING_SNAKE_CASE							:					List[Any]					,		):


           '''simple docstring'''

           __a							=    AddedToken(__SCREAMING_SNAKE_CASE					,		lstrip=__SCREAMING_SNAKE_CASE					,		rstrip=__SCREAMING_SNAKE_CASE) if isinstance(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE) else bos_token
           __a							=    AddedToken(__SCREAMING_SNAKE_CASE					,		lstrip=__SCREAMING_SNAKE_CASE					,		rstrip=__SCREAMING_SNAKE_CASE) if isinstance(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE) else eos_token
           __a							=    AddedToken(__SCREAMING_SNAKE_CASE					,		lstrip=__SCREAMING_SNAKE_CASE					,		rstrip=__SCREAMING_SNAKE_CASE) if isinstance(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE) else sep_token
           __a							=    AddedToken(__SCREAMING_SNAKE_CASE					,		lstrip=__SCREAMING_SNAKE_CASE					,		rstrip=__SCREAMING_SNAKE_CASE) if isinstance(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE) else cls_token
           __a							=    AddedToken(__SCREAMING_SNAKE_CASE					,		lstrip=__SCREAMING_SNAKE_CASE					,		rstrip=__SCREAMING_SNAKE_CASE) if isinstance(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE) else pad_token

           # Mask token behave like a normal word, i.e. include the space before it
           __a							=    AddedToken(__SCREAMING_SNAKE_CASE					,		lstrip=__SCREAMING_SNAKE_CASE					,		rstrip=__SCREAMING_SNAKE_CASE) if isinstance(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE) else mask_token

           super().__init__(
               bos_token=__SCREAMING_SNAKE_CASE					,		eos_token=__SCREAMING_SNAKE_CASE					,		sep_token=__SCREAMING_SNAKE_CASE					,		cls_token=__SCREAMING_SNAKE_CASE					,		pad_token=__SCREAMING_SNAKE_CASE					,		mask_token=__SCREAMING_SNAKE_CASE					,		add_prefix_space=__SCREAMING_SNAKE_CASE					,		model_max_length=__SCREAMING_SNAKE_CASE					,		**__SCREAMING_SNAKE_CASE					,		)

           # Creates a mapping for looking up the IDs of special symbols.
           __a							=    {}
           for codepoint, name in SPECIAL_CODEPOINTS.items():
               __a							=    codepoint

           # Creates a mapping for looking up the string forms of special symbol IDs.
           __a							=    {
               codepoint: name for name, codepoint in self._special_codepoints.items()
           }

           __a							=    UNICODE_VOCAB_SIZE
           __a							=    len(self._special_codepoints)



       @property
       def 		_lowerCamelCase      (		self							:					Union[str, Any]):


           '''simple docstring'''

           return self._unicode_vocab_size



       def 		_lowerCamelCase      (		self							:					str					,		__SCREAMING_SNAKE_CASE							:					List[str]):


           '''simple docstring'''

           return list(__SCREAMING_SNAKE_CASE)



       def 		_lowerCamelCase      (		self							:					int					,		__SCREAMING_SNAKE_CASE							:					List[Any]):


           '''simple docstring'''

           try:
               return ord(__SCREAMING_SNAKE_CASE)
           except TypeError:
               raise ValueError(F'invalid token: \'{token}\'')



       def 		_lowerCamelCase      (		self							:					Dict					,		__SCREAMING_SNAKE_CASE							:					Tuple):


           '''simple docstring'''

           try:
               if index in SPECIAL_CODEPOINTS:
                   return SPECIAL_CODEPOINTS[index]
               return chr(__SCREAMING_SNAKE_CASE)
           except TypeError:
               raise ValueError(F'invalid id: {index}')



       def 		_lowerCamelCase      (		self							:					Any					,		__SCREAMING_SNAKE_CASE							:					Tuple):


           '''simple docstring'''

           return "".join(__SCREAMING_SNAKE_CASE)



       def 		_lowerCamelCase      (		self							:					Union[str, Any]					,		__SCREAMING_SNAKE_CASE							:					str					,		__SCREAMING_SNAKE_CASE							:					List[Any] = None):


           '''simple docstring'''

           __a							=    [self.sep_token_id]
           __a							=    [self.cls_token_id]

           __a							=    cls + token_ids_a + sep
           if token_ids_a is not None:
               result += token_ids_a + sep
           return result



       def 		_lowerCamelCase      (		self							:					Optional[int]					,		__SCREAMING_SNAKE_CASE							:					Tuple					,		__SCREAMING_SNAKE_CASE							:					str = None					,		__SCREAMING_SNAKE_CASE							:					Optional[int] = False):


           '''simple docstring'''

           if already_has_special_tokens:
               return super().get_special_tokens_mask(
                   token_ids_a=__SCREAMING_SNAKE_CASE					,		token_ids_a=__SCREAMING_SNAKE_CASE					,		already_has_special_tokens=__SCREAMING_SNAKE_CASE)

           __a							=    [1] + ([0] * len(__SCREAMING_SNAKE_CASE)) + [1]
           if token_ids_a is not None:
               result += ([0] * len(__SCREAMING_SNAKE_CASE)) + [1]
           return result



       def 		_lowerCamelCase      (		self							:					str					,		__SCREAMING_SNAKE_CASE							:					Any					,		__SCREAMING_SNAKE_CASE							:					Dict = None):


           '''simple docstring'''

           __a							=    [self.sep_token_id]
           __a							=    [self.cls_token_id]

           __a							=    len(cls + token_ids_a + sep) * [0]
           if token_ids_a is not None:
               result += len(token_ids_a + sep) * [1]
           return result





       def 		_lowerCamelCase      (		self							:					int					,		__SCREAMING_SNAKE_CASE							:					Dict					,		__SCREAMING_SNAKE_CASE							:					Tuple = None):


           '''simple docstring'''

           return ()










from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


__snake_case						:Tuple						=					{
    '''configuration_bloom''': ['''BLOOM_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''BloomConfig''', '''BloomOnnxConfig'''],
}
try:
 if not is_tokenizers_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __snake_case						:List[Any]						=					['''BloomTokenizerFast''']

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __snake_case						:Any						=					[
     '''BLOOM_PRETRAINED_MODEL_ARCHIVE_LIST''',
     '''BloomForCausalLM''',
     '''BloomModel''',
     '''BloomPreTrainedModel''',
     '''BloomForSequenceClassification''',
     '''BloomForTokenClassification''',
     '''BloomForQuestionAnswering''',
 ]

if TYPE_CHECKING:
 from .configuration_bloom import BLOOM_PRETRAINED_CONFIG_ARCHIVE_MAP, BloomConfig, BloomOnnxConfig

 try:
  if not is_tokenizers_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .tokenization_bloom_fast import BloomTokenizerFast

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_bloom import (
      BLOOM_PRETRAINED_MODEL_ARCHIVE_LIST,
      BloomForCausalLM,
      BloomForQuestionAnswering,
      BloomForSequenceClassification,
      BloomForTokenClassification,
      BloomModel,
      BloomPreTrainedModel,
  )

else:
 import sys

 __snake_case						:Any						=					_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)





