





"""simple docstring"""




import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
						from .tokenization_big_bird import BigBirdTokenizer
else:
						UpperCamelCase_									=None

UpperCamelCase_									=logging.get_logger(__name__)
UpperCamelCase_									={"""vocab_file""": """spiece.model""", """tokenizer_file""": """tokenizer.json"""}

UpperCamelCase_									={
    """vocab_file""": {
        """google/bigbird-roberta-base""": """https://huggingface.co/google/bigbird-roberta-base/resolve/main/spiece.model""",
        """google/bigbird-roberta-large""": (
            """https://huggingface.co/google/bigbird-roberta-large/resolve/main/spiece.model"""
        ),
        """google/bigbird-base-trivia-itc""": (
            """https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/spiece.model"""
        ),
    },
    """tokenizer_file""": {
        """google/bigbird-roberta-base""": (
            """https://huggingface.co/google/bigbird-roberta-base/resolve/main/tokenizer.json"""
        ),
        """google/bigbird-roberta-large""": (
            """https://huggingface.co/google/bigbird-roberta-large/resolve/main/tokenizer.json"""
        ),
        """google/bigbird-base-trivia-itc""": (
            """https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/tokenizer.json"""
        ),
    },
}

UpperCamelCase_									={
    """google/bigbird-roberta-base""": 4_096,
    """google/bigbird-roberta-large""": 4_096,
    """google/bigbird-base-trivia-itc""": 4_096,
}


UpperCamelCase_									="""▁"""




class 			_a						(				_lowerCAmelCase    ):
				UpperCamelCase								=   VOCAB_FILES_NAMES
				UpperCamelCase								=   PRETRAINED_VOCAB_FILES_MAP
				UpperCamelCase								=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				UpperCamelCase								=   BigBirdTokenizer
				UpperCamelCase								=   ['''input_ids''', '''attention_mask''']
				UpperCamelCase								=   []






				def __init__(			self		:   Any,			lowerCAmelCase__		:   Tuple=None,			lowerCAmelCase__		:   Tuple=None,			lowerCAmelCase__		:   Optional[Any]="<unk>",			lowerCAmelCase__		:   Optional[int]="<s>",			lowerCAmelCase__		:   List[str]="</s>",			lowerCAmelCase__		:   Optional[Any]="<pad>",			lowerCAmelCase__		:   Optional[int]="[SEP]",			lowerCAmelCase__		:   int="[MASK]",			lowerCAmelCase__		:   Dict="[CLS]",			**lowerCAmelCase__		:   Any,			)     ->				Optional[Any]:




										'''simple docstring'''



										_UpperCamelCase      : Any         =   AddedToken(lowerCAmelCase__,			lstrip=lowerCAmelCase__,			rstrip=lowerCAmelCase__							) if isinstance(lowerCAmelCase__,			lowerCAmelCase__							) else bos_token
										_UpperCamelCase      : Dict         =   AddedToken(lowerCAmelCase__,			lstrip=lowerCAmelCase__,			rstrip=lowerCAmelCase__							) if isinstance(lowerCAmelCase__,			lowerCAmelCase__							) else eos_token
										_UpperCamelCase      : List[Any]         =   AddedToken(lowerCAmelCase__,			lstrip=lowerCAmelCase__,			rstrip=lowerCAmelCase__							) if isinstance(lowerCAmelCase__,			lowerCAmelCase__							) else unk_token
										_UpperCamelCase      : Optional[Any]         =   AddedToken(lowerCAmelCase__,			lstrip=lowerCAmelCase__,			rstrip=lowerCAmelCase__							) if isinstance(lowerCAmelCase__,			lowerCAmelCase__							) else pad_token
										_UpperCamelCase      : int         =   AddedToken(lowerCAmelCase__,			lstrip=lowerCAmelCase__,			rstrip=lowerCAmelCase__							) if isinstance(lowerCAmelCase__,			lowerCAmelCase__							) else cls_token
										_UpperCamelCase      : Tuple         =   AddedToken(lowerCAmelCase__,			lstrip=lowerCAmelCase__,			rstrip=lowerCAmelCase__							) if isinstance(lowerCAmelCase__,			lowerCAmelCase__							) else sep_token

										# Mask token behave like a normal word, i.e. include the space before it
										_UpperCamelCase      : Optional[Any]         =   AddedToken(lowerCAmelCase__,			lstrip=lowerCAmelCase__,			rstrip=lowerCAmelCase__							) if isinstance(lowerCAmelCase__,			lowerCAmelCase__							) else mask_token

										super().__init__(
										    lowerCAmelCase__,			tokenizer_file=lowerCAmelCase__,			bos_token=lowerCAmelCase__,			eos_token=lowerCAmelCase__,			unk_token=lowerCAmelCase__,			sep_token=lowerCAmelCase__,			pad_token=lowerCAmelCase__,			cls_token=lowerCAmelCase__,			mask_token=lowerCAmelCase__,			**lowerCAmelCase__,			)

										_UpperCamelCase      : Tuple         =   vocab_file
										_UpperCamelCase      : Any         =   False if not self.vocab_file else True






				def     snake_case     (			self		:   Union[str, Any],			lowerCAmelCase__		:   List[int],			lowerCAmelCase__		:   Optional[List[int]] = None							)     ->				List[int]:




										'''simple docstring'''



										_UpperCamelCase      : Optional[Any]         =   [self.sep_token_id]
										_UpperCamelCase      : Optional[int]         =   [self.cls_token_id]
										if token_ids_a is None:
																return cls + token_ids_a + sep
										return cls + token_ids_a + sep + token_ids_a + sep






				def     snake_case     (			self		:   Any,			lowerCAmelCase__		:   List[int],			lowerCAmelCase__		:   Optional[List[int]] = None,			lowerCAmelCase__		:   bool = False							)     ->				List[int]:




										'''simple docstring'''



										if already_has_special_tokens:
																if token_ids_a is not None:
																						raise ValueError(
																						    '''You should not supply a second sequence if the provided sequence of '''
																						    '''ids is already formatted with special tokens for the model.'''							)
																return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

										if token_ids_a is None:
																return [1] + ([0] * len(lowerCAmelCase__							)) + [1]
										return [1] + ([0] * len(lowerCAmelCase__							)) + [1] + ([0] * len(lowerCAmelCase__							)) + [1]






				def     snake_case     (			self		:   Optional[Any],			lowerCAmelCase__		:   List[int],			lowerCAmelCase__		:   Optional[List[int]] = None							)     ->				List[int]:




										'''simple docstring'''



										_UpperCamelCase      : Any         =   [self.sep_token_id]
										_UpperCamelCase      : Dict         =   [self.cls_token_id]

										if token_ids_a is None:
																return len(cls + token_ids_a + sep							) * [0]
										return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]






				def     snake_case     (			self		:   Optional[int],			lowerCAmelCase__		:   str,			lowerCAmelCase__		:   Optional[str] = None							)     ->				Tuple[str]:




										'''simple docstring'''



										if not self.can_save_slow_tokenizer:
																raise ValueError(
																    '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
																    '''tokenizer.'''							)

										if not os.path.isdir(lowerCAmelCase__							):
																logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""							)
																return
										_UpperCamelCase      : Tuple         =   os.path.join(
										    lowerCAmelCase__,			(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']							)

										if os.path.abspath(self.vocab_file							) != os.path.abspath(lowerCAmelCase__							):
																copyfile(self.vocab_file,			lowerCAmelCase__							)

										return (out_vocab_file,)










"""simple docstring"""




UpperCamelCase_									=[
    """DownloadConfig""",
    """DownloadManager""",
    """DownloadMode""",
    """StreamingDownloadManager""",
]

from .download_config import DownloadConfig
from .download_manager import DownloadManager, DownloadMode
from .streaming_download_manager import StreamingDownloadManager



