


"""simple docstring"""



import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
  from .tokenization_pegasus import PegasusTokenizer
else:
  __snake_case            =   None


__snake_case            =   logging.get_logger(__name__)


__snake_case            =   """▁"""

__snake_case            =   {"""vocab_file""": """spiece.model""", """tokenizer_file""": """tokenizer.json"""}

__snake_case            =   {
    """vocab_file""": {"""google/pegasus-xsum""": """https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model"""},
    """tokenizer_file""": {
        """google/pegasus-xsum""": """https://huggingface.co/google/pegasus-xsum/resolve/main/tokenizer.json"""
    },
}

__snake_case            =   {
    """google/pegasus-xsum""": 512,
}







class _lowerCAmelCase    (				_a     ):
   __UpperCAmelCase			:	Any							  =  VOCAB_FILES_NAMES
   __UpperCAmelCase			:	Dict							  =  PRETRAINED_VOCAB_FILES_MAP
   __UpperCAmelCase			:	str							  =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   __UpperCAmelCase			:	Optional[Any]							  =  PegasusTokenizer
   __UpperCAmelCase			:	str							  =  ['''input_ids''', '''attention_mask''']







   def __init__(     self						,     UpperCamelCase__=None						,     UpperCamelCase__=None						,     UpperCamelCase__="<pad>"						,     UpperCamelCase__="</s>"						,     UpperCamelCase__="<unk>"						,     UpperCamelCase__="<mask_2>"						,     UpperCamelCase__="<mask_1>"						,     UpperCamelCase__=None						,     UpperCamelCase__=103						,     **UpperCamelCase__						,     )     ->		Union[str, Any]:

       '''simple docstring'''




       snake_case   :				Union[str, Any]				 =   offset

       if additional_special_tokens is not None:
           if not isinstance(snake_case_						,     snake_case_						):
               raise TypeError(
                   F'additional_special_tokens should be of type {type(snake_case_						)}, but is'
                   F' {type(snake_case_						)}'						)

           snake_case   :				Any				 =   (
               ([mask_token_sent] + additional_special_tokens)
               if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
               else additional_special_tokens
           )
           # fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
           additional_special_tokens_extended += [
               F'<unk_{i}>' for i in range(len(snake_case_						)						,     self.offset - 1						)
           ]

           if len(set(snake_case_						)						) != len(snake_case_						):
               raise ValueError(
                   "Please make sure that the provided additional_special_tokens do not contain an incorrectly"
                   F' shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}.'						)
           snake_case   :				int				 =   additional_special_tokens_extended
       else:
           snake_case   :				List[Any]				 =   [mask_token_sent] if mask_token_sent is not None else []
           additional_special_tokens += [F'<unk_{i}>' for i in range(2						,     self.offset						)]

       super().__init__(
           snake_case_						,     tokenizer_file=snake_case_						,     pad_token=snake_case_						,     eos_token=snake_case_						,     unk_token=snake_case_						,     mask_token=snake_case_						,     mask_token_sent=snake_case_						,     offset=snake_case_						,     additional_special_tokens=snake_case_						,     **snake_case_						,     )
       snake_case   :				Dict				 =   vocab_file
       snake_case   :				Optional[int]				 =   False if not self.vocab_file else True







   def 			lowerCamelCase		(     self						,     UpperCamelCase__						)     ->		int:

       '''simple docstring'''




       snake_case   :				Dict				 =   set(self.all_special_ids						)  # call it once instead of inside list comp
       all_special_ids.remove(self.unk_token_id						)  # <unk> is only sometimes special

       if all_special_ids != set(range(len(self.additional_special_tokens						) + 3						)						):
           raise ValueError(
               "There should be 3 special tokens: mask_token, pad_token, and eos_token +"
               F' {len(self.additional_special_tokens						)} additional_special_tokens, but got {all_special_ids}'						)

       return [1 if x in all_special_ids else 0 for x in seq]







   def 			lowerCamelCase		(     self						,     UpperCamelCase__						,     UpperCamelCase__ = None						,     UpperCamelCase__ = False						)     ->		Any:

       '''simple docstring'''




       if already_has_special_tokens:
           return self._special_token_mask(snake_case_						)
       elif token_ids_a is None:
           return self._special_token_mask(snake_case_						) + [1]
       else:
           return self._special_token_mask(token_ids_a + token_ids_a						) + [1]







   def 			lowerCamelCase		(     self						,     UpperCamelCase__						,     UpperCamelCase__=None						)     ->		Optional[int]:

       '''simple docstring'''




       if token_ids_a is None:
           return token_ids_a + [self.eos_token_id]
       # We don't expect to process pairs, but leave the pair logic for API consistency
       return token_ids_a + token_ids_a + [self.eos_token_id]







   def 			lowerCamelCase		(     self						,     UpperCamelCase__						,     UpperCamelCase__ = None						)     ->		Dict:

       '''simple docstring'''




       if not self.can_save_slow_tokenizer:
           raise ValueError(
               "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
               "tokenizer."						)

       if not os.path.isdir(snake_case_						):
           logger.error(F'Vocabulary path ({save_directory}) should be a directory'						)
           return
       snake_case   :				Optional[int]				 =   os.path.join(
           snake_case_						,     (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]						)

       if os.path.abspath(self.vocab_file						) != os.path.abspath(snake_case_						):
           copyfile(self.vocab_file						,     snake_case_						)

       return (out_vocab_file,)









"""simple docstring"""



from . import __version__

# Backward compatibility imports, to make sure all those objects can be found in file_utils
from .utils import (
    CLOUDFRONT_DISTRIB_PREFIX,
    CONFIG_NAME,
    DISABLE_TELEMETRY,
    DUMMY_INPUTS,
    DUMMY_MASK,
    ENV_VARS_TRUE_AND_AUTO_VALUES,
    ENV_VARS_TRUE_VALUES,
    FEATURE_EXTRACTOR_NAME,
    FLAX_WEIGHTS_NAME,
    HF_MODULES_CACHE,
    HUGGINGFACE_CO_PREFIX,
    HUGGINGFACE_CO_RESOLVE_ENDPOINT,
    MODEL_CARD_NAME,
    MULTIPLE_CHOICE_DUMMY_INPUTS,
    PYTORCH_PRETRAINED_BERT_CACHE,
    PYTORCH_TRANSFORMERS_CACHE,
    S3_BUCKET_PREFIX,
    SENTENCEPIECE_UNDERLINE,
    SPIECE_UNDERLINE,
    TF2_WEIGHTS_NAME,
    TF_WEIGHTS_NAME,
    TORCH_FX_REQUIRED_VERSION,
    TRANSFORMERS_CACHE,
    TRANSFORMERS_DYNAMIC_MODULE_NAME,
    USE_JAX,
    USE_TF,
    USE_TORCH,
    WEIGHTS_INDEX_NAME,
    WEIGHTS_NAME,
    ContextManagers,
    DummyObject,
    EntryNotFoundError,
    ExplicitEnum,
    ModelOutput,
    PaddingStrategy,
    PushToHubMixin,
    RepositoryNotFoundError,
    RevisionNotFoundError,
    TensorType,
    _LazyModule,
    add_code_sample_docstrings,
    add_end_docstrings,
    add_start_docstrings,
    add_start_docstrings_to_model_forward,
    cached_property,
    copy_func,
    default_cache_path,
    define_sagemaker_information,
    get_cached_models,
    get_file_from_repo,
    get_full_repo_name,
    get_torch_version,
    has_file,
    http_user_agent,
    is_apex_available,
    is_bsa_available,
    is_coloredlogs_available,
    is_datasets_available,
    is_detectrona_available,
    is_faiss_available,
    is_flax_available,
    is_ftfy_available,
    is_in_notebook,
    is_ipex_available,
    is_librosa_available,
    is_offline_mode,
    is_onnx_available,
    is_pandas_available,
    is_phonemizer_available,
    is_protobuf_available,
    is_psutil_available,
    is_pyanvml_available,
    is_pyctcdecode_available,
    is_pytesseract_available,
    is_pytorch_quantization_available,
    is_rjieba_available,
    is_sagemaker_dp_enabled,
    is_sagemaker_mp_enabled,
    is_scipy_available,
    is_sentencepiece_available,
    is_seqio_available,
    is_sklearn_available,
    is_soundfile_availble,
    is_spacy_available,
    is_speech_available,
    is_tensor,
    is_tensorflow_probability_available,
    is_tfaonnx_available,
    is_tf_available,
    is_timm_available,
    is_tokenizers_available,
    is_torch_available,
    is_torch_bfaa_available,
    is_torch_cuda_available,
    is_torch_fx_available,
    is_torch_fx_proxy,
    is_torch_mps_available,
    is_torch_tfaa_available,
    is_torch_tpu_available,
    is_torchaudio_available,
    is_training_run_on_sagemaker,
    is_vision_available,
    replace_return_docstrings,
    requires_backends,
    to_numpy,
    to_py_obj,
    torch_only_method,
)





