




"""simple docstring"""


import os
import re
import warnings
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer


if TYPE_CHECKING:
     from ...tokenization_utils_base import TextInput
from ...utils import logging


A_      =    logging.get_logger(__name__)

A_      =    {'''vocab_file''': '''spiece.model'''}

A_      =    {
    '''vocab_file''': {
        '''t5-small''': '''https://huggingface.co/t5-small/resolve/main/spiece.model''',
        '''t5-base''': '''https://huggingface.co/t5-base/resolve/main/spiece.model''',
        '''t5-large''': '''https://huggingface.co/t5-large/resolve/main/spiece.model''',
        '''t5-3b''': '''https://huggingface.co/t5-3b/resolve/main/spiece.model''',
        '''t5-11b''': '''https://huggingface.co/t5-11b/resolve/main/spiece.model''',
    }
}


# TODO(PVP) - this should be removed in Transformers v5
A_      =    {
    '''t5-small''': 5_12,
    '''t5-base''': 5_12,
    '''t5-large''': 5_12,
    '''t5-3b''': 5_12,
    '''t5-11b''': 5_12,
}

A_      =    '''▁'''






class  lowercase(					__a			):







 '''simple docstring'''





 lowercase__			 = VOCAB_FILES_NAMES
 lowercase__			 = PRETRAINED_VOCAB_FILES_MAP
 lowercase__			 = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 lowercase__			 = ["input_ids", "attention_mask"]


 def __init__(    self:		Optional[Any],				a_:		Any,				a_:		List[Any]="</s>",				a_:		Tuple="<unk>",				a_:		int="<pad>",				a_:		List[str]=100,				a_:		int=None,				a_:		Optional[Dict[str, Any]] = None,				a_:		Optional[Any]=True,				**a_:		str,				):




   '''simple docstring'''



   if extra_ids > 0 and additional_special_tokens is None:
     _snake_case							:		Tuple								=  [f"<extra_id_{i}>" for i in range(a_		)]
   elif extra_ids > 0 and additional_special_tokens is not None:
     # Check that we have the right number of extra_id special tokens
     _snake_case							:		List[Any]								=  len(set(filter(lambda a_		: bool("""extra_id""" in str(a_		)		),				a_		)		)		)
     if extra_tokens != extra_ids:
       raise ValueError(
           f"Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are"
           """ provided to T5Tokenizer. In this case the additional_special_tokens must include the extra_ids"""
           """ tokens"""		)
   if legacy:
     logger.warning_once(
         f"You are using the legacy behaviour of the {self.__class__}. This means that tokens that come after special tokens will not be properly handled. We recommend you to"
         """ read the related pull request available at https://github.com/huggingface/transformers/pull/24565"""		)

   _snake_case							:		Union[str, Any]								=  legacy
   _snake_case							:		Optional[Any]								=  {} if sp_model_kwargs is None else sp_model_kwargs

   super().__init__(
       eos_token=a_,				unk_token=a_,				pad_token=a_,				extra_ids=a_,				additional_special_tokens=a_,				sp_model_kwargs=self.sp_model_kwargs,				legacy=a_,				**a_,				)

   _snake_case							:		Tuple								=  vocab_file
   _snake_case							:		Any								=  extra_ids

   _snake_case							:		Tuple								=  spm.SentencePieceProcessor(**self.sp_model_kwargs		)
   self.sp_model.Load(a_		)


 @staticmethod
 def 			UpperCamelCase_       (    a_:		Union[str, Any],				a_:		int,				a_:		int		):




   '''simple docstring'''



   if pretrained_model_name_or_path in TaTokenizer.max_model_input_sizes:
     _snake_case							:		Tuple								=  TaTokenizer.max_model_input_sizes[pretrained_model_name_or_path]
     if init_max_model_length is not None and init_max_model_length != max_model_length:
       return init_max_model_length
     elif init_max_model_length is None:
       warnings.warn(
           """This tokenizer was incorrectly instantiated with a model max length of"""
           f" {deprecated_max_model_length} which will be corrected in Transformers v5.\nFor now, this"
           """ behavior is kept to avoid breaking backwards compatibility when padding/encoding with"""
           """ `truncation is True`.\n- Be aware that you SHOULD NOT rely on"""
           f" {pretrained_model_name_or_path} automatically truncating your input to"
           f" {deprecated_max_model_length} when padding/encoding.\n- If you want to encode/pad to sequences"
           f" longer than {deprecated_max_model_length} you can either instantiate this tokenizer with"
           """ `model_max_length` or pass `max_length` when encoding/padding.\n- To avoid this warning, please"""
           """ instantiate this tokenizer with `model_max_length` set to your preferred value.""",				a_,				)

   return max_model_length


 @property
 def 			UpperCamelCase_       (    self:		str		):




   '''simple docstring'''



   return self.sp_model.get_piece_size() + self._extra_ids


 def 			UpperCamelCase_       (    self:		List[Any]		):




   '''simple docstring'''



   _snake_case							:		List[str]								=  {self.convert_ids_to_tokens(a_		): i for i in range(self.vocab_size		)}
   vocab.update(self.added_tokens_encoder		)
   return vocab


 def 			UpperCamelCase_       (    self:		List[str],				a_:		List[int],				a_:		Optional[List[int]] = None,				a_:		bool = False		):




   '''simple docstring'''



   if already_has_special_tokens:
     return super().get_special_tokens_mask(
         token_ids_a=a_,				token_ids_a=a_,				already_has_special_tokens=a_		)

   # normal case: some special tokens
   if token_ids_a is None:
     return ([0] * len(a_		)) + [1]
   return ([0] * len(a_		)) + [1] + ([0] * len(a_		)) + [1]


 def 			UpperCamelCase_       (    self:		List[Any]		):




   '''simple docstring'''



   return list(
       set(filter(lambda a_		: bool(re.search(r"""<extra_id_\d+>""",				a_		)		) is not None,				self.additional_special_tokens		)		)		)


 def 			UpperCamelCase_       (    self:		Union[str, Any]		):




   '''simple docstring'''



   return [self._convert_token_to_id(a_		) for token in self.get_sentinel_tokens()]


 def 			UpperCamelCase_       (    self:		str,				a_:		List[int]		):




   '''simple docstring'''



   if len(a_		) > 0 and token_ids[-1] == self.eos_token_id:
     warnings.warn(
         f"This sequence already has {self.eos_token}. In future versions this behavior may lead to duplicated"
         """ eos tokens being added."""		)
     return token_ids
   else:
     return token_ids + [self.eos_token_id]


 def 			UpperCamelCase_       (    self:		List[Any],				a_:		List[int],				a_:		Optional[List[int]] = None		):




   '''simple docstring'''



   _snake_case							:		Optional[int]								=  [self.eos_token_id]

   if token_ids_a is None:
     return len(token_ids_a + eos		) * [0]
   return len(token_ids_a + eos + token_ids_a + eos		) * [0]


 def 			UpperCamelCase_       (    self:		List[str],				a_:		List[int],				a_:		Optional[List[int]] = None		):




   '''simple docstring'''



   _snake_case							:		List[str]								=  self._add_eos_if_not_present(a_		)
   if token_ids_a is None:
     return token_ids_a
   else:
     _snake_case							:		Tuple								=  self._add_eos_if_not_present(a_		)
     return token_ids_a + token_ids_a


 def __getstate__(    self:		str		):




   '''simple docstring'''



   _snake_case							:		Optional[int]								=  self.__dict__.copy()
   _snake_case							:		Optional[int]								=  None
   return state


 def __setstate__(    self:		Optional[int],				a_:		Dict		):




   '''simple docstring'''



   _snake_case							:		Optional[Any]								=  d

   # for backward compatibility
   if not hasattr(self,				"""sp_model_kwargs"""		):
     _snake_case							:		Tuple								=  {}

   _snake_case							:		int								=  spm.SentencePieceProcessor(**self.sp_model_kwargs		)
   self.sp_model.Load(self.vocab_file		)


 def 			UpperCamelCase_       (    self:		List[Any],				a_:		"TextInput",				**a_:		Optional[Any]		):




   '''simple docstring'''



   if not self.legacy:
     _snake_case							:		int								=  SPIECE_UNDERLINE + text.replace(a_,				""" """		)
   return super().tokenize(a_,				**a_		)


 def 			UpperCamelCase_       (    self:		Any,				a_:		List[str],				**a_:		Tuple		):




   '''simple docstring'''



   if not self.legacy:
     _snake_case							:		Tuple								=  text.startswith(a_		)
     if is_first:
       _snake_case							:		Optional[int]								=  text[1:]

   _snake_case							:		Optional[Any]								=  self.sp_model.encode(a_,				out_type=a_		)

   if not self.legacy and not is_first and not text.startswith(""" """		) and tokens[0].startswith(a_		):
     _snake_case							:		int								=  ([tokens[0][1:]] if len(tokens[0]		) > 1 else []) + tokens[1:]
   return tokens


 def 			UpperCamelCase_       (    self:		List[str],				a_:		Optional[Any]		):




   '''simple docstring'''



   if token.startswith("""<extra_id_"""		):
     _snake_case							:		Dict								=  re.match(r"""<extra_id_(\d+)>""",				a_		)
     _snake_case							:		int								=  int(match.group(1		)		)
     return self.vocab_size - num - 1
   return self.sp_model.piece_to_id(a_		)


 def 			UpperCamelCase_       (    self:		Union[str, Any],				a_:		Dict		):




   '''simple docstring'''



   if index < self.sp_model.get_piece_size():
     _snake_case							:		Union[str, Any]								=  self.sp_model.IdToPiece(a_		)
   else:
     _snake_case							:		int								=  f"<extra_id_{self.vocab_size - 1 - index}>"
   return token


 def 			UpperCamelCase_       (    self:		Optional[int],				a_:		Optional[int]		):




   '''simple docstring'''



   _snake_case							:		Dict								=  []
   _snake_case							:		Optional[Any]								=  """"""
   _snake_case							:		str								=  False
   for token in tokens:
     # make sure that special tokens are not decoded using sentencepiece model
     if token in self.all_special_tokens:
       if not prev_is_special:
         out_string += " "
       out_string += self.sp_model.decode(a_		) + token
       _snake_case							:		List[Any]								=  True
       _snake_case							:		str								=  []
     else:
       current_sub_tokens.append(a_		)
       _snake_case							:		Union[str, Any]								=  False
   out_string += self.sp_model.decode(a_		)
   return out_string.strip()


 def 			UpperCamelCase_       (    self:		Optional[Any],				a_:		str,				a_:		Optional[str] = None		):




   '''simple docstring'''



   if not os.path.isdir(a_		):
     logger.error(f"Vocabulary path ({save_directory}) should be a directory"		)
     return
   _snake_case							:		str								=  os.path.join(
       a_,				(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]		)

   if os.path.abspath(self.vocab_file		) != os.path.abspath(a_		) and os.path.isfile(self.vocab_file		):
     copyfile(self.vocab_file,				a_		)
   elif not os.path.isfile(self.vocab_file		):
     with open(a_,				"""wb"""		) as fi:
       _snake_case							:		Dict								=  self.sp_model.serialized_model_proto()
       fi.write(a_		)

   return (out_vocab_file,)








"""simple docstring"""


from .data_collator import (
    DataCollatorForLanguageModeling,
    DataCollatorForPermutationLanguageModeling,
    DataCollatorForSeqaSeq,
    DataCollatorForSOP,
    DataCollatorForTokenClassification,
    DataCollatorForWholeWordMask,
    DataCollatorWithPadding,
    DefaultDataCollator,
    default_data_collator,
)
from .metrics import glue_compute_metrics, xnli_compute_metrics
from .processors import (
    DataProcessor,
    InputExample,
    InputFeatures,
    SingleSentenceClassificationProcessor,
    SquadExample,
    SquadFeatures,
    SquadVaProcessor,
    SquadVaProcessor,
    glue_convert_examples_to_features,
    glue_output_modes,
    glue_processors,
    glue_tasks_num_labels,
    squad_convert_examples_to_features,
    xnli_output_modes,
    xnli_processors,
    xnli_tasks_num_labels,
)


