





"""simple docstring"""




import os
import re
import warnings
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
						from .tokenization_ta import TaTokenizer
else:
						_SCREAMING_SNAKE_CASE 		=						None


_SCREAMING_SNAKE_CASE 		=						logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE 		=						{"""vocab_file""": """spiece.model""", """tokenizer_file""": """tokenizer.json"""}

_SCREAMING_SNAKE_CASE 		=						{
    """vocab_file""": {
        """t5-small""": """https://huggingface.co/t5-small/resolve/main/spiece.model""",
        """t5-base""": """https://huggingface.co/t5-base/resolve/main/spiece.model""",
        """t5-large""": """https://huggingface.co/t5-large/resolve/main/spiece.model""",
        """t5-3b""": """https://huggingface.co/t5-3b/resolve/main/spiece.model""",
        """t5-11b""": """https://huggingface.co/t5-11b/resolve/main/spiece.model""",
    },
    """tokenizer_file""": {
        """t5-small""": """https://huggingface.co/t5-small/resolve/main/tokenizer.json""",
        """t5-base""": """https://huggingface.co/t5-base/resolve/main/tokenizer.json""",
        """t5-large""": """https://huggingface.co/t5-large/resolve/main/tokenizer.json""",
        """t5-3b""": """https://huggingface.co/t5-3b/resolve/main/tokenizer.json""",
        """t5-11b""": """https://huggingface.co/t5-11b/resolve/main/tokenizer.json""",
    },
}


# TODO(PVP) - this should be removed in Transformers v5
_SCREAMING_SNAKE_CASE 		=						{
    """t5-small""": 5_12,
    """t5-base""": 5_12,
    """t5-large""": 5_12,
    """t5-3b""": 5_12,
    """t5-11b""": 5_12,
}
class SCREAMING_SNAKE_CASE_  (   __lowercase   ):
						__magic_name__:     List[str]	 =    VOCAB_FILES_NAMES
						__magic_name__:     Optional[int]	 =    PRETRAINED_VOCAB_FILES_MAP
						__magic_name__:     Dict	 =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						__magic_name__:     Any	 =    ['''input_ids''', '''attention_mask''']
						__magic_name__:     Any	 =    TaTokenizer

						__magic_name__:     List[int]	 =    []






						def __init__( self : List[str]						,				_A : Dict=None						,				_A : Optional[int]=None						,				_A : List[str]="</s>"						,				_A : Any="<unk>"						,				_A : Any="<pad>"						,				_A : int=100						,				_A : List[Any]=None						,				**_A : Optional[Any]						,				)							->      Tuple:







							"""simple docstring"""


							if extra_ids > 0 and additional_special_tokens is None:
								snake_case_						:     Optional[int]     =		[F"""<extra_id_{i}>""" for i in range(_A		)]
							elif extra_ids > 0 and additional_special_tokens is not None:
								# Check that we have the right number of extra special tokens
								snake_case_						:     Tuple     =		len(set(filter(lambda _A		: bool('extra_id_' in str(_A		)		)						,				_A		)		)		)
								if extra_tokens != extra_ids:
									raise ValueError(
									    F"""Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are"""
									    ' provided to T5Tokenizer. In this case the additional_special_tokens must include the extra_ids'
									    ' tokens'		)

							super().__init__(
							    _A						,				tokenizer_file=_A						,				eos_token=_A						,				unk_token=_A						,				pad_token=_A						,				extra_ids=_A						,				additional_special_tokens=_A						,				**_A						,				)

							snake_case_						:     Optional[int]     =		vocab_file
							snake_case_						:     Any     =		False if not self.vocab_file else True
							snake_case_						:     Union[str, Any]     =		extra_ids






						@staticmethod
						def UpperCAmelCase_			( _A : Tuple						,				_A : Any						,				_A : str		)							->      Union[str, Any]:







							"""simple docstring"""


							if pretrained_model_name_or_path in TaTokenizerFast.max_model_input_sizes:
								snake_case_						:     Dict     =		TaTokenizerFast.max_model_input_sizes[pretrained_model_name_or_path]
								if init_max_model_length is not None and init_max_model_length != max_model_length:
									return init_max_model_length
								elif init_max_model_length is None:
									warnings.warn(
									    'This tokenizer was incorrectly instantiated with a model max length of'
									    F""" {deprecated_max_model_length} which will be corrected in Transformers v5.\nFor now, this"""
									    ' behavior is kept to avoid breaking backwards compatibility when padding/encoding with'
									    ' `truncation is True`.\n- Be aware that you SHOULD NOT rely on'
									    F""" {pretrained_model_name_or_path} automatically truncating your input to"""
									    F""" {deprecated_max_model_length} when padding/encoding.\n- If you want to encode/pad to sequences"""
									    F""" longer than {deprecated_max_model_length} you can either instantiate this tokenizer with"""
									    ' `model_max_length` or pass `max_length` when encoding/padding.\n- To avoid this warning, please'
									    ' instantiate this tokenizer with `model_max_length` set to your preferred value.'						,				_A						,				)

							return max_model_length






						def UpperCAmelCase_			( self : Union[str, Any]						,				_A : str						,				_A : Optional[str] = None		)							->      Tuple[str]:







							"""simple docstring"""


							if not self.can_save_slow_tokenizer:
								raise ValueError(
								    'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
								    'tokenizer.'		)

							if not os.path.isdir(_A		):
								logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""		)
								return
							snake_case_						:     Dict     =		os.path.join(
							    _A						,				(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']		)

							if os.path.abspath(self.vocab_file		) != os.path.abspath(_A		):
								copyfile(self.vocab_file						,				_A		)
								logger.info(F"""Copy vocab file to {out_vocab_file}"""		)

							return (out_vocab_file,)






						def UpperCAmelCase_			( self : int						,				_A : List[int]						,				_A : Optional[List[int]] = None		)							->      List[int]:







							"""simple docstring"""


							snake_case_						:     Dict     =		token_ids_a + [self.eos_token_id]
							if token_ids_a is None:
								return self.prefix_tokens + token_ids_a
							else:
								snake_case_						:     int     =		token_ids_a + [self.eos_token_id]
								return self.prefix_tokens + token_ids_a + token_ids_a






						def UpperCAmelCase_			( self : str						,				_A : List[int]						,				_A : Optional[List[int]] = None		)							->      List[int]:







							"""simple docstring"""


							snake_case_						:     Optional[Any]     =		[self.eos_token_id]

							if token_ids_a is None:
								return len(token_ids_a + eos		) * [0]
							return len(token_ids_a + eos + token_ids_a + eos		) * [0]






						def UpperCAmelCase_			( self : str		)							->      Union[str, Any]:







							"""simple docstring"""


							return list(
							    set(filter(lambda _A		: bool(re.search(R'<extra_id_\d+>'						,				_A		)		) is not None						,				self.additional_special_tokens		)		)		)






						def UpperCAmelCase_			( self : Optional[Any]		)							->      Tuple:







							"""simple docstring"""


							return [self.convert_tokens_to_ids(_A		) for token in self.get_sentinel_tokens()]








from typing import TYPE_CHECKING

from ....utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


_SCREAMING_SNAKE_CASE 		=						{
    """configuration_mctct""": ["""MCTCT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """MCTCTConfig"""],
    """feature_extraction_mctct""": ["""MCTCTFeatureExtractor"""],
    """processing_mctct""": ["""MCTCTProcessor"""],
}


try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						_SCREAMING_SNAKE_CASE 		=						[
						    """MCTCT_PRETRAINED_MODEL_ARCHIVE_LIST""",
						    """MCTCTForCTC""",
						    """MCTCTModel""",
						    """MCTCTPreTrainedModel""",
						]


if TYPE_CHECKING:
						from .configuration_mctct import MCTCT_PRETRAINED_CONFIG_ARCHIVE_MAP, MCTCTConfig
						from .feature_extraction_mctct import MCTCTFeatureExtractor
						from .processing_mctct import MCTCTProcessor

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_mctct import MCTCT_PRETRAINED_MODEL_ARCHIVE_LIST, MCTCTForCTC, MCTCTModel, MCTCTPreTrainedModel

else:
						import sys

						_SCREAMING_SNAKE_CASE 		=						_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)

