






'''simple docstring'''







import multiprocessing
import os
from typing import BinaryIO, Optional, Union

import fsspec

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.json.json import Json
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader

class _lowercase     (    _UpperCAmelCase		):




							def __init__(							self:    List[Any]      ,							UpperCamelCase__:    NestedDataStructureLike[PathLike]      ,							UpperCamelCase__:    Optional[NamedSplit] = None      ,							UpperCamelCase__:    Optional[Features] = None      ,							UpperCamelCase__:    str = None      ,							UpperCamelCase__:    bool = False      ,							UpperCamelCase__:    bool = False      ,							UpperCamelCase__:    Optional[str] = None      ,							UpperCamelCase__:    Optional[int] = None      ,							**UpperCamelCase__:    Union[str, Any]      ,							):
														super().__init__(
														    lowercase_      ,							split=lowercase_      ,							features=lowercase_      ,							cache_dir=lowercase_      ,							keep_in_memory=lowercase_      ,							streaming=lowercase_      ,							num_proc=lowercase_      ,							**lowercase_      ,							)
														lowerCamelCase__ :     Optional[int]	  =	field
														lowerCamelCase__ :     Dict	  =	path_or_paths if isinstance(lowercase_      ,							lowercase_					) else {self.split: path_or_paths}
														lowerCamelCase__ :     int	  =	Json(
														    cache_dir=lowercase_      ,							data_files=lowercase_      ,							features=lowercase_      ,							field=lowercase_      ,							**lowercase_      ,							)




							def 		lowerCamelCase_    (							self:    int					):
														# Build iterable dataset
														if self.streaming:
																					lowerCamelCase__ :     Optional[Any]	  =	self.builder.as_streaming_dataset(split=self.split					)
														# Build regular (map-style) dataset
														else:
																					lowerCamelCase__ :     str	  =	None
																					lowerCamelCase__ :     Dict	  =	None
																					lowerCamelCase__ :     str	  =	None
																					lowerCamelCase__ :     List[str]	  =	None

																					self.builder.download_and_prepare(
																					    download_config=lowercase_      ,							download_mode=lowercase_      ,							verification_mode=lowercase_      ,							base_path=lowercase_      ,							num_proc=self.num_proc      ,							)
																					lowerCamelCase__ :     Any	  =	self.builder.as_dataset(
																					    split=self.split      ,							verification_mode=lowercase_      ,							in_memory=self.keep_in_memory					)
														return dataset



class _lowercase     :




							def __init__(							self:    List[Any]      ,							UpperCamelCase__:    Dataset      ,							UpperCamelCase__:    Union[PathLike, BinaryIO]      ,							UpperCamelCase__:    Optional[int] = None      ,							UpperCamelCase__:    Optional[int] = None      ,							**UpperCamelCase__:    List[Any]      ,							):
														if num_proc is not None and num_proc <= 0:
																					raise ValueError(F'''num_proc {num_proc} must be an integer > 0.'''					)

														lowerCamelCase__ :     int	  =	dataset
														lowerCamelCase__ :     Tuple	  =	path_or_buf
														lowerCamelCase__ :     Dict	  =	batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
														lowerCamelCase__ :     List[str]	  =	num_proc
														lowerCamelCase__ :     Optional[Any]	  =	"""utf-8"""
														lowerCamelCase__ :     Dict	  =	to_json_kwargs




							def 		lowerCamelCase_    (							self:    str					):
														lowerCamelCase__ :     Dict	  =	self.to_json_kwargs.pop("""path_or_buf"""      ,							lowercase_					)
														lowerCamelCase__ :     Dict	  =	self.to_json_kwargs.pop("""orient"""      ,							"""records"""					)
														lowerCamelCase__ :     Tuple	  =	self.to_json_kwargs.pop("""lines"""      ,							True if orient == """records""" else False					)
														lowerCamelCase__ :     Any	  =	self.to_json_kwargs.pop("""index"""      ,							False if orient in ["""split""", """table"""] else True					)
														lowerCamelCase__ :     Optional[Any]	  =	self.to_json_kwargs.pop("""compression"""      ,							lowercase_					)

														if compression not in [None, "infer", "gzip", "bz2", "xz"]:
																					raise NotImplementedError(F'''`datasets` currently does not support {compression} compression'''					)

														if isinstance(self.path_or_buf      ,							(str, bytes, os.PathLike)					):
																					with fsspec.open(self.path_or_buf      ,							"""wb"""      ,							compression=lowercase_					) as buffer:
																												lowerCamelCase__ :     Optional[Any]	  =	self._write(file_obj=lowercase_      ,							orient=lowercase_      ,							lines=lowercase_      ,							index=lowercase_      ,							**self.to_json_kwargs					)
														else:
																					if compression:
																												raise NotImplementedError(
																												    F'''The compression parameter is not supported when writing to a buffer, but compression={compression}'''
																												    """ was passed. Please provide a local path instead."""					)
																					lowerCamelCase__ :     List[Any]	  =	self._write(
																					    file_obj=self.path_or_buf      ,							orient=lowercase_      ,							lines=lowercase_      ,							index=lowercase_      ,							**self.to_json_kwargs					)
														return written




							def 		lowerCamelCase_    (							self:    str      ,							UpperCamelCase__:    Union[str, Any]					):
														lowerCamelCase__ :     Optional[int]	  =	args

														lowerCamelCase__ :     List[str]	  =	query_table(
														    table=self.dataset.data      ,							key=slice(lowercase_      ,							offset + self.batch_size					)      ,							indices=self.dataset._indices      ,							)
														lowerCamelCase__ :     str	  =	batch.to_pandas().to_json(
														    path_or_buf=lowercase_      ,							orient=lowercase_      ,							lines=lowercase_      ,							index=lowercase_      ,							**lowercase_					)
														if not json_str.endswith("""\n"""					):
																					json_str += "\n"
														return json_str.encode(self.encoding					)




							def 		lowerCamelCase_    (							self:    List[Any]      ,							UpperCamelCase__:    BinaryIO      ,							UpperCamelCase__:    List[Any]      ,							UpperCamelCase__:    Tuple      ,							UpperCamelCase__:    Tuple      ,							**UpperCamelCase__:    str      ,							):
														lowerCamelCase__ :     Dict	  =	0

														if self.num_proc is None or self.num_proc == 1:
																					for offset in logging.tqdm(
																					    range(0      ,							len(self.dataset					)      ,							self.batch_size					)      ,							unit="""ba"""      ,							disable=not logging.is_progress_bar_enabled()      ,							desc="""Creating json from Arrow format"""      ,							):
																												lowerCamelCase__ :     List[str]	  =	self._batch_json((offset, orient, lines, index, to_json_kwargs)					)
																												written += file_obj.write(lowercase_					)
														else:
																					lowerCamelCase__ :     Any	  =	len(self.dataset					), self.batch_size
																					with multiprocessing.Pool(self.num_proc					) as pool:
																												for json_str in logging.tqdm(
																												    pool.imap(
																												        self._batch_json      ,							[(offset, orient, lines, index, to_json_kwargs) for offset in range(0      ,							lowercase_      ,							lowercase_					)]      ,							)      ,							total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size      ,							unit="""ba"""      ,							disable=not logging.is_progress_bar_enabled()      ,							desc="""Creating json from Arrow format"""      ,							):
																																			written += file_obj.write(lowercase_					)

														return written









'''simple docstring'''







import os
import re
import warnings
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer


if TYPE_CHECKING:
			from ...tokenization_utils_base import TextInput
from ...utils import logging


_A     :      Union[str, Any]   					=logging.get_logger(__name__)

_A     :      Optional[Any]   					={'''vocab_file''': '''spiece.model'''}

_A     :      Optional[Any]   					={
    '''vocab_file''': {
        '''t5-small''': '''https://huggingface.co/t5-small/resolve/main/spiece.model''',
        '''t5-base''': '''https://huggingface.co/t5-base/resolve/main/spiece.model''',
        '''t5-large''': '''https://huggingface.co/t5-large/resolve/main/spiece.model''',
        '''t5-3b''': '''https://huggingface.co/t5-3b/resolve/main/spiece.model''',
        '''t5-11b''': '''https://huggingface.co/t5-11b/resolve/main/spiece.model''',
    }
}


# TODO(PVP) - this should be removed in Transformers v5
_A     :      Union[str, Any]   					={
    '''t5-small''': 512,
    '''t5-base''': 512,
    '''t5-large''': 512,
    '''t5-3b''': 512,
    '''t5-11b''': 512,
}

_A     :      int   					='''▁'''

class _lowercase     (    _lowercase		):
							a    				=	VOCAB_FILES_NAMES
							a    				=	PRETRAINED_VOCAB_FILES_MAP
							a    				=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							a    				=	["""input_ids""", """attention_mask"""]




							def __init__(							self:    int      ,							UpperCamelCase__:    int      ,							UpperCamelCase__:    List[str]="</s>"      ,							UpperCamelCase__:    Optional[Any]="<unk>"      ,							UpperCamelCase__:    Dict="<pad>"      ,							UpperCamelCase__:    List[Any]=100      ,							UpperCamelCase__:    Dict=None      ,							UpperCamelCase__:    Optional[Dict[str, Any]] = None      ,							UpperCamelCase__:    Union[str, Any]=True      ,							**UpperCamelCase__:    Dict      ,							):
														# Add extra_ids to the special token list
														if extra_ids > 0 and additional_special_tokens is None:
																					lowerCamelCase__ :     Union[str, Any]	  =	[F'''<extra_id_{i}>''' for i in range(UpperCamelCase__					)]
														elif extra_ids > 0 and additional_special_tokens is not None:
																					# Check that we have the right number of extra_id special tokens
																					lowerCamelCase__ :     Optional[Any]	  =	len(set(filter(lambda UpperCamelCase__					: bool("""extra_id""" in str(UpperCamelCase__					)					)      ,							UpperCamelCase__					)					)					)
																					if extra_tokens != extra_ids:
																												raise ValueError(
																												    F'''Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are'''
																												    """ provided to T5Tokenizer. In this case the additional_special_tokens must include the extra_ids"""
																												    """ tokens"""					)
														if legacy:
																					logger.warning_once(
																					    F'''You are using the legacy behaviour of the {self.__class__}. This means that tokens that come after special tokens will not be properly handled. We recommend you to'''
																					    """ read the related pull request available at https://github.com/huggingface/transformers/pull/24565"""					)

														lowerCamelCase__ :     Optional[int]	  =	legacy
														lowerCamelCase__ :     Any	  =	{} if sp_model_kwargs is None else sp_model_kwargs

														super().__init__(
														    eos_token=UpperCamelCase__      ,							unk_token=UpperCamelCase__      ,							pad_token=UpperCamelCase__      ,							extra_ids=UpperCamelCase__      ,							additional_special_tokens=UpperCamelCase__      ,							sp_model_kwargs=self.sp_model_kwargs      ,							legacy=UpperCamelCase__      ,							**UpperCamelCase__      ,							)

														lowerCamelCase__ :     Tuple	  =	vocab_file
														lowerCamelCase__ :     Dict	  =	extra_ids

														lowerCamelCase__ :     Optional[Any]	  =	spm.SentencePieceProcessor(**self.sp_model_kwargs					)
														self.sp_model.Load(UpperCamelCase__					)




							@staticmethod
							def 		lowerCamelCase_    (							UpperCamelCase__:    Union[str, Any]      ,							UpperCamelCase__:    Optional[Any]      ,							UpperCamelCase__:    int					):
														if pretrained_model_name_or_path in TaTokenizer.max_model_input_sizes:
																					lowerCamelCase__ :     Any	  =	TaTokenizer.max_model_input_sizes[pretrained_model_name_or_path]
																					if init_max_model_length is not None and init_max_model_length != max_model_length:
																												return init_max_model_length
																					elif init_max_model_length is None:
																												warnings.warn(
																												    """This tokenizer was incorrectly instantiated with a model max length of"""
																												    F''' {deprecated_max_model_length} which will be corrected in Transformers v5.\nFor now, this'''
																												    """ behavior is kept to avoid breaking backwards compatibility when padding/encoding with"""
																												    """ `truncation is True`.\n- Be aware that you SHOULD NOT rely on"""
																												    F''' {pretrained_model_name_or_path} automatically truncating your input to'''
																												    F''' {deprecated_max_model_length} when padding/encoding.\n- If you want to encode/pad to sequences'''
																												    F''' longer than {deprecated_max_model_length} you can either instantiate this tokenizer with'''
																												    """ `model_max_length` or pass `max_length` when encoding/padding.\n- To avoid this warning, please"""
																												    """ instantiate this tokenizer with `model_max_length` set to your preferred value."""      ,							UpperCamelCase__      ,							)

														return max_model_length




							@property
							def 		lowerCamelCase_    (							self:    Any					):
														return self.sp_model.get_piece_size() + self._extra_ids




							def 		lowerCamelCase_    (							self:    Dict					):
														lowerCamelCase__ :     str	  =	{self.convert_ids_to_tokens(UpperCamelCase__					): i for i in range(self.vocab_size					)}
														vocab.update(self.added_tokens_encoder					)
														return vocab




							def 		lowerCamelCase_    (							self:    Optional[Any]      ,							UpperCamelCase__:    List[int]      ,							UpperCamelCase__:    Optional[List[int]] = None      ,							UpperCamelCase__:    bool = False					):
														if already_has_special_tokens:
																					return super().get_special_tokens_mask(
																					    token_ids_a=UpperCamelCase__      ,							token_ids_a=UpperCamelCase__      ,							already_has_special_tokens=UpperCamelCase__					)

														# normal case: some special tokens
														if token_ids_a is None:
																					return ([0] * len(UpperCamelCase__					)) + [1]
														return ([0] * len(UpperCamelCase__					)) + [1] + ([0] * len(UpperCamelCase__					)) + [1]




							def 		lowerCamelCase_    (							self:    Dict					):
														return list(
														    set(filter(lambda UpperCamelCase__					: bool(re.search(R"""<extra_id_\d+>"""      ,							UpperCamelCase__					)					) is not None      ,							self.additional_special_tokens					)					)					)




							def 		lowerCamelCase_    (							self:    str					):
														return [self._convert_token_to_id(UpperCamelCase__					) for token in self.get_sentinel_tokens()]




							def 		lowerCamelCase_    (							self:    Optional[Any]      ,							UpperCamelCase__:    List[int]					):
														if len(UpperCamelCase__					) > 0 and token_ids[-1] == self.eos_token_id:
																					warnings.warn(
																					    F'''This sequence already has {self.eos_token}. In future versions this behavior may lead to duplicated'''
																					    """ eos tokens being added."""					)
																					return token_ids
														else:
																					return token_ids + [self.eos_token_id]




							def 		lowerCamelCase_    (							self:    str      ,							UpperCamelCase__:    List[int]      ,							UpperCamelCase__:    Optional[List[int]] = None					):
														lowerCamelCase__ :     Optional[Any]	  =	[self.eos_token_id]

														if token_ids_a is None:
																					return len(token_ids_a + eos					) * [0]
														return len(token_ids_a + eos + token_ids_a + eos					) * [0]




							def 		lowerCamelCase_    (							self:    int      ,							UpperCamelCase__:    List[int]      ,							UpperCamelCase__:    Optional[List[int]] = None					):
														lowerCamelCase__ :     List[str]	  =	self._add_eos_if_not_present(UpperCamelCase__					)
														if token_ids_a is None:
																					return token_ids_a
														else:
																					lowerCamelCase__ :     int	  =	self._add_eos_if_not_present(UpperCamelCase__					)
																					return token_ids_a + token_ids_a




							def __getstate__(							self:    List[str]					):
														lowerCamelCase__ :     Optional[int]	  =	self.__dict__.copy()
														lowerCamelCase__ :     Optional[Any]	  =	None
														return state




							def __setstate__(							self:    List[Any]      ,							UpperCamelCase__:    Any					):
														lowerCamelCase__ :     Tuple	  =	d

														# for backward compatibility
														if not hasattr(self      ,							"""sp_model_kwargs"""					):
																					lowerCamelCase__ :     str	  =	{}

														lowerCamelCase__ :     Optional[Any]	  =	spm.SentencePieceProcessor(**self.sp_model_kwargs					)
														self.sp_model.Load(self.vocab_file					)




							def 		lowerCamelCase_    (							self:    Optional[Any]      ,							UpperCamelCase__:    "TextInput"      ,							**UpperCamelCase__:    List[str]					):
														# Replace the SPIECE_UNDERLINE with a space to make sure SPIECE_UNDERLINE is only used at
														# the beginning of the text
														if not self.legacy:
																					lowerCamelCase__ :     List[Any]	  =	SPIECE_UNDERLINE + text.replace(UpperCamelCase__      ,							""" """					)
														return super().tokenize(UpperCamelCase__      ,							**UpperCamelCase__					)




							def 		lowerCamelCase_    (							self:    List[Any]      ,							UpperCamelCase__:    str      ,							**UpperCamelCase__:    str					):
														if not self.legacy:
																					lowerCamelCase__ :     List[Any]	  =	text.startswith(UpperCamelCase__					)
																					if is_first:
																												lowerCamelCase__ :     Optional[int]	  =	text[1:]

														lowerCamelCase__ :     int	  =	self.sp_model.encode(UpperCamelCase__      ,							out_type=UpperCamelCase__					)

														if not self.legacy and not is_first and not text.startswith(""" """					) and tokens[0].startswith(UpperCamelCase__					):
																					lowerCamelCase__ :     str	  =	([tokens[0][1:]] if len(tokens[0]					) > 1 else []) + tokens[1:]
														return tokens




							def 		lowerCamelCase_    (							self:    Optional[int]      ,							UpperCamelCase__:    Optional[Any]					):
														if token.startswith("""<extra_id_"""					):
																					lowerCamelCase__ :     List[Any]	  =	re.match(R"""<extra_id_(\d+)>"""      ,							UpperCamelCase__					)
																					lowerCamelCase__ :     List[Any]	  =	int(match.group(1					)					)
																					return self.vocab_size - num - 1
														return self.sp_model.piece_to_id(UpperCamelCase__					)




							def 		lowerCamelCase_    (							self:    List[str]      ,							UpperCamelCase__:    int					):
														if index < self.sp_model.get_piece_size():
																					lowerCamelCase__ :     str	  =	self.sp_model.IdToPiece(UpperCamelCase__					)
														else:
																					lowerCamelCase__ :     Tuple	  =	F'''<extra_id_{self.vocab_size - 1 - index}>'''
														return token




							def 		lowerCamelCase_    (							self:    str      ,							UpperCamelCase__:    Tuple					):
														lowerCamelCase__ :     str	  =	[]
														lowerCamelCase__ :     Any	  =	""""""
														lowerCamelCase__ :     Union[str, Any]	  =	False
														for token in tokens:
																					# make sure that special tokens are not decoded using sentencepiece model
																					if token in self.all_special_tokens:
																												if not prev_is_special:
																																			out_string += " "
																												out_string += self.sp_model.decode(UpperCamelCase__					) + token
																												lowerCamelCase__ :     Dict	  =	True
																												lowerCamelCase__ :     str	  =	[]
																					else:
																												current_sub_tokens.append(UpperCamelCase__					)
																												lowerCamelCase__ :     List[str]	  =	False
														out_string += self.sp_model.decode(UpperCamelCase__					)
														return out_string.strip()




							def 		lowerCamelCase_    (							self:    Optional[int]      ,							UpperCamelCase__:    str      ,							UpperCamelCase__:    Optional[str] = None					):
														if not os.path.isdir(UpperCamelCase__					):
																					logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''					)
																					return
														lowerCamelCase__ :     List[Any]	  =	os.path.join(
														    UpperCamelCase__      ,							(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]					)

														if os.path.abspath(self.vocab_file					) != os.path.abspath(UpperCamelCase__					) and os.path.isfile(self.vocab_file					):
																					copyfile(self.vocab_file      ,							UpperCamelCase__					)
														elif not os.path.isfile(self.vocab_file					):
																					with open(UpperCamelCase__      ,							"""wb"""					) as fi:
																												lowerCamelCase__ :     Union[str, Any]	  =	self.sp_model.serialized_model_proto()
																												fi.write(UpperCamelCase__					)

														return (out_vocab_file,)

