





import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


if TYPE_CHECKING:
						from transformers.pipelines.conversational import Conversation


_SCREAMING_SNAKE_CASE 		=						logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE 		=						{"""tokenizer_file""": """tokenizer.json"""}

_SCREAMING_SNAKE_CASE 		=						{
    """tokenizer_file""": {
        """bigscience/tokenizer""": """https://huggingface.co/bigscience/tokenizer/blob/main/tokenizer.json""",
        """bigscience/bloom-560m""": """https://huggingface.co/bigscience/bloom-560m/blob/main/tokenizer.json""",
        """bigscience/bloom-1b1""": """https://huggingface.co/bigscience/bloom-1b1/blob/main/tokenizer.json""",
        """bigscience/bloom-1b7""": """https://huggingface.co/bigscience/bloom-1b7/blob/main/tokenizer.json""",
        """bigscience/bloom-3b""": """https://huggingface.co/bigscience/bloom-3b/blob/main/tokenizer.json""",
        """bigscience/bloom-7b1""": """https://huggingface.co/bigscience/bloom-7b1/blob/main/tokenizer.json""",
        """bigscience/bloom""": """https://huggingface.co/bigscience/bloom/blob/main/tokenizer.json""",
    },
}
class SCREAMING_SNAKE_CASE_  (   snake_case_   ):
						__magic_name__:     List[str]	 =    VOCAB_FILES_NAMES
						__magic_name__:     int	 =    PRETRAINED_VOCAB_FILES_MAP
						__magic_name__:     Any	 =    ["input_ids", "attention_mask"]
						__magic_name__:     Any	 =    None






						def __init__( self : str						,				_A : Optional[Any]=None						,				_A : Optional[Any]=None						,				_A : List[str]=None						,				_A : List[Any]="<unk>"						,				_A : Optional[int]="<s>"						,				_A : Optional[int]="</s>"						,				_A : int="<pad>"						,				_A : List[Any]=False						,				_A : Dict=False						,				**_A : Optional[Any]						,				)							->      int:







							"""simple docstring"""


							super().__init__(
							    _A						,				_A						,				tokenizer_file=_A						,				unk_token=_A						,				bos_token=_A						,				eos_token=_A						,				pad_token=_A						,				add_prefix_space=_A						,				clean_up_tokenization_spaces=_A						,				**_A						,				)
							snake_case_						:     List[str]     =		json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()		)
							if pre_tok_state.get('add_prefix_space'						,				_A		) != add_prefix_space:
								snake_case_						:     Dict     =		getattr(_A						,				pre_tok_state.pop('type'		)		)
								snake_case_						:     List[Any]     =		add_prefix_space
								snake_case_						:     int     =		pre_tok_class(**_A		)

							snake_case_						:     Tuple     =		add_prefix_space






						def UpperCAmelCase_			( self : List[Any]						,				*_A : Dict						,				**_A : Dict		)							->      BatchEncoding:







							"""simple docstring"""


							snake_case_						:     int     =		kwargs.get('is_split_into_words'						,				_A		)
							if not (self.add_prefix_space or not is_split_into_words):
								raise Exception(
								    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with"""
								    ' pretokenized inputs.'		)

							return super()._batch_encode_plus(*_A						,				**_A		)






						def UpperCAmelCase_			( self : int						,				*_A : Tuple						,				**_A : Optional[int]		)							->      BatchEncoding:







							"""simple docstring"""


							snake_case_						:     List[str]     =		kwargs.get('is_split_into_words'						,				_A		)

							if not (self.add_prefix_space or not is_split_into_words):
								raise Exception(
								    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with"""
								    ' pretokenized inputs.'		)

							return super()._encode_plus(*_A						,				**_A		)






						def UpperCAmelCase_			( self : Tuple						,				_A : str						,				_A : Optional[str] = None		)							->      Tuple[str]:







							"""simple docstring"""


							snake_case_						:     Any     =		self._tokenizer.model.save(_A						,				name=_A		)
							return tuple(_A		)






						def UpperCAmelCase_			( self : List[str]						,				_A : "Conversation"		)							->      List[int]:







							"""simple docstring"""


							snake_case_						:     Any     =		[]
							for is_user, text in conversation.iter_texts():
								input_ids.extend(self.encode(_A						,				add_special_tokens=_A		) + [self.eos_token_id]		)

							if len(_A		) > self.model_max_length:
								snake_case_						:     str     =		input_ids[-self.model_max_length :]
							return input_ids








import warnings

from ..trainer import Trainer
from ..utils import logging


_SCREAMING_SNAKE_CASE 		=						logging.get_logger(__name__)
class SCREAMING_SNAKE_CASE_  (   snake_case_   ):






						def __init__( self : Any						,				_A : str=None						,				**_A : Union[str, Any]		)							->      Any:







							"""simple docstring"""


							warnings.warn(
							    '`SageMakerTrainer` is deprecated and will be removed in v5 of Transformers. You can use `Trainer` '
							    'instead.'						,				_A						,				)
							super().__init__(args=_A						,				**_A		)

