


'''simple docstring'''



import unittest

from transformers import (
    MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
    TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
    TextaTextGenerationPipeline,
    pipeline,
)
from transformers.testing_utils import is_pipeline_test, require_tf, require_torch
from transformers.utils import is_torch_available

from .test_pipelines_common import ANY


if is_torch_available():
	import torch






@is_pipeline_test
class 	UpperCamelCase__(       unittest.TestCase	):
	__magic_name__    :				Optional[int]  		=	MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING
	__magic_name__    :				Tuple  		=	TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING




	def 					a__(		self      :			str				,   lowerCAmelCase      :			Optional[Any]				,   lowerCAmelCase      :			Optional[Any]				,   lowerCAmelCase      :			Union[str, Any]				)->					Dict:

							"""simple docstring"""




							UpperCAmelCase					    =     TextaTextGenerationPipeline(model=__a				,   tokenizer=__a				)
							return generator, ["Something to write", "Something else"]




	def 					a__(		self      :			List[Any]				,   lowerCAmelCase      :			Any				,   lowerCAmelCase      :			int				)->					Optional[Any]:

							"""simple docstring"""




							UpperCAmelCase					    =     generator('''Something there'''				)
							self.assertEqual(__a				,   [{'''generated_text''': ANY(__a				)}]				)
							# These are encoder decoder, they don't just append to incoming string
							self.assertFalse(outputs[0]['''generated_text'''].startswith('''Something there'''				)				)

							UpperCAmelCase					    =     generator(['''This is great !''', '''Something else''']				,   num_return_sequences=2				,   do_sample=__a				)
							self.assertEqual(
							    __a				,   [
							        [{'''generated_text''': ANY(__a				)}, {'''generated_text''': ANY(__a				)}],
							        [{'''generated_text''': ANY(__a				)}, {'''generated_text''': ANY(__a				)}],
							    ]				,   )

							UpperCAmelCase					    =     generator(
							    ['''This is great !''', '''Something else''']				,   num_return_sequences=2				,   batch_size=2				,   do_sample=__a				)
							self.assertEqual(
							    __a				,   [
							        [{'''generated_text''': ANY(__a				)}, {'''generated_text''': ANY(__a				)}],
							        [{'''generated_text''': ANY(__a				)}, {'''generated_text''': ANY(__a				)}],
							    ]				,   )

							with self.assertRaises(__a				):
													generator(4				)




	@require_torch
	def 					a__(		self      :			List[Any]				)->					Union[str, Any]:

							"""simple docstring"""




							UpperCAmelCase					    =     pipeline('''text2text-generation'''				,   model='''patrickvonplaten/t5-tiny-random'''				,   framework='''pt'''				)
							# do_sample=False necessary for reproducibility
							UpperCAmelCase					    =     generator('''Something there'''				,   do_sample=__a				)
							self.assertEqual(__a				,   [{'''generated_text''': ''''''}]				)

							UpperCAmelCase					    =     3
							UpperCAmelCase					    =     generator(
							    '''Something there'''				,   num_return_sequences=__a				,   num_beams=__a				,   )
							UpperCAmelCase					    =     [
							    {'''generated_text''': '''Beide Beide Beide Beide Beide Beide Beide Beide Beide'''},
							    {'''generated_text''': '''Beide Beide Beide Beide Beide Beide Beide Beide'''},
							    {'''generated_text''': ''''''},
							]
							self.assertEqual(__a				,   __a				)

							UpperCAmelCase					    =     generator('''This is a test'''				,   do_sample=__a				,   num_return_sequences=2				,   return_tensors=__a				)
							self.assertEqual(
							    __a				,   [
							        {'''generated_token_ids''': ANY(torch.Tensor				)},
							        {'''generated_token_ids''': ANY(torch.Tensor				)},
							    ]				,   )
							UpperCAmelCase					    =     generator.model.config.eos_token_id
							UpperCAmelCase					    =     '''<pad>'''
							UpperCAmelCase					    =     generator(
							    ['''This is a test''', '''This is a second test''']				,   do_sample=__a				,   num_return_sequences=2				,   batch_size=2				,   return_tensors=__a				,   )
							self.assertEqual(
							    __a				,   [
							        [
							            {'''generated_token_ids''': ANY(torch.Tensor				)},
							            {'''generated_token_ids''': ANY(torch.Tensor				)},
							        ],
							        [
							            {'''generated_token_ids''': ANY(torch.Tensor				)},
							            {'''generated_token_ids''': ANY(torch.Tensor				)},
							        ],
							    ]				,   )





	@require_tf
	def 					a__(		self      :			List[Any]				)->					int:

							"""simple docstring"""




							UpperCAmelCase					    =     pipeline('''text2text-generation'''				,   model='''patrickvonplaten/t5-tiny-random'''				,   framework='''tf'''				)
							# do_sample=False necessary for reproducibility
							UpperCAmelCase					    =     generator('''Something there'''				,   do_sample=__a				)
							self.assertEqual(__a				,   [{'''generated_text''': ''''''}]				)









'''simple docstring'''



import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


if TYPE_CHECKING:
	from transformers.pipelines.conversational import Conversation


_lowercase      :      Optional[Any]            =					logging.get_logger(__name__)

_lowercase      :      List[str]            =					{"""tokenizer_file""": """tokenizer.json"""}

_lowercase      :      Optional[Any]            =					{
    """tokenizer_file""": {
        """bigscience/tokenizer""": """https://huggingface.co/bigscience/tokenizer/blob/main/tokenizer.json""",
        """bigscience/bloom-560m""": """https://huggingface.co/bigscience/bloom-560m/blob/main/tokenizer.json""",
        """bigscience/bloom-1b1""": """https://huggingface.co/bigscience/bloom-1b1/blob/main/tokenizer.json""",
        """bigscience/bloom-1b7""": """https://huggingface.co/bigscience/bloom-1b7/blob/main/tokenizer.json""",
        """bigscience/bloom-3b""": """https://huggingface.co/bigscience/bloom-3b/blob/main/tokenizer.json""",
        """bigscience/bloom-7b1""": """https://huggingface.co/bigscience/bloom-7b1/blob/main/tokenizer.json""",
        """bigscience/bloom""": """https://huggingface.co/bigscience/bloom/blob/main/tokenizer.json""",
    },
}






class 	UpperCamelCase__(       lowerCAmelCase	):
				__magic_name__    :				List[str]  		=	VOCAB_FILES_NAMES
				__magic_name__    :				Optional[int]  		=	PRETRAINED_VOCAB_FILES_MAP
				__magic_name__    :				Any  		=	["input_ids", "attention_mask"]
				__magic_name__    :				Optional[int]  		=	None




				def __init__(		self      :			List[Any]				,   lowerCAmelCase      :			str=None				,   lowerCAmelCase      :			Tuple=None				,   lowerCAmelCase      :			str=None				,   lowerCAmelCase      :			Optional[int]="<unk>"				,   lowerCAmelCase      :			int="<s>"				,   lowerCAmelCase      :			Dict="</s>"				,   lowerCAmelCase      :			Union[str, Any]="<pad>"				,   lowerCAmelCase      :			int=False				,   lowerCAmelCase      :			Optional[int]=False				,   **lowerCAmelCase      :			Optional[int]				,   )->					Union[str, Any]:

										"""simple docstring"""




										super().__init__(
										    lowerCAmelCase				,   lowerCAmelCase				,   tokenizer_file=lowerCAmelCase				,   unk_token=lowerCAmelCase				,   bos_token=lowerCAmelCase				,   eos_token=lowerCAmelCase				,   pad_token=lowerCAmelCase				,   add_prefix_space=lowerCAmelCase				,   clean_up_tokenization_spaces=lowerCAmelCase				,   **lowerCAmelCase				,   )
										UpperCAmelCase					    =     json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()				)
										if pre_tok_state.get('''add_prefix_space'''				,   lowerCAmelCase				) != add_prefix_space:
																UpperCAmelCase					    =     getattr(lowerCAmelCase				,   pre_tok_state.pop('''type'''				)				)
																UpperCAmelCase					    =     add_prefix_space
																UpperCAmelCase					    =     pre_tok_class(**lowerCAmelCase				)

										UpperCAmelCase					    =     add_prefix_space




				def 					a__(		self      :			str				,   *lowerCAmelCase      :			Dict				,   **lowerCAmelCase      :			List[str]				)->					BatchEncoding:

										"""simple docstring"""




										UpperCAmelCase					    =     kwargs.get('''is_split_into_words'''				,   lowerCAmelCase				)
										if not (self.add_prefix_space or not is_split_into_words):
																raise Exception(
																    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with"""
																    ''' pretokenized inputs.'''				)

										return super()._batch_encode_plus(*lowerCAmelCase				,   **lowerCAmelCase				)




				def 					a__(		self      :			Union[str, Any]				,   *lowerCAmelCase      :			List[str]				,   **lowerCAmelCase      :			Union[str, Any]				)->					BatchEncoding:

										"""simple docstring"""




										UpperCAmelCase					    =     kwargs.get('''is_split_into_words'''				,   lowerCAmelCase				)

										if not (self.add_prefix_space or not is_split_into_words):
																raise Exception(
																    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with"""
																    ''' pretokenized inputs.'''				)

										return super()._encode_plus(*lowerCAmelCase				,   **lowerCAmelCase				)




				def 					a__(		self      :			Union[str, Any]				,   lowerCAmelCase      :			str				,   lowerCAmelCase      :			Optional[str] = None				)->					Tuple[str]:

										"""simple docstring"""




										UpperCAmelCase					    =     self._tokenizer.model.save(lowerCAmelCase				,   name=lowerCAmelCase				)
										return tuple(lowerCAmelCase				)





				def 					a__(		self      :			List[Any]				,   lowerCAmelCase      :			"Conversation"				)->					List[int]:

										"""simple docstring"""




										UpperCAmelCase					    =     []
										for is_user, text in conversation.iter_texts():
																input_ids.extend(self.encode(lowerCAmelCase				,   add_special_tokens=lowerCAmelCase				) + [self.eos_token_id]				)

										if len(lowerCAmelCase				) > self.model_max_length:
																UpperCAmelCase					    =     input_ids[-self.model_max_length :]
										return input_ids





