


"""simple docstring"""



import unittest

from transformers import (
    MODEL_FOR_CAUSAL_LM_MAPPING,
    TF_MODEL_FOR_CAUSAL_LM_MAPPING,
    TextGenerationPipeline,
    logging,
    pipeline,
)
from transformers.testing_utils import (
    CaptureLogger,
    is_pipeline_test,
    require_accelerate,
    require_tf,
    require_torch,
    require_torch_gpu,
    require_torch_or_tf,
)

from .test_pipelines_common import ANY







@is_pipeline_test
@require_torch_or_tf
class _lowerCAmelCase    (				unittest.TestCase     ):
   __UpperCAmelCase			:	Dict							  =  MODEL_FOR_CAUSAL_LM_MAPPING
   __UpperCAmelCase			:	Tuple							  =  TF_MODEL_FOR_CAUSAL_LM_MAPPING







   @require_torch
   def 			lowerCamelCase		(     self						)     ->		Union[str, Any]:

       '''simple docstring'''




       snake_case   :				Tuple				 =   pipeline(task="text-generation"						,     model="sshleifer/tiny-ctrl"						,     framework="pt"						)
       # Using `do_sample=False` to force deterministic output
       snake_case   :				Any				 =   text_generator("This is a test"						,     do_sample=UpperCamelCase__						)
       self.assertEqual(
           UpperCamelCase__						,     [
               {
                   "generated_text": (
                       "This is a test ☃ ☃ segmental segmental segmental 议议eski eski flutter flutter Lacy oscope."
                       " oscope. FiliFili@@"
                   )
               }
           ]						,     )

       snake_case   :				int				 =   text_generator(["This is a test", "This is a second test"]						)
       self.assertEqual(
           UpperCamelCase__						,     [
               [
                   {
                       "generated_text": (
                           "This is a test ☃ ☃ segmental segmental segmental 议议eski eski flutter flutter Lacy oscope."
                           " oscope. FiliFili@@"
                       )
                   }
               ],
               [
                   {
                       "generated_text": (
                           "This is a second test ☃ segmental segmental segmental 议议eski eski flutter flutter Lacy"
                           " oscope. oscope. FiliFili@@"
                       )
                   }
               ],
           ]						,     )

       snake_case   :				Any				 =   text_generator("This is a test"						,     do_sample=UpperCamelCase__						,     num_return_sequences=2						,     return_tensors=UpperCamelCase__						)
       self.assertEqual(
           UpperCamelCase__						,     [
               {"generated_token_ids": ANY(UpperCamelCase__						)},
               {"generated_token_ids": ANY(UpperCamelCase__						)},
           ]						,     )
       snake_case   :				Any				 =   text_generator.model.config.eos_token_id
       snake_case   :				Optional[Any]				 =   "<pad>"
       snake_case   :				Optional[int]				 =   text_generator(
           ["This is a test", "This is a second test"]						,     do_sample=UpperCamelCase__						,     num_return_sequences=2						,     batch_size=2						,     return_tensors=UpperCamelCase__						,     )
       self.assertEqual(
           UpperCamelCase__						,     [
               [
                   {"generated_token_ids": ANY(UpperCamelCase__						)},
                   {"generated_token_ids": ANY(UpperCamelCase__						)},
               ],
               [
                   {"generated_token_ids": ANY(UpperCamelCase__						)},
                   {"generated_token_ids": ANY(UpperCamelCase__						)},
               ],
           ]						,     )







   @require_tf
   def 			lowerCamelCase		(     self						)     ->		Dict:

       '''simple docstring'''




       snake_case   :				Optional[Any]				 =   pipeline(task="text-generation"						,     model="sshleifer/tiny-ctrl"						,     framework="tf"						)

       # Using `do_sample=False` to force deterministic output
       snake_case   :				Any				 =   text_generator("This is a test"						,     do_sample=UpperCamelCase__						)
       self.assertEqual(
           UpperCamelCase__						,     [
               {
                   "generated_text": (
                       "This is a test FeyFeyFey(Croatis.), s.), Cannes Cannes Cannes 閲閲Cannes Cannes Cannes 攵"
                       " please,"
                   )
               }
           ]						,     )

       snake_case   :				Optional[int]				 =   text_generator(["This is a test", "This is a second test"]						,     do_sample=UpperCamelCase__						)
       self.assertEqual(
           UpperCamelCase__						,     [
               [
                   {
                       "generated_text": (
                           "This is a test FeyFeyFey(Croatis.), s.), Cannes Cannes Cannes 閲閲Cannes Cannes Cannes 攵"
                           " please,"
                       )
                   }
               ],
               [
                   {
                       "generated_text": (
                           "This is a second test Chieftain Chieftain prefecture prefecture prefecture Cannes Cannes"
                           " Cannes 閲閲Cannes Cannes Cannes 攵 please,"
                       )
                   }
               ],
           ]						,     )







   def 			lowerCamelCase		(     self						,     UpperCamelCase__						,     UpperCamelCase__						,     UpperCamelCase__						)     ->		str:

       '''simple docstring'''




       snake_case   :				Dict				 =   TextGenerationPipeline(model=UpperCamelCase__						,     tokenizer=UpperCamelCase__						)
       return text_generator, ["This is a test", "Another test"]







   def 			lowerCamelCase		(     self						)     ->		List[str]:

       '''simple docstring'''




       snake_case   :				List[Any]				 =   "Hello I believe in"
       snake_case   :				Dict				 =   pipeline("text-generation"						,     model="hf-internal-testing/tiny-random-gpt2"						)
       snake_case   :				List[Any]				 =   text_generator(UpperCamelCase__						)
       self.assertEqual(
           UpperCamelCase__						,     [{"generated_text": "Hello I believe in fe fe fe fe fe fe fe fe fe fe fe fe"}]						,     )

       snake_case   :				int				 =   text_generator(UpperCamelCase__						,     stop_sequence=" fe"						)
       self.assertEqual(UpperCamelCase__						,     [{"generated_text": "Hello I believe in fe"}]						)







   def 			lowerCamelCase		(     self						,     UpperCamelCase__						,     UpperCamelCase__						)     ->		Optional[int]:

       '''simple docstring'''




       snake_case   :				List[Any]				 =   text_generator.model
       snake_case   :				List[Any]				 =   text_generator.tokenizer

       snake_case   :				List[Any]				 =   text_generator("This is a test"						)
       self.assertEqual(UpperCamelCase__						,     [{"generated_text": ANY(UpperCamelCase__						)}]						)
       self.assertTrue(outputs[0]["generated_text"].startswith("This is a test"						)						)

       snake_case   :				Optional[Any]				 =   text_generator("This is a test"						,     return_full_text=UpperCamelCase__						)
       self.assertEqual(UpperCamelCase__						,     [{"generated_text": ANY(UpperCamelCase__						)}]						)
       self.assertNotIn("This is a test"						,     outputs[0]["generated_text"]						)

       snake_case   :				Tuple				 =   pipeline(task="text-generation"						,     model=UpperCamelCase__						,     tokenizer=UpperCamelCase__						,     return_full_text=UpperCamelCase__						)
       snake_case   :				Dict				 =   text_generator("This is a test"						)
       self.assertEqual(UpperCamelCase__						,     [{"generated_text": ANY(UpperCamelCase__						)}]						)
       self.assertNotIn("This is a test"						,     outputs[0]["generated_text"]						)

       snake_case   :				List[str]				 =   text_generator("This is a test"						,     return_full_text=UpperCamelCase__						)
       self.assertEqual(UpperCamelCase__						,     [{"generated_text": ANY(UpperCamelCase__						)}]						)
       self.assertTrue(outputs[0]["generated_text"].startswith("This is a test"						)						)

       snake_case   :				Optional[int]				 =   text_generator(["This is great !", "Something else"]						,     num_return_sequences=2						,     do_sample=UpperCamelCase__						)
       self.assertEqual(
           UpperCamelCase__						,     [
               [{"generated_text": ANY(UpperCamelCase__						)}, {"generated_text": ANY(UpperCamelCase__						)}],
               [{"generated_text": ANY(UpperCamelCase__						)}, {"generated_text": ANY(UpperCamelCase__						)}],
           ]						,     )

       if text_generator.tokenizer.pad_token is not None:
           snake_case   :				Union[str, Any]				 =   text_generator(
               ["This is great !", "Something else"]						,     num_return_sequences=2						,     batch_size=2						,     do_sample=UpperCamelCase__						)
           self.assertEqual(
               UpperCamelCase__						,     [
                   [{"generated_text": ANY(UpperCamelCase__						)}, {"generated_text": ANY(UpperCamelCase__						)}],
                   [{"generated_text": ANY(UpperCamelCase__						)}, {"generated_text": ANY(UpperCamelCase__						)}],
               ]						,     )

       with self.assertRaises(UpperCamelCase__						):
           snake_case   :				Tuple				 =   text_generator("test"						,     return_full_text=UpperCamelCase__						,     return_text=UpperCamelCase__						)
       with self.assertRaises(UpperCamelCase__						):
           snake_case   :				Dict				 =   text_generator("test"						,     return_full_text=UpperCamelCase__						,     return_tensors=UpperCamelCase__						)
       with self.assertRaises(UpperCamelCase__						):
           snake_case   :				int				 =   text_generator("test"						,     return_text=UpperCamelCase__						,     return_tensors=UpperCamelCase__						)

       # Empty prompt is slighly special
       # it requires BOS token to exist.
       # Special case for Pegasus which will always append EOS so will
       # work even without BOS.
       if (
           text_generator.tokenizer.bos_token_id is not None
           or "Pegasus" in tokenizer.__class__.__name__
           or "Git" in model.__class__.__name__
       ):
           snake_case   :				str				 =   text_generator(""						)
           self.assertEqual(UpperCamelCase__						,     [{"generated_text": ANY(UpperCamelCase__						)}]						)
       else:
           with self.assertRaises((ValueError, AssertionError)						):
               snake_case   :				Any				 =   text_generator(""						)

       if text_generator.framework == "tf":
           # TF generation does not support max_new_tokens, and it's impossible
           # to control long generation with only max_length without
           # fancy calculation, dismissing tests for now.
           return
       # We don't care about infinite range models.
       # They already work.
       # Skip this test for XGLM, since it uses sinusoidal positional embeddings which are resized on-the-fly.
       snake_case   :				int				 =   ["RwkvForCausalLM", "XGLMForCausalLM", "GPTNeoXForCausalLM"]
       if (
           tokenizer.model_max_length < 1_0000
           and text_generator.model.__class__.__name__ not in EXTRA_MODELS_CAN_HANDLE_LONG_INPUTS
       ):
           # Handling of large generations
           with self.assertRaises((RuntimeError, IndexError, ValueError, AssertionError)						):
               text_generator("This is a test" * 500						,     max_new_tokens=20						)

           snake_case   :				Tuple				 =   text_generator("This is a test" * 500						,     handle_long_generation="hole"						,     max_new_tokens=20						)
           # Hole strategy cannot work
           with self.assertRaises(UpperCamelCase__						):
               text_generator(
                   "This is a test" * 500						,     handle_long_generation="hole"						,     max_new_tokens=tokenizer.model_max_length + 10						,     )







   @require_torch
   @require_accelerate
   @require_torch_gpu
   def 			lowerCamelCase		(     self						)     ->		int:

       '''simple docstring'''




       import torch

       # Classic `model_kwargs`
       snake_case   :				List[Any]				 =   pipeline(
           model="hf-internal-testing/tiny-random-bloom"						,     model_kwargs={"device_map": "auto", "torch_dtype": torch.bfloataa}						,     )
       self.assertEqual(pipe.model.device						,     torch.device(0						)						)
       self.assertEqual(pipe.model.lm_head.weight.dtype						,     torch.bfloataa						)
       snake_case   :				Union[str, Any]				 =   pipe("This is a test"						)
       self.assertEqual(
           UpperCamelCase__						,     [
               {
                   "generated_text": (
                       "This is a test test test test test test test test test test test test test test test test"
                       " test"
                   )
               }
           ]						,     )

       # Upgraded those two to real pipeline arguments (they just get sent for the model as they're unlikely to mean anything else.)
       snake_case   :				List[Any]				 =   pipeline(model="hf-internal-testing/tiny-random-bloom"						,     device_map="auto"						,     torch_dtype=torch.bfloataa						)
       self.assertEqual(pipe.model.device						,     torch.device(0						)						)
       self.assertEqual(pipe.model.lm_head.weight.dtype						,     torch.bfloataa						)
       snake_case   :				Tuple				 =   pipe("This is a test"						)
       self.assertEqual(
           UpperCamelCase__						,     [
               {
                   "generated_text": (
                       "This is a test test test test test test test test test test test test test test test test"
                       " test"
                   )
               }
           ]						,     )

       # torch_dtype will be automatically set to float32 if not provided - check: https://github.com/huggingface/transformers/pull/20602
       snake_case   :				Optional[int]				 =   pipeline(model="hf-internal-testing/tiny-random-bloom"						,     device_map="auto"						)
       self.assertEqual(pipe.model.device						,     torch.device(0						)						)
       self.assertEqual(pipe.model.lm_head.weight.dtype						,     torch.floataa						)
       snake_case   :				str				 =   pipe("This is a test"						)
       self.assertEqual(
           UpperCamelCase__						,     [
               {
                   "generated_text": (
                       "This is a test test test test test test test test test test test test test test test test"
                       " test"
                   )
               }
           ]						,     )







   @require_torch
   @require_torch_gpu
   def 			lowerCamelCase		(     self						)     ->		Optional[Any]:

       '''simple docstring'''




       import torch

       snake_case   :				Dict				 =   pipeline(model="hf-internal-testing/tiny-random-bloom"						,     device=0						,     torch_dtype=torch.floataa						)
       pipe("This is a test"						)







   @require_torch
   @require_accelerate
   @require_torch_gpu
   def 			lowerCamelCase		(     self						)     ->		Optional[int]:

       '''simple docstring'''




       import torch

       snake_case   :				Optional[int]				 =   pipeline(model="hf-internal-testing/tiny-random-bloom"						,     device_map="auto"						,     torch_dtype=torch.floataa						)
       pipe("This is a test"						,     do_sample=UpperCamelCase__						,     top_p=0.5						)







   def 			lowerCamelCase		(     self						)     ->		int:

       '''simple docstring'''




       snake_case   :				Any				 =   "Hello world"
       snake_case   :				List[Any]				 =   pipeline("text-generation"						,     model="hf-internal-testing/tiny-random-gpt2"						)
       if text_generator.model.framework == "tf":
           snake_case   :				Optional[Any]				 =   logging.get_logger("transformers.generation.tf_utils"						)
       else:
           snake_case   :				Optional[int]				 =   logging.get_logger("transformers.generation.utils"						)
       snake_case   :				Any				 =   "Both `max_new_tokens`"  # The beggining of the message to be checked in this test

       # Both are set by the user -> log warning
       with CaptureLogger(UpperCamelCase__						) as cl:
           snake_case   :				Union[str, Any]				 =   text_generator(UpperCamelCase__						,     max_length=10						,     max_new_tokens=1						)
       self.assertIn(UpperCamelCase__						,     cl.out						)

       # The user only sets one -> no warning
       with CaptureLogger(UpperCamelCase__						) as cl:
           snake_case   :				Union[str, Any]				 =   text_generator(UpperCamelCase__						,     max_new_tokens=1						)
       self.assertNotIn(UpperCamelCase__						,     cl.out						)

       with CaptureLogger(UpperCamelCase__						) as cl:
           snake_case   :				str				 =   text_generator(UpperCamelCase__						,     max_length=10						)
       self.assertNotIn(UpperCamelCase__						,     cl.out						)









"""simple docstring"""



import warnings
from typing import List, Optional, Union

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
from ...utils import TensorType







class _lowerCAmelCase    (				snake_case_     ):
   __UpperCAmelCase			:	List[str]							  =  ['''image_processor''', '''tokenizer''']
   __UpperCAmelCase			:	str							  =  '''LayoutLMv2ImageProcessor'''
   __UpperCAmelCase			:	Dict							  =  ('''LayoutXLMTokenizer''', '''LayoutXLMTokenizerFast''')







   def __init__(     self						,     UpperCamelCase__=None						,     UpperCamelCase__=None						,     **UpperCamelCase__						)     ->		Union[str, Any]:

       '''simple docstring'''




       if "feature_extractor" in kwargs:
           warnings.warn(
               "The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`"
               " instead."						,     UpperCamelCase__						,     )
           snake_case   :				Dict				 =   kwargs.pop("feature_extractor"						)

       snake_case   :				int				 =   image_processor if image_processor is not None else feature_extractor
       if image_processor is None:
           raise ValueError("You need to specify an `image_processor`."						)
       if tokenizer is None:
           raise ValueError("You need to specify a `tokenizer`."						)

       super().__init__(UpperCamelCase__						,     UpperCamelCase__						)







   def __call__(     self						,     UpperCamelCase__						,     UpperCamelCase__ = None						,     UpperCamelCase__ = None						,     UpperCamelCase__ = None						,     UpperCamelCase__ = None						,     UpperCamelCase__ = True						,     UpperCamelCase__ = False						,     UpperCamelCase__ = None						,     UpperCamelCase__ = None						,     UpperCamelCase__ = 0						,     UpperCamelCase__ = None						,     UpperCamelCase__ = None						,     UpperCamelCase__ = None						,     UpperCamelCase__ = False						,     UpperCamelCase__ = False						,     UpperCamelCase__ = False						,     UpperCamelCase__ = False						,     UpperCamelCase__ = True						,     UpperCamelCase__ = None						,     **UpperCamelCase__						,     )     ->		BatchEncoding:

       '''simple docstring'''




       if self.image_processor.apply_ocr and (boxes is not None):
           raise ValueError(
               "You cannot provide bounding boxes "
               "if you initialized the image processor with apply_ocr set to True."						)

       if self.image_processor.apply_ocr and (word_labels is not None):
           raise ValueError(
               "You cannot provide word labels if you initialized the image processor with apply_ocr set to True."						)

       if return_overflowing_tokens is True and return_offsets_mapping is False:
           raise ValueError("You cannot return overflowing tokens without returning the offsets mapping."						)

       # first, apply the image processor
       snake_case   :				Any				 =   self.image_processor(images=UpperCamelCase__						,     return_tensors=UpperCamelCase__						)

       # second, apply the tokenizer
       if text is not None and self.image_processor.apply_ocr and text_pair is None:
           if isinstance(UpperCamelCase__						,     UpperCamelCase__						):
               snake_case   :				Optional[int]				 =   [text]  # add batch dimension (as the image processor always adds a batch dimension)
           snake_case   :				Optional[Any]				 =   features["words"]

       snake_case   :				Dict				 =   self.tokenizer(
           text=text if text is not None else features["words"]						,     text_pair=text_pair if text_pair is not None else None						,     boxes=boxes if boxes is not None else features["boxes"]						,     word_labels=UpperCamelCase__						,     add_special_tokens=UpperCamelCase__						,     padding=UpperCamelCase__						,     truncation=UpperCamelCase__						,     max_length=UpperCamelCase__						,     stride=UpperCamelCase__						,     pad_to_multiple_of=UpperCamelCase__						,     return_token_type_ids=UpperCamelCase__						,     return_attention_mask=UpperCamelCase__						,     return_overflowing_tokens=UpperCamelCase__						,     return_special_tokens_mask=UpperCamelCase__						,     return_offsets_mapping=UpperCamelCase__						,     return_length=UpperCamelCase__						,     verbose=UpperCamelCase__						,     return_tensors=UpperCamelCase__						,     **UpperCamelCase__						,     )

       # add pixel values
       snake_case   :				Dict				 =   features.pop("pixel_values"						)
       if return_overflowing_tokens is True:
           snake_case   :				Any				 =   self.get_overflowing_images(UpperCamelCase__						,     encoded_inputs["overflow_to_sample_mapping"]						)
       snake_case   :				str				 =   images

       return encoded_inputs







   def 			lowerCamelCase		(     self						,     UpperCamelCase__						,     UpperCamelCase__						)     ->		str:

       '''simple docstring'''




       snake_case   :				Union[str, Any]				 =   []
       for sample_idx in overflow_to_sample_mapping:
           images_with_overflow.append(images[sample_idx]						)

       if len(UpperCamelCase__						) != len(UpperCamelCase__						):
           raise ValueError(
               "Expected length of images to be the same as the length of `overflow_to_sample_mapping`, but got"
               F' {len(UpperCamelCase__						)} and {len(UpperCamelCase__						)}'						)

       return images_with_overflow







   def 			lowerCamelCase		(     self						,     *UpperCamelCase__						,     **UpperCamelCase__						)     ->		Optional[Any]:

       '''simple docstring'''




       return self.tokenizer.batch_decode(*UpperCamelCase__						,     **UpperCamelCase__						)







   def 			lowerCamelCase		(     self						,     *UpperCamelCase__						,     **UpperCamelCase__						)     ->		Tuple:

       '''simple docstring'''




       return self.tokenizer.decode(*UpperCamelCase__						,     **UpperCamelCase__						)







   @property
   def 			lowerCamelCase		(     self						)     ->		Dict:

       '''simple docstring'''




       return ["input_ids", "bbox", "attention_mask", "image"]







   @property
   def 			lowerCamelCase		(     self						)     ->		Optional[Any]:

       '''simple docstring'''




       warnings.warn(
           "`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead."						,     UpperCamelCase__						,     )
       return self.image_processor_class







   @property
   def 			lowerCamelCase		(     self						)     ->		Any:

       '''simple docstring'''




       warnings.warn(
           "`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead."						,     UpperCamelCase__						,     )
       return self.image_processor





