


'''simple docstring'''



from typing import Dict, List, Optional, Union

import numpy as np

from transformers.utils import is_vision_available
from transformers.utils.generic import TensorType

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    is_valid_image,
    to_numpy_array,
    valid_images,
)
from ...utils import logging


if is_vision_available():
	import PIL

_lowercase      :      Union[str, Any]            =					logging.get_logger(__name__)


def 				lowerCamelCase__				(	A       :      Any					):




						'''simple docstring'''






						if isinstance(a__				,       (list, tuple)					) and isinstance(videos[0]				,       (list, tuple)					) and is_valid_image(videos[0][0]					):
												return videos

						elif isinstance(a__				,       (list, tuple)					) and is_valid_image(videos[0]					):
												return [videos]

						elif is_valid_image(a__					):
												return [[videos]]

						raise ValueError(f"""Could not make batched video from {videos}"""					)






class 	UpperCamelCase__(       _a		):
				__magic_name__    :				Optional[int]  		=	["""pixel_values"""]




				def __init__(		self      :			List[Any]				,   lowerCAmelCase      :			Optional[Any] = True				,   lowerCAmelCase      :			List[str] = None				,   lowerCAmelCase      :			List[str] = PILImageResampling.BILINEAR				,   lowerCAmelCase      :			List[Any] = True				,   lowerCAmelCase      :			Optional[Any] = None				,   lowerCAmelCase      :			str = True				,   lowerCAmelCase      :			Union[str, Any] = 1 / 255				,   lowerCAmelCase      :			Optional[Any] = True				,   lowerCAmelCase      :			int = True				,   lowerCAmelCase      :			Tuple = None				,   lowerCAmelCase      :			Tuple = None				,   **lowerCAmelCase      :			int				,   )->					int:

										"""simple docstring"""




										super().__init__(**__lowerCAmelCase				)
										UpperCAmelCase					    =     size if size is not None else {'''shortest_edge''': 256}
										UpperCAmelCase					    =     get_size_dict(__lowerCAmelCase				,   default_to_square=__lowerCAmelCase				)
										UpperCAmelCase					    =     crop_size if crop_size is not None else {'''height''': 224, '''width''': 224}
										UpperCAmelCase					    =     get_size_dict(__lowerCAmelCase				,   param_name='''crop_size'''				)

										UpperCAmelCase					    =     do_resize
										UpperCAmelCase					    =     size
										UpperCAmelCase					    =     do_center_crop
										UpperCAmelCase					    =     crop_size
										UpperCAmelCase					    =     resample
										UpperCAmelCase					    =     do_rescale
										UpperCAmelCase					    =     rescale_factor
										UpperCAmelCase					    =     offset
										UpperCAmelCase					    =     do_normalize
										UpperCAmelCase					    =     image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
										UpperCAmelCase					    =     image_std if image_std is not None else IMAGENET_STANDARD_STD




				def 					a__(		self      :			str				,   lowerCAmelCase      :			int				,   lowerCAmelCase      :			Tuple				,   lowerCAmelCase      :			Tuple = PILImageResampling.BILINEAR				,   lowerCAmelCase      :			Union[str, Any] = None				,   **lowerCAmelCase      :			Optional[int]				,   )->					Union[str, Any]:

										"""simple docstring"""




										UpperCAmelCase					    =     get_size_dict(__lowerCAmelCase				,   default_to_square=__lowerCAmelCase				)
										if "shortest_edge" in size:
																UpperCAmelCase					    =     get_resize_output_image_size(__lowerCAmelCase				,   size['''shortest_edge''']				,   default_to_square=__lowerCAmelCase				)
										elif "height" in size and "width" in size:
																UpperCAmelCase					    =     (size['''height'''], size['''width'''])
										else:
																raise ValueError(F"""Size must have 'height' and 'width' or 'shortest_edge' as keys. Got {size.keys()}"""				)
										return resize(__lowerCAmelCase				,   size=__lowerCAmelCase				,   resample=__lowerCAmelCase				,   data_format=__lowerCAmelCase				,   **__lowerCAmelCase				)




				def 					a__(		self      :			Union[str, Any]				,   lowerCAmelCase      :			Optional[int]				,   lowerCAmelCase      :			str				,   lowerCAmelCase      :			List[Any] = None				,   **lowerCAmelCase      :			List[Any]				,   )->					int:

										"""simple docstring"""




										UpperCAmelCase					    =     get_size_dict(__lowerCAmelCase				)
										if "height" not in size or "width" not in size:
																raise ValueError(F"""Size must have 'height' and 'width' as keys. Got {size.keys()}"""				)
										return center_crop(__lowerCAmelCase				,   size=(size['''height'''], size['''width'''])				,   data_format=__lowerCAmelCase				,   **__lowerCAmelCase				)




				def 					a__(		self      :			Optional[Any]				,   lowerCAmelCase      :			Optional[int]				,   lowerCAmelCase      :			List[str]				,   lowerCAmelCase      :			Dict = True				,   lowerCAmelCase      :			Any = None				,   **lowerCAmelCase      :			List[str]				,   )->					Union[str, Any]:

										"""simple docstring"""




										UpperCAmelCase					    =     image.astype(np.floataa				)
										if offset:
																UpperCAmelCase					    =     image - (scale / 2)
										return rescale(__lowerCAmelCase				,   scale=__lowerCAmelCase				,   data_format=__lowerCAmelCase				,   **__lowerCAmelCase				)




				def 					a__(		self      :			List[Any]				,   lowerCAmelCase      :			List[str]				,   lowerCAmelCase      :			Dict				,   lowerCAmelCase      :			Any				,   lowerCAmelCase      :			str = None				,   **lowerCAmelCase      :			List[Any]				,   )->					int:

										"""simple docstring"""




										return normalize(__lowerCAmelCase				,   mean=__lowerCAmelCase				,   std=__lowerCAmelCase				,   data_format=__lowerCAmelCase				,   **__lowerCAmelCase				)




				def 					a__(		self      :			str				,   lowerCAmelCase      :			Optional[int]				,   lowerCAmelCase      :			str = None				,   lowerCAmelCase      :			Any = None				,   lowerCAmelCase      :			Optional[int] = None				,   lowerCAmelCase      :			List[Any] = None				,   lowerCAmelCase      :			Dict = None				,   lowerCAmelCase      :			str = None				,   lowerCAmelCase      :			Optional[int] = None				,   lowerCAmelCase      :			str = None				,   lowerCAmelCase      :			Union[str, Any] = None				,   lowerCAmelCase      :			List[Any] = None				,   lowerCAmelCase      :			Tuple = None				,   lowerCAmelCase      :			Any = ChannelDimension.FIRST				,   )->					Union[str, Any]:

										"""simple docstring"""




										if do_resize and size is None or resample is None:
																raise ValueError('''Size and resample must be specified if do_resize is True.'''				)

										if do_center_crop and crop_size is None:
																raise ValueError('''Crop size must be specified if do_center_crop is True.'''				)

										if do_rescale and rescale_factor is None:
																raise ValueError('''Rescale factor must be specified if do_rescale is True.'''				)

										if do_normalize and (image_mean is None or image_std is None):
																raise ValueError('''Image mean and std must be specified if do_normalize is True.'''				)

										if offset and not do_rescale:
																raise ValueError('''For offset, do_rescale must also be set to True.'''				)

										# All transformations expect numpy arrays.
										UpperCAmelCase					    =     to_numpy_array(__lowerCAmelCase				)

										if do_resize:
																UpperCAmelCase					    =     self.resize(image=__lowerCAmelCase				,   size=__lowerCAmelCase				,   resample=__lowerCAmelCase				)

										if do_center_crop:
																UpperCAmelCase					    =     self.center_crop(__lowerCAmelCase				,   size=__lowerCAmelCase				)

										if do_rescale:
																UpperCAmelCase					    =     self.rescale(image=__lowerCAmelCase				,   scale=__lowerCAmelCase				,   offset=__lowerCAmelCase				)

										if do_normalize:
																UpperCAmelCase					    =     self.normalize(image=__lowerCAmelCase				,   mean=__lowerCAmelCase				,   std=__lowerCAmelCase				)

										UpperCAmelCase					    =     to_channel_dimension_format(__lowerCAmelCase				,   __lowerCAmelCase				)
										return image





				def 					a__(		self      :			Optional[int]				,   lowerCAmelCase      :			Any				,   lowerCAmelCase      :			str = None				,   lowerCAmelCase      :			Union[str, Any] = None				,   lowerCAmelCase      :			Optional[int] = None				,   lowerCAmelCase      :			Dict = None				,   lowerCAmelCase      :			Optional[int] = None				,   lowerCAmelCase      :			Dict = None				,   lowerCAmelCase      :			Dict = None				,   lowerCAmelCase      :			Any = None				,   lowerCAmelCase      :			str = None				,   lowerCAmelCase      :			List[Any] = None				,   lowerCAmelCase      :			List[Any] = None				,   lowerCAmelCase      :			Optional[int] = None				,   lowerCAmelCase      :			Tuple = ChannelDimension.FIRST				,   **lowerCAmelCase      :			List[Any]				,   )->					Dict:

										"""simple docstring"""




										UpperCAmelCase					    =     do_resize if do_resize is not None else self.do_resize
										UpperCAmelCase					    =     resample if resample is not None else self.resample
										UpperCAmelCase					    =     do_center_crop if do_center_crop is not None else self.do_center_crop
										UpperCAmelCase					    =     do_rescale if do_rescale is not None else self.do_rescale
										UpperCAmelCase					    =     rescale_factor if rescale_factor is not None else self.rescale_factor
										UpperCAmelCase					    =     offset if offset is not None else self.offset
										UpperCAmelCase					    =     do_normalize if do_normalize is not None else self.do_normalize
										UpperCAmelCase					    =     image_mean if image_mean is not None else self.image_mean
										UpperCAmelCase					    =     image_std if image_std is not None else self.image_std

										UpperCAmelCase					    =     size if size is not None else self.size
										UpperCAmelCase					    =     get_size_dict(__lowerCAmelCase				,   default_to_square=__lowerCAmelCase				)
										UpperCAmelCase					    =     crop_size if crop_size is not None else self.crop_size
										UpperCAmelCase					    =     get_size_dict(__lowerCAmelCase				,   param_name='''crop_size'''				)

										if not valid_images(__lowerCAmelCase				):
																raise ValueError(
																    '''Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '''
																    '''torch.Tensor, tf.Tensor or jax.ndarray.'''				)

										UpperCAmelCase					    =     make_batched(__lowerCAmelCase				)

										UpperCAmelCase					    =     [
										    [
										        self._preprocess_image(
										            image=__lowerCAmelCase				,   do_resize=__lowerCAmelCase				,   size=__lowerCAmelCase				,   resample=__lowerCAmelCase				,   do_center_crop=__lowerCAmelCase				,   crop_size=__lowerCAmelCase				,   do_rescale=__lowerCAmelCase				,   rescale_factor=__lowerCAmelCase				,   offset=__lowerCAmelCase				,   do_normalize=__lowerCAmelCase				,   image_mean=__lowerCAmelCase				,   image_std=__lowerCAmelCase				,   data_format=__lowerCAmelCase				,   )
										        for img in video
										    ]
										    for video in videos
										]

										UpperCAmelCase					    =     {'''pixel_values''': videos}
										return BatchFeature(data=__lowerCAmelCase				,   tensor_type=__lowerCAmelCase				)









'''simple docstring'''



import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_gpta import GPTaTokenizer


if TYPE_CHECKING:
	from transformers.pipelines.conversational import Conversation


_lowercase      :      str            =					logging.get_logger(__name__)

_lowercase      :      Any            =					{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

_lowercase      :      Tuple            =					{
    """vocab_file""": {
        """gpt2""": """https://huggingface.co/gpt2/resolve/main/vocab.json""",
        """gpt2-medium""": """https://huggingface.co/gpt2-medium/resolve/main/vocab.json""",
        """gpt2-large""": """https://huggingface.co/gpt2-large/resolve/main/vocab.json""",
        """gpt2-xl""": """https://huggingface.co/gpt2-xl/resolve/main/vocab.json""",
        """distilgpt2""": """https://huggingface.co/distilgpt2/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """gpt2""": """https://huggingface.co/gpt2/resolve/main/merges.txt""",
        """gpt2-medium""": """https://huggingface.co/gpt2-medium/resolve/main/merges.txt""",
        """gpt2-large""": """https://huggingface.co/gpt2-large/resolve/main/merges.txt""",
        """gpt2-xl""": """https://huggingface.co/gpt2-xl/resolve/main/merges.txt""",
        """distilgpt2""": """https://huggingface.co/distilgpt2/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """gpt2""": """https://huggingface.co/gpt2/resolve/main/tokenizer.json""",
        """gpt2-medium""": """https://huggingface.co/gpt2-medium/resolve/main/tokenizer.json""",
        """gpt2-large""": """https://huggingface.co/gpt2-large/resolve/main/tokenizer.json""",
        """gpt2-xl""": """https://huggingface.co/gpt2-xl/resolve/main/tokenizer.json""",
        """distilgpt2""": """https://huggingface.co/distilgpt2/resolve/main/tokenizer.json""",
    },
}

_lowercase      :      Union[str, Any]            =					{
    """gpt2""": 1024,
    """gpt2-medium""": 1024,
    """gpt2-large""": 1024,
    """gpt2-xl""": 1024,
    """distilgpt2""": 1024,
}






class 	UpperCamelCase__(       lowerCAmelCase	):
				__magic_name__    :				Dict  		=	VOCAB_FILES_NAMES
				__magic_name__    :				str  		=	PRETRAINED_VOCAB_FILES_MAP
				__magic_name__    :				Any  		=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__magic_name__    :				List[Any]  		=	["input_ids", "attention_mask"]
				__magic_name__    :				List[Any]  		=	GPTaTokenizer




				def __init__(		self      :			Tuple				,   lowerCAmelCase      :			List[Any]=None				,   lowerCAmelCase      :			Dict=None				,   lowerCAmelCase      :			Dict=None				,   lowerCAmelCase      :			Tuple="<|endoftext|>"				,   lowerCAmelCase      :			Union[str, Any]="<|endoftext|>"				,   lowerCAmelCase      :			Union[str, Any]="<|endoftext|>"				,   lowerCAmelCase      :			Optional[int]=False				,   **lowerCAmelCase      :			Tuple				,   )->					int:

										"""simple docstring"""




										super().__init__(
										    lowerCAmelCase				,   lowerCAmelCase				,   tokenizer_file=lowerCAmelCase				,   unk_token=lowerCAmelCase				,   bos_token=lowerCAmelCase				,   eos_token=lowerCAmelCase				,   add_prefix_space=lowerCAmelCase				,   **lowerCAmelCase				,   )

										UpperCAmelCase					    =     kwargs.pop('''add_bos_token'''				,   lowerCAmelCase				)

										UpperCAmelCase					    =     json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()				)
										if pre_tok_state.get('''add_prefix_space'''				,   lowerCAmelCase				) != add_prefix_space:
																UpperCAmelCase					    =     getattr(lowerCAmelCase				,   pre_tok_state.pop('''type'''				)				)
																UpperCAmelCase					    =     add_prefix_space
																UpperCAmelCase					    =     pre_tok_class(**lowerCAmelCase				)

										UpperCAmelCase					    =     add_prefix_space




				def 					a__(		self      :			Union[str, Any]				,   *lowerCAmelCase      :			Union[str, Any]				,   **lowerCAmelCase      :			Dict				)->					BatchEncoding:

										"""simple docstring"""




										UpperCAmelCase					    =     kwargs.get('''is_split_into_words'''				,   lowerCAmelCase				)
										assert self.add_prefix_space or not is_split_into_words, (
										    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
										    "to use it with pretokenized inputs."
										)

										return super()._batch_encode_plus(*lowerCAmelCase				,   **lowerCAmelCase				)




				def 					a__(		self      :			List[str]				,   *lowerCAmelCase      :			Any				,   **lowerCAmelCase      :			Tuple				)->					BatchEncoding:

										"""simple docstring"""




										UpperCAmelCase					    =     kwargs.get('''is_split_into_words'''				,   lowerCAmelCase				)

										assert self.add_prefix_space or not is_split_into_words, (
										    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
										    "to use it with pretokenized inputs."
										)

										return super()._encode_plus(*lowerCAmelCase				,   **lowerCAmelCase				)




				def 					a__(		self      :			str				,   lowerCAmelCase      :			str				,   lowerCAmelCase      :			Optional[str] = None				)->					Tuple[str]:

										"""simple docstring"""




										UpperCAmelCase					    =     self._tokenizer.model.save(lowerCAmelCase				,   name=lowerCAmelCase				)
										return tuple(lowerCAmelCase				)





				def 					a__(		self      :			List[Any]				,   lowerCAmelCase      :			"Conversation"				)->					List[int]:

										"""simple docstring"""




										UpperCAmelCase					    =     []
										for is_user, text in conversation.iter_texts():
																input_ids.extend(self.encode(lowerCAmelCase				,   add_special_tokens=lowerCAmelCase				) + [self.eos_token_id]				)

										if len(lowerCAmelCase				) > self.model_max_length:
																UpperCAmelCase					    =     input_ids[-self.model_max_length :]
										return input_ids





