


'''simple docstring'''


from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import symbol_database as _symbol_database
from google.protobuf.internal import builder as _builder


# @@protoc_insertion_point(imports)

lowerCAmelCase :					List[str] 							=_symbol_database.Default()


lowerCAmelCase :					Dict 							=_descriptor_pool.Default().AddSerializedFile(
    B'''\n\x19sentencepiece_model.proto\x12\rsentencepiece"\x80\x0c\n\x0bTrainerSpec\x12\r\n\x05input\x18\x01 \x03(\t\x12\x14\n\x0cinput_format\x18\x07 \x01(\t\x12\x14\n\x0cmodel_prefix\x18\x02 \x01(\t\x12\x41\n\nmodel_type\x18\x03 \x01(\x0e\x32$.sentencepiece.TrainerSpec.ModelType:\x07UNIGRAM\x12\x18\n\nvocab_size\x18\x04 \x01(\x05:\x04\x38\x30\x30\x30\x12\x17\n\x0f\x61\x63\x63\x65pt_language\x18\x05 \x03(\t\x12 \n\x15self_test_sample_size\x18\x06 \x01(\x05:\x01\x30\x12*\n\x1b\x65nable_differential_privacy\x18\x32 \x01(\x08:\x05\x66\x61lse\x12+\n differential_privacy_noise_level\x18\x33 \x01(\x02:\x01\x30\x12\x32\n\'differential_privacy_clipping_threshold\x18\x34 \x01(\x04:\x01\x30\x12"\n\x12\x63haracter_coverage\x18\n \x01(\x02:\x06\x30.9995\x12\x1e\n\x13input_sentence_size\x18\x0b \x01(\x04:\x01\x30\x12$\n\x16shuffle_input_sentence\x18\x13 \x01(\x08:\x04true\x12 \n\x14mining_sentence_size\x18\x0c \x01(\x05\x42\x02\x18\x01\x12"\n\x16training_sentence_size\x18\r \x01(\x05\x42\x02\x18\x01\x12(\n\x17seed_sentencepiece_size\x18\x0e \x01(\x05:\x07\x31\x30\x30\x30\x30\x30\x30\x12\x1e\n\x10shrinking_factor\x18\x0f \x01(\x02:\x04\x30.75\x12!\n\x13max_sentence_length\x18\x12 \x01(\x05:\x04\x34\x31\x39\x32\x12\x17\n\x0bnum_threads\x18\x10 \x01(\x05:\x02\x31\x36\x12\x1d\n\x12num_sub_iterations\x18\x11 \x01(\x05:\x01\x32\x12$\n\x18max_sentencepiece_length\x18\x14 \x01(\x05:\x02\x31\x36\x12%\n\x17split_by_unicode_script\x18\x15 \x01(\x08:\x04true\x12\x1d\n\x0fsplit_by_number\x18\x17 \x01(\x08:\x04true\x12!\n\x13split_by_whitespace\x18\x16 \x01(\x08:\x04true\x12)\n\x1atreat_whitespace_as_suffix\x18\x18 \x01(\x08:\x05\x66\x61lse\x12+\n\x1c\x61llow_whitespace_only_pieces\x18\x1a \x01(\x08:\x05\x66\x61lse\x12\x1b\n\x0csplit_digits\x18\x19 \x01(\x08:\x05\x66\x61lse\x12#\n\x19pretokenization_delimiter\x18\x35 \x01(\t:\x00\x12\x17\n\x0f\x63ontrol_symbols\x18\x1e \x03(\t\x12\x1c\n\x14user_defined_symbols\x18\x1f \x03(\t\x12\x16\n\x0erequired_chars\x18$ \x01(\t\x12\x1c\n\rbyte_fallback\x18# \x01(\x08:\x05\x66\x61lse\x12+\n\x1dvocabulary_output_piece_score\x18  \x01(\x08:\x04true\x12\x1e\n\x10hard_vocab_limit\x18! \x01(\x08:\x04true\x12\x1c\n\ruse_all_vocab\x18" \x01(\x08:\x05\x66\x61lse\x12\x11\n\x06unk_id\x18( \x01(\x05:\x01\x30\x12\x11\n\x06\x62os_id\x18) \x01(\x05:\x01\x31\x12\x11\n\x06\x65os_id\x18* \x01(\x05:\x01\x32\x12\x12\n\x06pad_id\x18+ \x01(\x05:\x02-1\x12\x18\n\tunk_piece\x18- \x01(\t:\x05<unk>\x12\x16\n\tbos_piece\x18. \x01(\t:\x03<s>\x12\x17\n\teos_piece\x18/ \x01(\t:\x04</s>\x12\x18\n\tpad_piece\x18\x30 \x01(\t:\x05<pad>\x12\x1a\n\x0bunk_surface\x18, \x01(\t:\x05 \xe2\x81\x87 \x12+\n\x1ctrain_extremely_large_corpus\x18\x31 \x01(\x08:\x05\x66\x61lse"5\n\tModelType\x12\x0b\n\x07UNIGRAM\x10\x01\x12\x07\n\x03\x42PE\x10\x02\x12\x08\n\x04WORD\x10\x03\x12\x08\n\x04\x43HAR\x10\x04*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02"\xd1\x01\n\x0eNormalizerSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x1c\n\x14precompiled_charsmap\x18\x02 \x01(\x0c\x12\x1e\n\x10\x61\x64\x64_dummy_prefix\x18\x03 \x01(\x08:\x04true\x12&\n\x18remove_extra_whitespaces\x18\x04 \x01(\x08:\x04true\x12 \n\x12\x65scape_whitespaces\x18\x05 \x01(\x08:\x04true\x12\x1e\n\x16normalization_rule_tsv\x18\x06 \x01(\t*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02"y\n\x0cSelfTestData\x12\x33\n\x07samples\x18\x01 \x03(\x0b\x32".sentencepiece.SelfTestData.Sample\x1a)\n\x06Sample\x12\r\n\x05input\x18\x01 \x01(\t\x12\x10\n\x08\x65xpected\x18\x02 \x01(\t*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02"\xfe\x03\n\nModelProto\x12\x37\n\x06pieces\x18\x01 \x03(\x0b\x32\'.sentencepiece.ModelProto.SentencePiece\x12\x30\n\x0ctrainer_spec\x18\x02 \x01(\x0b\x32\x1a.sentencepiece.TrainerSpec\x12\x36\n\x0fnormalizer_spec\x18\x03 \x01(\x0b\x32\x1d.sentencepiece.NormalizerSpec\x12\x33\n\x0eself_test_data\x18\x04 \x01(\x0b\x32\x1b.sentencepiece.SelfTestData\x12\x38\n\x11\x64\x65normalizer_spec\x18\x05 \x01(\x0b\x32\x1d.sentencepiece.NormalizerSpec\x1a\xd2\x01\n\rSentencePiece\x12\r\n\x05piece\x18\x01 \x01(\t\x12\r\n\x05score\x18\x02 \x01(\x02\x12\x42\n\x04type\x18\x03 \x01(\x0e\x32,.sentencepiece.ModelProto.SentencePiece.Type:\x06NORMAL"T\n\x04Type\x12\n\n\x06NORMAL\x10\x01\x12\x0b\n\x07UNKNOWN\x10\x02\x12\x0b\n\x07\x43ONTROL\x10\x03\x12\x10\n\x0cUSER_DEFINED\x10\x04\x12\x08\n\x04\x42YTE\x10\x06\x12\n\n\x06UNUSED\x10\x05*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02\x42\x02H\x03'''
)

lowerCAmelCase :					Any 							=globals()
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, '''sentencepiece_model_pb2''', _globals)
if _descriptor._USE_C_DESCRIPTORS is False:
 lowerCAmelCase :					Dict 							=None
 lowerCAmelCase :					Any 							=B'''H\003'''
 # (generated by protobuf compiler, but `_TRAINERSPEC` is not defined)
 # _TRAINERSPEC.fields_by_name["mining_sentence_size"]._options = None
 # _TRAINERSPEC.fields_by_name["mining_sentence_size"]._serialized_options = b"\030\001"
 # _TRAINERSPEC.fields_by_name["training_sentence_size"]._options = None
 # _TRAINERSPEC.fields_by_name["training_sentence_size"]._serialized_options = b"\030\001"
 lowerCAmelCase :					Union[str, Any] 							=45
 lowerCAmelCase :					Tuple 							=1_581
 lowerCAmelCase :					List[Any] 							=1_517
 lowerCAmelCase :					Union[str, Any] 							=1_570
 lowerCAmelCase :					Dict 							=1_584
 lowerCAmelCase :					Any 							=1_793
 lowerCAmelCase :					Optional[Any] 							=1_795
 lowerCAmelCase :					Tuple 							=1_916
 lowerCAmelCase :					int 							=1_864
 lowerCAmelCase :					Union[str, Any] 							=1_905
 lowerCAmelCase :					Dict 							=1_919
 lowerCAmelCase :					Tuple 							=2_429
 lowerCAmelCase :					str 							=2_208
 lowerCAmelCase :					Optional[Any] 							=2_418
 lowerCAmelCase :					Optional[Any] 							=2_323
 lowerCAmelCase :					List[Any] 							=2_407
# @@protoc_insertion_point(module_scope)









'''simple docstring'''


from typing import Dict, List, Optional, Union

import numpy as np

from transformers.utils import is_vision_available
from transformers.utils.generic import TensorType

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    is_valid_image,
    to_numpy_array,
    valid_images,
)
from ...utils import logging


if is_vision_available():
 import PIL

lowerCAmelCase :					Any 							=logging.get_logger(__name__)
def 							UpperCAmelCase_  (						__lowerCamelCase : List[str]			):
   if isinstance(__lowerCamelCase     ,(list, tuple)			) and isinstance(videos[0]     ,(list, tuple)			) and is_valid_image(videos[0][0]			):
      return videos

   elif isinstance(__lowerCamelCase     ,(list, tuple)			) and is_valid_image(videos[0]			):
      return [videos]

   elif is_valid_image(__lowerCamelCase			):
      return [[videos]]

   raise ValueError(F'Could not make batched video from {videos}'			)


class 				a_       (    _lowerCAmelCase							):
     __A								=      ["pixel_values"]





     def __init__(			self		: List[str]     ,				lowercase		: bool = True     ,				lowercase		: Dict[str, int] = None     ,				lowercase		: PILImageResampling = PILImageResampling.BILINEAR     ,				lowercase		: bool = True     ,				lowercase		: Dict[str, int] = None     ,				lowercase		: bool = True     ,				lowercase		: Union[int, float] = 1 / 255     ,				lowercase		: bool = True     ,				lowercase		: bool = True     ,				lowercase		: Optional[Union[float, List[float]]] = None     ,				lowercase		: Optional[Union[float, List[float]]] = None     ,				**lowercase		: Tuple     ,				):





        """simple docstring"""




        super().__init__(**lowercase						)
        lowercase_      :Any         =			size if size is not None else {"shortest_edge": 256}
        lowercase_      :int         =			get_size_dict(lowercase     ,				default_to_square=lowercase						)
        lowercase_      :str         =			crop_size if crop_size is not None else {"height": 224, "width": 224}
        lowercase_      :List[str]         =			get_size_dict(lowercase     ,				param_name="crop_size"						)

        lowercase_      :List[str]         =			do_resize
        lowercase_      :Any         =			size
        lowercase_      :Union[str, Any]         =			do_center_crop
        lowercase_      :Union[str, Any]         =			crop_size
        lowercase_      :Optional[Any]         =			resample
        lowercase_      :List[str]         =			do_rescale
        lowercase_      :List[Any]         =			rescale_factor
        lowercase_      :Dict         =			offset
        lowercase_      :Optional[Any]         =			do_normalize
        lowercase_      :Dict         =			image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
        lowercase_      :Optional[int]         =			image_std if image_std is not None else IMAGENET_STANDARD_STD





     def 							lowercase__				(			self		: Union[str, Any]     ,				lowercase		: np.ndarray     ,				lowercase		: Dict[str, int]     ,				lowercase		: PILImageResampling = PILImageResampling.BILINEAR     ,				lowercase		: Optional[Union[str, ChannelDimension]] = None     ,				**lowercase		: Optional[Any]     ,				):





        """simple docstring"""




        lowercase_      :List[Any]         =			get_size_dict(lowercase     ,				default_to_square=lowercase						)
        if "shortest_edge" in size:
           lowercase_      :int         =			get_resize_output_image_size(lowercase     ,				size["shortest_edge"]     ,				default_to_square=lowercase						)
        elif "height" in size and "width" in size:
           lowercase_      :Union[str, Any]         =			(size["height"], size["width"])
        else:
           raise ValueError(F'Size must have \'height\' and \'width\' or \'shortest_edge\' as keys. Got {size.keys()}'						)
        return resize(lowercase     ,				size=lowercase     ,				resample=lowercase     ,				data_format=lowercase     ,				**lowercase						)





     def 							lowercase__				(			self		: str     ,				lowercase		: np.ndarray     ,				lowercase		: Dict[str, int]     ,				lowercase		: Optional[Union[str, ChannelDimension]] = None     ,				**lowercase		: str     ,				):





        """simple docstring"""




        lowercase_      :Any         =			get_size_dict(lowercase						)
        if "height" not in size or "width" not in size:
           raise ValueError(F'Size must have \'height\' and \'width\' as keys. Got {size.keys()}'						)
        return center_crop(lowercase     ,				size=(size["height"], size["width"])     ,				data_format=lowercase     ,				**lowercase						)





     def 							lowercase__				(			self		: List[str]     ,				lowercase		: np.ndarray     ,				lowercase		: Union[int, float]     ,				lowercase		: bool = True     ,				lowercase		: Optional[Union[str, ChannelDimension]] = None     ,				**lowercase		: List[str]     ,				):





        """simple docstring"""




        lowercase_      :List[str]         =			image.astype(np.floataa						)
        if offset:
           lowercase_      :List[str]         =			image - (scale / 2)
        return rescale(lowercase     ,				scale=lowercase     ,				data_format=lowercase     ,				**lowercase						)





     def 							lowercase__				(			self		: Tuple     ,				lowercase		: np.ndarray     ,				lowercase		: Union[float, List[float]]     ,				lowercase		: Union[float, List[float]]     ,				lowercase		: Optional[Union[str, ChannelDimension]] = None     ,				**lowercase		: Dict     ,				):





        """simple docstring"""




        return normalize(lowercase     ,				mean=lowercase     ,				std=lowercase     ,				data_format=lowercase     ,				**lowercase						)





     def 							lowercase__				(			self		: Tuple     ,				lowercase		: ImageInput     ,				lowercase		: bool = None     ,				lowercase		: Dict[str, int] = None     ,				lowercase		: PILImageResampling = None     ,				lowercase		: bool = None     ,				lowercase		: Dict[str, int] = None     ,				lowercase		: bool = None     ,				lowercase		: float = None     ,				lowercase		: bool = None     ,				lowercase		: bool = None     ,				lowercase		: Optional[Union[float, List[float]]] = None     ,				lowercase		: Optional[Union[float, List[float]]] = None     ,				lowercase		: Optional[ChannelDimension] = ChannelDimension.FIRST     ,				):





        """simple docstring"""




        if do_resize and size is None or resample is None:
           raise ValueError("Size and resample must be specified if do_resize is True."						)

        if do_center_crop and crop_size is None:
           raise ValueError("Crop size must be specified if do_center_crop is True."						)

        if do_rescale and rescale_factor is None:
           raise ValueError("Rescale factor must be specified if do_rescale is True."						)

        if do_normalize and (image_mean is None or image_std is None):
           raise ValueError("Image mean and std must be specified if do_normalize is True."						)

        if offset and not do_rescale:
           raise ValueError("For offset, do_rescale must also be set to True."						)

        # All transformations expect numpy arrays.
        lowercase_      :Optional[int]         =			to_numpy_array(lowercase						)

        if do_resize:
           lowercase_      :Tuple         =			self.resize(image=lowercase     ,				size=lowercase     ,				resample=lowercase						)

        if do_center_crop:
           lowercase_      :Any         =			self.center_crop(lowercase     ,				size=lowercase						)

        if do_rescale:
           lowercase_      :Optional[Any]         =			self.rescale(image=lowercase     ,				scale=lowercase     ,				offset=lowercase						)

        if do_normalize:
           lowercase_      :Tuple         =			self.normalize(image=lowercase     ,				mean=lowercase     ,				std=lowercase						)

        lowercase_      :Optional[Any]         =			to_channel_dimension_format(lowercase     ,				lowercase						)
        return image







     def 							lowercase__				(			self		: Dict     ,				lowercase		: ImageInput     ,				lowercase		: bool = None     ,				lowercase		: Dict[str, int] = None     ,				lowercase		: PILImageResampling = None     ,				lowercase		: bool = None     ,				lowercase		: Dict[str, int] = None     ,				lowercase		: bool = None     ,				lowercase		: float = None     ,				lowercase		: bool = None     ,				lowercase		: bool = None     ,				lowercase		: Optional[Union[float, List[float]]] = None     ,				lowercase		: Optional[Union[float, List[float]]] = None     ,				lowercase		: Optional[Union[str, TensorType]] = None     ,				lowercase		: ChannelDimension = ChannelDimension.FIRST     ,				**lowercase		: Optional[int]     ,				):





        """simple docstring"""




        lowercase_      :str         =			do_resize if do_resize is not None else self.do_resize
        lowercase_      :Optional[Any]         =			resample if resample is not None else self.resample
        lowercase_      :Tuple         =			do_center_crop if do_center_crop is not None else self.do_center_crop
        lowercase_      :Dict         =			do_rescale if do_rescale is not None else self.do_rescale
        lowercase_      :Optional[int]         =			rescale_factor if rescale_factor is not None else self.rescale_factor
        lowercase_      :Dict         =			offset if offset is not None else self.offset
        lowercase_      :Tuple         =			do_normalize if do_normalize is not None else self.do_normalize
        lowercase_      :int         =			image_mean if image_mean is not None else self.image_mean
        lowercase_      :Optional[int]         =			image_std if image_std is not None else self.image_std

        lowercase_      :int         =			size if size is not None else self.size
        lowercase_      :Optional[int]         =			get_size_dict(lowercase     ,				default_to_square=lowercase						)
        lowercase_      :List[Any]         =			crop_size if crop_size is not None else self.crop_size
        lowercase_      :List[str]         =			get_size_dict(lowercase     ,				param_name="crop_size"						)

        if not valid_images(lowercase						):
           raise ValueError(
               "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
               "torch.Tensor, tf.Tensor or jax.ndarray."						)

        lowercase_      :List[str]         =			make_batched(lowercase						)

        lowercase_      :List[Any]         =			[
            [
                self._preprocess_image(
                    image=lowercase     ,				do_resize=lowercase     ,				size=lowercase     ,				resample=lowercase     ,				do_center_crop=lowercase     ,				crop_size=lowercase     ,				do_rescale=lowercase     ,				rescale_factor=lowercase     ,				offset=lowercase     ,				do_normalize=lowercase     ,				image_mean=lowercase     ,				image_std=lowercase     ,				data_format=lowercase     ,				)
                for img in video
            ]
            for video in videos
        ]

        lowercase_      :Optional[int]         =			{"pixel_values": videos}
        return BatchFeature(data=lowercase     ,				tensor_type=lowercase						)


