





'''simple docstring'''


import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase :					List[str] 							=logging.get_logger(__name__)

lowerCAmelCase :					str 							='''▁'''

lowerCAmelCase :					int 							={'''vocab_file''': '''sentencepiece.bpe.model'''}

lowerCAmelCase :					List[str] 							={
    '''vocab_file''': {
        '''facebook/xglm-564M''': '''https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model''',
    }
}

lowerCAmelCase :					str 							={
    '''facebook/xglm-564M''': 2_048,
}
class 				a_       (    _lowerCAmelCase							):
     __A								=      VOCAB_FILES_NAMES
     __A								=      PRETRAINED_VOCAB_FILES_MAP
     __A								=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __A								=      ["input_ids", "attention_mask"]





     def __init__(			self		: Dict     ,				lowercase		: Union[str, Any]     ,				lowercase		: Optional[Any]="<s>"     ,				lowercase		: int="</s>"     ,				lowercase		: Union[str, Any]="</s>"     ,				lowercase		: Union[str, Any]="<s>"     ,				lowercase		: int="<unk>"     ,				lowercase		: Optional[int]="<pad>"     ,				lowercase		: Optional[Dict[str, Any]] = None     ,				**lowercase		: Optional[Any]     ,				):





        """simple docstring"""




        lowercase_      :int         =			{} if sp_model_kwargs is None else sp_model_kwargs

        # Compatibility with the original tokenizer
        lowercase_      :List[str]         =			7
        lowercase_      :Tuple         =			[F'<madeupword{i}>' for i in range(self.num_madeup_words						)]

        lowercase_      :Union[str, Any]         =			kwargs.get("additional_special_tokens"     ,				[]						)
        kwargs["additional_special_tokens"] += [
            word for word in madeup_words if word not in kwargs["additional_special_tokens"]
        ]

        super().__init__(
            bos_token=lowercase     ,				eos_token=lowercase     ,				unk_token=lowercase     ,				sep_token=lowercase     ,				cls_token=lowercase     ,				pad_token=lowercase     ,				sp_model_kwargs=self.sp_model_kwargs     ,				**lowercase     ,				)

        lowercase_      :List[str]         =			spm.SentencePieceProcessor(**self.sp_model_kwargs						)
        self.sp_model.Load(str(lowercase						)						)
        lowercase_      :Any         =			vocab_file

        # Original fairseq vocab and spm vocab must be "aligned":
        # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
        # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
        # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
        # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

        # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
        lowercase_      :List[str]         =			1

        # Mimic fairseq token-to-id alignment for the first 4 token
        lowercase_      :Optional[Any]         =			{"<s>": 0, "<pad>": 1, "</s>": 2, "<unk>": 3}

        lowercase_      :str         =			len(self.sp_model						)
        lowercase_      :List[str]         =			{F'<madeupword{i}>': sp_size + i + self.fairseq_offset for i in range(self.num_madeup_words						)}
        self.fairseq_tokens_to_ids.update(lowercase						)

        lowercase_      :Any         =			{v: k for k, v in self.fairseq_tokens_to_ids.items()}





     def __getstate__(			self		: Union[str, Any]						):





        """simple docstring"""




        lowercase_      :List[str]         =			self.__dict__.copy()
        lowercase_      :Tuple         =			None
        lowercase_      :Optional[Any]         =			self.sp_model.serialized_model_proto()
        return state





     def __setstate__(			self		: Optional[int]     ,				lowercase		: int						):





        """simple docstring"""




        lowercase_      :Optional[Any]         =			d

        # for backward compatibility
        if not hasattr(self     ,				"sp_model_kwargs"						):
           lowercase_      :Tuple         =			{}

        lowercase_      :Union[str, Any]         =			spm.SentencePieceProcessor(**self.sp_model_kwargs						)
        self.sp_model.LoadFromSerializedProto(self.sp_model_proto						)





     def 							lowercase__				(			self		: Optional[int]     ,				lowercase		: List[int]     ,				lowercase		: Optional[List[int]] = None						):





        """simple docstring"""




        if token_ids_a is None:
           return [self.sep_token_id] + token_ids_a
        lowercase_      :List[Any]         =			[self.sep_token_id]
        return sep + token_ids_a + sep + sep + token_ids_a





     def 							lowercase__				(			self		: Optional[Any]     ,				lowercase		: List[int]     ,				lowercase		: Optional[List[int]] = None     ,				lowercase		: bool = False						):





        """simple docstring"""




        if already_has_special_tokens:
           return super().get_special_tokens_mask(
               token_ids_a=lowercase     ,				token_ids_a=lowercase     ,				already_has_special_tokens=lowercase						)

        if token_ids_a is None:
           return [1] + ([0] * len(lowercase						))
        return [1] + ([0] * len(lowercase						)) + [1, 1] + ([0] * len(lowercase						))





     def 							lowercase__				(			self		: List[Any]     ,				lowercase		: List[int]     ,				lowercase		: Optional[List[int]] = None						):





        """simple docstring"""




        lowercase_      :str         =			[self.sep_token_id]

        if token_ids_a is None:
           return len(sep + token_ids_a						) * [0]
        return len(sep + token_ids_a + sep + sep + token_ids_a						) * [0]





     @property
     def 							lowercase__				(			self		: Optional[int]						):





        """simple docstring"""




        return len(self.sp_model						) + self.fairseq_offset + self.num_madeup_words





     def 							lowercase__				(			self		: List[str]						):





        """simple docstring"""




        lowercase_      :str         =			{self.convert_ids_to_tokens(lowercase						): i for i in range(self.vocab_size						)}
        vocab.update(self.added_tokens_encoder						)
        return vocab





     def 							lowercase__				(			self		: Any     ,				lowercase		: str						):





        """simple docstring"""




        return self.sp_model.encode(lowercase     ,				out_type=lowercase						)





     def 							lowercase__				(			self		: Dict     ,				lowercase		: Tuple						):





        """simple docstring"""




        if token in self.fairseq_tokens_to_ids:
           return self.fairseq_tokens_to_ids[token]
        lowercase_      :str         =			self.sp_model.PieceToId(lowercase						)

        # Need to return unknown token if the SP model returned 0
        return spm_id + self.fairseq_offset if spm_id else self.unk_token_id





     def 							lowercase__				(			self		: Any     ,				lowercase		: Optional[Any]						):





        """simple docstring"""




        if index in self.fairseq_ids_to_tokens:
           return self.fairseq_ids_to_tokens[index]
        return self.sp_model.IdToPiece(index - self.fairseq_offset						)





     def 							lowercase__				(			self		: str     ,				lowercase		: Optional[int]						):





        """simple docstring"""




        lowercase_      :int         =			"".join(lowercase						).replace(lowercase     ,				" "						).strip()
        return out_string







     def 							lowercase__				(			self		: Union[str, Any]     ,				lowercase		: str     ,				lowercase		: Optional[str] = None						):





        """simple docstring"""




        if not os.path.isdir(lowercase						):
           logger.error(F'Vocabulary path ({save_directory}) should be a directory'						)
           return
        lowercase_      :List[str]         =			os.path.join(
            lowercase     ,				(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]						)

        if os.path.abspath(self.vocab_file						) != os.path.abspath(lowercase						) and os.path.isfile(self.vocab_file						):
           copyfile(self.vocab_file     ,				lowercase						)
        elif not os.path.isfile(self.vocab_file						):
           with open(lowercase     ,				"wb"						) as fi:
              lowercase_      :List[Any]         =			self.sp_model.serialized_model_proto()
              fi.write(lowercase						)

        return (out_vocab_file,)









'''simple docstring'''


from typing import Dict, List, Optional, Union

import numpy as np

from transformers.utils import is_vision_available
from transformers.utils.generic import TensorType

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    is_valid_image,
    to_numpy_array,
    valid_images,
)
from ...utils import logging


if is_vision_available():
 import PIL

lowerCAmelCase :					Any 							=logging.get_logger(__name__)
def 							UpperCAmelCase_  (						__lowerCamelCase : List[str]			):
   if isinstance(__lowerCamelCase     ,(list, tuple)			) and isinstance(videos[0]     ,(list, tuple)			) and is_valid_image(videos[0][0]			):
      return videos

   elif isinstance(__lowerCamelCase     ,(list, tuple)			) and is_valid_image(videos[0]			):
      return [videos]

   elif is_valid_image(__lowerCamelCase			):
      return [[videos]]

   raise ValueError(F'Could not make batched video from {videos}'			)


class 				a_       (    _lowerCAmelCase							):
     __A								=      ["pixel_values"]





     def __init__(			self		: List[str]     ,				lowercase		: bool = True     ,				lowercase		: Dict[str, int] = None     ,				lowercase		: PILImageResampling = PILImageResampling.BILINEAR     ,				lowercase		: bool = True     ,				lowercase		: Dict[str, int] = None     ,				lowercase		: bool = True     ,				lowercase		: Union[int, float] = 1 / 255     ,				lowercase		: bool = True     ,				lowercase		: bool = True     ,				lowercase		: Optional[Union[float, List[float]]] = None     ,				lowercase		: Optional[Union[float, List[float]]] = None     ,				**lowercase		: Tuple     ,				):





        """simple docstring"""




        super().__init__(**lowercase						)
        lowercase_      :Any         =			size if size is not None else {"shortest_edge": 256}
        lowercase_      :int         =			get_size_dict(lowercase     ,				default_to_square=lowercase						)
        lowercase_      :str         =			crop_size if crop_size is not None else {"height": 224, "width": 224}
        lowercase_      :List[str]         =			get_size_dict(lowercase     ,				param_name="crop_size"						)

        lowercase_      :List[str]         =			do_resize
        lowercase_      :Any         =			size
        lowercase_      :Union[str, Any]         =			do_center_crop
        lowercase_      :Union[str, Any]         =			crop_size
        lowercase_      :Optional[Any]         =			resample
        lowercase_      :List[str]         =			do_rescale
        lowercase_      :List[Any]         =			rescale_factor
        lowercase_      :Dict         =			offset
        lowercase_      :Optional[Any]         =			do_normalize
        lowercase_      :Dict         =			image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
        lowercase_      :Optional[int]         =			image_std if image_std is not None else IMAGENET_STANDARD_STD





     def 							lowercase__				(			self		: Union[str, Any]     ,				lowercase		: np.ndarray     ,				lowercase		: Dict[str, int]     ,				lowercase		: PILImageResampling = PILImageResampling.BILINEAR     ,				lowercase		: Optional[Union[str, ChannelDimension]] = None     ,				**lowercase		: Optional[Any]     ,				):





        """simple docstring"""




        lowercase_      :List[Any]         =			get_size_dict(lowercase     ,				default_to_square=lowercase						)
        if "shortest_edge" in size:
           lowercase_      :int         =			get_resize_output_image_size(lowercase     ,				size["shortest_edge"]     ,				default_to_square=lowercase						)
        elif "height" in size and "width" in size:
           lowercase_      :Union[str, Any]         =			(size["height"], size["width"])
        else:
           raise ValueError(F'Size must have \'height\' and \'width\' or \'shortest_edge\' as keys. Got {size.keys()}'						)
        return resize(lowercase     ,				size=lowercase     ,				resample=lowercase     ,				data_format=lowercase     ,				**lowercase						)





     def 							lowercase__				(			self		: str     ,				lowercase		: np.ndarray     ,				lowercase		: Dict[str, int]     ,				lowercase		: Optional[Union[str, ChannelDimension]] = None     ,				**lowercase		: str     ,				):





        """simple docstring"""




        lowercase_      :Any         =			get_size_dict(lowercase						)
        if "height" not in size or "width" not in size:
           raise ValueError(F'Size must have \'height\' and \'width\' as keys. Got {size.keys()}'						)
        return center_crop(lowercase     ,				size=(size["height"], size["width"])     ,				data_format=lowercase     ,				**lowercase						)





     def 							lowercase__				(			self		: List[str]     ,				lowercase		: np.ndarray     ,				lowercase		: Union[int, float]     ,				lowercase		: bool = True     ,				lowercase		: Optional[Union[str, ChannelDimension]] = None     ,				**lowercase		: List[str]     ,				):





        """simple docstring"""




        lowercase_      :List[str]         =			image.astype(np.floataa						)
        if offset:
           lowercase_      :List[str]         =			image - (scale / 2)
        return rescale(lowercase     ,				scale=lowercase     ,				data_format=lowercase     ,				**lowercase						)





     def 							lowercase__				(			self		: Tuple     ,				lowercase		: np.ndarray     ,				lowercase		: Union[float, List[float]]     ,				lowercase		: Union[float, List[float]]     ,				lowercase		: Optional[Union[str, ChannelDimension]] = None     ,				**lowercase		: Dict     ,				):





        """simple docstring"""




        return normalize(lowercase     ,				mean=lowercase     ,				std=lowercase     ,				data_format=lowercase     ,				**lowercase						)





     def 							lowercase__				(			self		: Tuple     ,				lowercase		: ImageInput     ,				lowercase		: bool = None     ,				lowercase		: Dict[str, int] = None     ,				lowercase		: PILImageResampling = None     ,				lowercase		: bool = None     ,				lowercase		: Dict[str, int] = None     ,				lowercase		: bool = None     ,				lowercase		: float = None     ,				lowercase		: bool = None     ,				lowercase		: bool = None     ,				lowercase		: Optional[Union[float, List[float]]] = None     ,				lowercase		: Optional[Union[float, List[float]]] = None     ,				lowercase		: Optional[ChannelDimension] = ChannelDimension.FIRST     ,				):





        """simple docstring"""




        if do_resize and size is None or resample is None:
           raise ValueError("Size and resample must be specified if do_resize is True."						)

        if do_center_crop and crop_size is None:
           raise ValueError("Crop size must be specified if do_center_crop is True."						)

        if do_rescale and rescale_factor is None:
           raise ValueError("Rescale factor must be specified if do_rescale is True."						)

        if do_normalize and (image_mean is None or image_std is None):
           raise ValueError("Image mean and std must be specified if do_normalize is True."						)

        if offset and not do_rescale:
           raise ValueError("For offset, do_rescale must also be set to True."						)

        # All transformations expect numpy arrays.
        lowercase_      :Optional[int]         =			to_numpy_array(lowercase						)

        if do_resize:
           lowercase_      :Tuple         =			self.resize(image=lowercase     ,				size=lowercase     ,				resample=lowercase						)

        if do_center_crop:
           lowercase_      :Any         =			self.center_crop(lowercase     ,				size=lowercase						)

        if do_rescale:
           lowercase_      :Optional[Any]         =			self.rescale(image=lowercase     ,				scale=lowercase     ,				offset=lowercase						)

        if do_normalize:
           lowercase_      :Tuple         =			self.normalize(image=lowercase     ,				mean=lowercase     ,				std=lowercase						)

        lowercase_      :Optional[Any]         =			to_channel_dimension_format(lowercase     ,				lowercase						)
        return image







     def 							lowercase__				(			self		: Dict     ,				lowercase		: ImageInput     ,				lowercase		: bool = None     ,				lowercase		: Dict[str, int] = None     ,				lowercase		: PILImageResampling = None     ,				lowercase		: bool = None     ,				lowercase		: Dict[str, int] = None     ,				lowercase		: bool = None     ,				lowercase		: float = None     ,				lowercase		: bool = None     ,				lowercase		: bool = None     ,				lowercase		: Optional[Union[float, List[float]]] = None     ,				lowercase		: Optional[Union[float, List[float]]] = None     ,				lowercase		: Optional[Union[str, TensorType]] = None     ,				lowercase		: ChannelDimension = ChannelDimension.FIRST     ,				**lowercase		: Optional[int]     ,				):





        """simple docstring"""




        lowercase_      :str         =			do_resize if do_resize is not None else self.do_resize
        lowercase_      :Optional[Any]         =			resample if resample is not None else self.resample
        lowercase_      :Tuple         =			do_center_crop if do_center_crop is not None else self.do_center_crop
        lowercase_      :Dict         =			do_rescale if do_rescale is not None else self.do_rescale
        lowercase_      :Optional[int]         =			rescale_factor if rescale_factor is not None else self.rescale_factor
        lowercase_      :Dict         =			offset if offset is not None else self.offset
        lowercase_      :Tuple         =			do_normalize if do_normalize is not None else self.do_normalize
        lowercase_      :int         =			image_mean if image_mean is not None else self.image_mean
        lowercase_      :Optional[int]         =			image_std if image_std is not None else self.image_std

        lowercase_      :int         =			size if size is not None else self.size
        lowercase_      :Optional[int]         =			get_size_dict(lowercase     ,				default_to_square=lowercase						)
        lowercase_      :List[Any]         =			crop_size if crop_size is not None else self.crop_size
        lowercase_      :List[str]         =			get_size_dict(lowercase     ,				param_name="crop_size"						)

        if not valid_images(lowercase						):
           raise ValueError(
               "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
               "torch.Tensor, tf.Tensor or jax.ndarray."						)

        lowercase_      :List[str]         =			make_batched(lowercase						)

        lowercase_      :List[Any]         =			[
            [
                self._preprocess_image(
                    image=lowercase     ,				do_resize=lowercase     ,				size=lowercase     ,				resample=lowercase     ,				do_center_crop=lowercase     ,				crop_size=lowercase     ,				do_rescale=lowercase     ,				rescale_factor=lowercase     ,				offset=lowercase     ,				do_normalize=lowercase     ,				image_mean=lowercase     ,				image_std=lowercase     ,				data_format=lowercase     ,				)
                for img in video
            ]
            for video in videos
        ]

        lowercase_      :Optional[int]         =			{"pixel_values": videos}
        return BatchFeature(data=lowercase     ,				tensor_type=lowercase						)


