"""simple docstring"""







from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import convert_to_rgb, normalize, rescale, resize, to_channel_dimension_format
from ...image_utils import (
    OPENAI_CLIP_MEAN,
    OPENAI_CLIP_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


if is_vision_available():
	import PIL


_a					:					str=			logging.get_logger(__name__)

class 		UpperCamelCase      (   lowerCamelCase__						):
				UpperCAmelCase    :				Union[str, Any]   =		['pixel_values']



				def __init__(self    :				Any     ,  _A    :				Optional[Any] = True     ,  _A    :				Optional[Any] = None     ,  _A    :				List[Any] = PILImageResampling.BICUBIC     ,  _A    :				List[str] = True     ,  _A    :				Optional[Any] = 1 / 2_55     ,  _A    :				str = True     ,  _A    :				str = None     ,  _A    :				List[Any] = None     ,  _A    :				Optional[Any] = True     ,  **_A    :				Dict     ,  )   ->     List[str]:
					super().__init__(**_A)
					__snake_case      :							Dict						=			size if size is not None else {'height': 3_84, 'width': 3_84}
					__snake_case      :							Union[str, Any]						=			get_size_dict(_A     ,  default_to_square=_A)

					__snake_case      :							Tuple						=			do_resize
					__snake_case      :							Dict						=			size
					__snake_case      :							Union[str, Any]						=			resample
					__snake_case      :							str						=			do_rescale
					__snake_case      :							Union[str, Any]						=			rescale_factor
					__snake_case      :							Optional[int]						=			do_normalize
					__snake_case      :							Dict						=			image_mean if image_mean is not None else OPENAI_CLIP_MEAN
					__snake_case      :							str						=			image_std if image_std is not None else OPENAI_CLIP_STD
					__snake_case      :							Optional[int]						=			do_convert_rgb



				def  _lowercase					(self    :				int     ,  _A    :				Optional[int]     ,  _A    :				Tuple     ,  _A    :				List[Any] = PILImageResampling.BICUBIC     ,  _A    :				int = None     ,  **_A    :				Tuple     ,  )   ->     List[Any]:
					__snake_case      :							str						=			get_size_dict(_A     ,  default_to_square=_A)
					if "height" not in size or "width" not in size:
						raise ValueError(f"The `size` dictionary must contain the keys `height` and `width`. Got {size.keys()}")
					__snake_case      :							List[str]						=			(size['height'], size['width'])
					return resize(_A     ,  size=_A     ,  resample=_A     ,  data_format=_A     ,  **_A)



				def  _lowercase					(self    :				int     ,  _A    :				Union[str, Any]     ,  _A    :				Union[str, Any]     ,  _A    :				Optional[Any] = None     ,  **_A    :				Optional[Any]     ,  )   ->     Union[str, Any]:
					return rescale(_A     ,  scale=_A     ,  data_format=_A     ,  **_A)



				def  _lowercase					(self    :				Union[str, Any]     ,  _A    :				int     ,  _A    :				Optional[Any]     ,  _A    :				Any     ,  _A    :				Union[str, Any] = None     ,  **_A    :				Any     ,  )   ->     Optional[int]:
					return normalize(_A     ,  mean=_A     ,  std=_A     ,  data_format=_A     ,  **_A)






				def  _lowercase					(self    :				Dict     ,  _A    :				List[str]     ,  _A    :				Tuple = None     ,  _A    :				str = None     ,  _A    :				Any = None     ,  _A    :				Optional[Any] = None     ,  _A    :				Any = None     ,  _A    :				Union[str, Any] = None     ,  _A    :				List[str] = None     ,  _A    :				Optional[Any] = None     ,  _A    :				int = None     ,  _A    :				str = None     ,  _A    :				Dict = ChannelDimension.FIRST     ,  **_A    :				List[str]     ,  )   ->     Any:
					__snake_case      :							Union[str, Any]						=			do_resize if do_resize is not None else self.do_resize
					__snake_case      :							Optional[int]						=			resample if resample is not None else self.resample
					__snake_case      :							Any						=			do_rescale if do_rescale is not None else self.do_rescale
					__snake_case      :							int						=			rescale_factor if rescale_factor is not None else self.rescale_factor
					__snake_case      :							int						=			do_normalize if do_normalize is not None else self.do_normalize
					__snake_case      :							Tuple						=			image_mean if image_mean is not None else self.image_mean
					__snake_case      :							Optional[Any]						=			image_std if image_std is not None else self.image_std
					__snake_case      :							Tuple						=			do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb

					__snake_case      :							Union[str, Any]						=			size if size is not None else self.size
					__snake_case      :							Union[str, Any]						=			get_size_dict(_A     ,  default_to_square=_A)

					__snake_case      :							Optional[Any]						=			make_list_of_images(_A)

					if not valid_images(_A):
						raise ValueError(
						    'Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '
						    'torch.Tensor, tf.Tensor or jax.ndarray.')

					if do_resize and size is None or resample is None:
						raise ValueError('Size and resample must be specified if do_resize is True.')

					if do_rescale and rescale_factor is None:
						raise ValueError('Rescale factor must be specified if do_rescale is True.')

					if do_normalize and (image_mean is None or image_std is None):
						raise ValueError('Image mean and std must be specified if do_normalize is True.')

					# PIL RGBA images are converted to RGB
					if do_convert_rgb:
						__snake_case      :							Union[str, Any]						=			[convert_to_rgb(_A) for image in images]

					# All transformations expect numpy arrays.
					__snake_case      :							Any						=			[to_numpy_array(_A) for image in images]

					if do_resize:
						__snake_case      :							Optional[Any]						=			[self.resize(image=_A     ,  size=_A     ,  resample=_A) for image in images]

					if do_rescale:
						__snake_case      :							List[Any]						=			[self.rescale(image=_A     ,  scale=_A) for image in images]

					if do_normalize:
						__snake_case      :							List[str]						=			[self.normalize(image=_A     ,  mean=_A     ,  std=_A) for image in images]

					__snake_case      :							Dict						=			[to_channel_dimension_format(_A     ,  _A) for image in images]

					__snake_case      :							Any						=			BatchFeature(data={'pixel_values': images}     ,  tensor_type=_A)

					return encoded_outputs





"""simple docstring"""







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


_a					:					Any=			logging.get_logger(__name__)

_a					:					str=			{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

_a					:					Optional[Any]=			[
    "small",
    "small-base",
    "medium",
    "medium-base",
    "intermediate",
    "intermediate-base",
    "large",
    "large-base",
    "xlarge",
    "xlarge-base",
]

_a					:					List[Any]=			{
    "vocab_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt",
        "funnel-transformer/small-base": "https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt",
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt",
        "funnel-transformer/large-base": "https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt",
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json",
        "funnel-transformer/small-base": (
            "https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json",
        "funnel-transformer/large-base": (
            "https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json"
        ),
    },
}
_a					:					str=			{f'''funnel-transformer/{name}''': 512 for name in _model_names}
_a					:					List[Any]=			{f'''funnel-transformer/{name}''': {"do_lower_case": True} for name in _model_names}

class 		UpperCamelCase      (   lowercase						):
				UpperCAmelCase    :				List[str]   =		VOCAB_FILES_NAMES
				UpperCAmelCase    :				Tuple   =		PRETRAINED_VOCAB_FILES_MAP
				UpperCAmelCase    :				int   =		PRETRAINED_INIT_CONFIGURATION
				UpperCAmelCase    :				Tuple   =		FunnelTokenizer
				UpperCAmelCase    :				Tuple   =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				UpperCAmelCase    :				int   =		2



				def __init__(self    :				int     ,  _A    :				Any=None     ,  _A    :				Union[str, Any]=None     ,  _A    :				Union[str, Any]=True     ,  _A    :				List[str]="<unk>"     ,  _A    :				Any="<sep>"     ,  _A    :				Dict="<pad>"     ,  _A    :				Tuple="<cls>"     ,  _A    :				Dict="<mask>"     ,  _A    :				Optional[Any]="<s>"     ,  _A    :				List[Any]="</s>"     ,  _A    :				Optional[int]=True     ,  _A    :				Dict=True     ,  _A    :				Tuple=None     ,  _A    :				int="##"     ,  **_A    :				Any     ,  )   ->     str:
					super().__init__(
					    _A     ,  tokenizer_file=_A     ,  do_lower_case=_A     ,  unk_token=_A     ,  sep_token=_A     ,  pad_token=_A     ,  cls_token=_A     ,  mask_token=_A     ,  bos_token=_A     ,  eos_token=_A     ,  clean_text=_A     ,  tokenize_chinese_chars=_A     ,  strip_accents=_A     ,  wordpieces_prefix=_A     ,  **_A     ,  )

					__snake_case      :							Optional[int]						=			json.loads(self.backend_tokenizer.normalizer.__getstate__())
					if (
					    normalizer_state.get('lowercase'     ,  _A) != do_lower_case
					    or normalizer_state.get('strip_accents'     ,  _A) != strip_accents
					    or normalizer_state.get('handle_chinese_chars'     ,  _A) != tokenize_chinese_chars
					):
						__snake_case      :							List[str]						=			getattr(_A     ,  normalizer_state.pop('type'))
						__snake_case      :							int						=			do_lower_case
						__snake_case      :							Optional[int]						=			strip_accents
						__snake_case      :							str						=			tokenize_chinese_chars
						__snake_case      :							Optional[int]						=			normalizer_class(**_A)

					__snake_case      :							str						=			do_lower_case



				def  _lowercase					(self    :				Optional[Any]     ,  _A    :				Dict     ,  _A    :				Tuple=None)   ->     Any:
					__snake_case      :							List[Any]						=			[self.cls_token_id] + token_ids_a + [self.sep_token_id]

					if token_ids_a:
						output += token_ids_a + [self.sep_token_id]

					return output



				def  _lowercase					(self    :				str     ,  _A    :				List[int]     ,  _A    :				Optional[List[int]] = None)   ->     List[int]:
					__snake_case      :							Union[str, Any]						=			[self.sep_token_id]
					__snake_case      :							Union[str, Any]						=			[self.cls_token_id]
					if token_ids_a is None:
						return len(cls) * [self.cls_token_type_id] + len(token_ids_a + sep) * [0]
					return len(cls) * [self.cls_token_type_id] + len(token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]






				def  _lowercase					(self    :				Tuple     ,  _A    :				str     ,  _A    :				Optional[str] = None)   ->     Tuple[str]:
					__snake_case      :							int						=			self._tokenizer.model.save(_A     ,  name=_A)
					return tuple(_A)





