

"""simple docstring"""




import collections
import os
from typing import List, Optional, Tuple

from transformers.utils import is_jieba_available, requires_backends


if is_jieba_available():
						import jieba

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase							:  List[Any]            =						logging.get_logger(__name__)

lowerCAmelCase							:  Dict            =						{"""vocab_file""": """vocab.txt"""}

lowerCAmelCase							:  List[Any]            =						{
    """vocab_file""": {
        """openbmb/cpm-ant-10b""": """https://huggingface.co/openbmb/cpm-ant-10b/blob/main/vocab.txt""",
    },
}

lowerCAmelCase							:  Any            =						{
    """openbmb/cpm-ant-10b""": 1024,
}







def       A_				(   _UpperCAmelCase   ):
						SCREAMING_SNAKE_CASE_:			Union[str, Any]  		=	collections.OrderedDict()
						with open(SCREAMING_SNAKE_CASE_    ,     "r"    ,     encoding="utf-8"   ) as reader:
												SCREAMING_SNAKE_CASE_:			List[Any]  		=	reader.readlines()
						for index, token in enumerate(SCREAMING_SNAKE_CASE_   ):
												SCREAMING_SNAKE_CASE_:			Any  		=	token.rstrip("\n"   )
												SCREAMING_SNAKE_CASE_:			List[str]  		=	index
						return vocab



class   __lowercase    (			snake_case__     ):




			"""simple docstring"""







			def __init__(			self  :					Optional[Any]       ,						lowerCAmelCase__  :					List[str]       ,						lowerCAmelCase__  :					str="<unk>"       ,						lowerCAmelCase__  :					str=200):
									SCREAMING_SNAKE_CASE_:			Any  		=	vocab
									SCREAMING_SNAKE_CASE_:			List[str]  		=	unk_token
									SCREAMING_SNAKE_CASE_:			Tuple  		=	max_input_chars_per_word




			def 	_SCREAMING_SNAKE_CASE      (			self  :					Optional[int]       ,						lowerCAmelCase__  :					Dict):
									SCREAMING_SNAKE_CASE_:			Union[str, Any]  		=	list(_A)
									if len(_A) > self.max_input_chars_per_word:
															return [self.unk_token]

									SCREAMING_SNAKE_CASE_:			Optional[int]  		=	0
									SCREAMING_SNAKE_CASE_:			Tuple  		=	[]
									while start < len(_A):
															SCREAMING_SNAKE_CASE_:			Union[str, Any]  		=	len(_A)
															SCREAMING_SNAKE_CASE_:			Tuple  		=	None
															while start < end:
																					SCREAMING_SNAKE_CASE_:			Dict  		=	"".join(chars[start:end])
																					if substr in self.vocab:
																											SCREAMING_SNAKE_CASE_:			Optional[int]  		=	substr
																											break
																					end -= 1
															if cur_substr is None:
																					sub_tokens.append(self.unk_token)
																					start += 1
															else:
																					sub_tokens.append(_A)
																					SCREAMING_SNAKE_CASE_:			Tuple  		=	end

									return sub_tokens







class   __lowercase    (			snake_case__     ):




			"""simple docstring"""







			_UpperCAmelCase       :							Union[str, Any]      							=					VOCAB_FILES_NAMES
			_UpperCAmelCase       :							Optional[Any]      							=					PRETRAINED_VOCAB_FILES_MAP
			_UpperCAmelCase       :							Any      							=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			_UpperCAmelCase       :							List[Any]      							=					["""input_ids""", """attention_mask"""]
			_UpperCAmelCase       :							Dict      							=					False



			def __init__(			self  :					str       ,						lowerCAmelCase__  :					Optional[Any]       ,						lowerCAmelCase__  :					Optional[Any]="<d>"       ,						lowerCAmelCase__  :					int="</d>"       ,						lowerCAmelCase__  :					int="<s>"       ,						lowerCAmelCase__  :					str="</s>"       ,						lowerCAmelCase__  :					List[Any]="<pad>"       ,						lowerCAmelCase__  :					Any="<unk>"       ,						lowerCAmelCase__  :					str="</n>"       ,						lowerCAmelCase__  :					Tuple="</_>"       ,						lowerCAmelCase__  :					Optional[int]="left"       ,						**lowerCAmelCase__  :					List[Any]       ,						):
									requires_backends(self       ,						["jieba"])
									super().__init__(
									    bod_token=_A       ,						eod_token=_A       ,						bos_token=_A       ,						eos_token=_A       ,						pad_token=_A       ,						unk_token=_A       ,						line_token=_A       ,						space_token=_A       ,						padding_side=_A       ,						**_A       ,						)
									SCREAMING_SNAKE_CASE_:			List[Any]  		=	bod_token
									SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	eod_token
									SCREAMING_SNAKE_CASE_:			List[Any]  		=	load_vocab(_A)
									SCREAMING_SNAKE_CASE_:			List[str]  		=	self.encoder[space_token]
									SCREAMING_SNAKE_CASE_:			Any  		=	self.encoder[line_token]

									del self.encoder[space_token]
									del self.encoder[line_token]

									SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	collections.OrderedDict(sorted(self.encoder.items()       ,						key=lambda lowerCAmelCase__: x[1]))
									SCREAMING_SNAKE_CASE_:			Optional[int]  		=	{v: k for k, v in self.encoder.items()}

									SCREAMING_SNAKE_CASE_:			int  		=	WordpieceTokenizer(vocab=self.encoder       ,						unk_token=self.unk_token)



			@property
			def 	_SCREAMING_SNAKE_CASE      (			self  :					Optional[Any]):
									return self.encoder[self.bod_token]



			@property
			def 	_SCREAMING_SNAKE_CASE      (			self  :					Dict):
									return self.encoder[self.eod_token]



			@property
			def 	_SCREAMING_SNAKE_CASE      (			self  :					Dict):
									return self.encoder["\n"]



			@property
			def 	_SCREAMING_SNAKE_CASE      (			self  :					Optional[Any]):
									return len(self.encoder)



			def 	_SCREAMING_SNAKE_CASE      (			self  :					Union[str, Any]):
									return dict(self.encoder       ,						**self.added_tokens_encoder)



			def 	_SCREAMING_SNAKE_CASE      (			self  :					Any       ,						lowerCAmelCase__  :					int):
									SCREAMING_SNAKE_CASE_:			str  		=	[]
									for x in jieba.cut(_A       ,						cut_all=_A):
															output_tokens.extend(self.wordpiece_tokenizer.tokenize(_A))
									return output_tokens



			def 	_SCREAMING_SNAKE_CASE      (			self  :					Optional[Any]       ,						lowerCAmelCase__  :					List[str]       ,						**lowerCAmelCase__  :					List[str]):
									SCREAMING_SNAKE_CASE_:			List[Any]  		=	[i for i in token_ids if i >= 0]
									SCREAMING_SNAKE_CASE_:			List[str]  		=	[
									    x for x in token_ids if x != self.pad_token_id and x != self.eos_token_id and x != self.bos_token_id
									]
									return super()._decode(_A       ,						**_A)



			def 	_SCREAMING_SNAKE_CASE      (			self  :					List[Any]       ,						lowerCAmelCase__  :					Optional[int]):
									return token in self.encoder



			def 	_SCREAMING_SNAKE_CASE      (			self  :					int       ,						lowerCAmelCase__  :					Dict):
									return "".join(_A)



			def 	_SCREAMING_SNAKE_CASE      (			self  :					List[str]       ,						lowerCAmelCase__  :					Union[str, Any]):
									return self.encoder.get(_A       ,						self.encoder.get(self.unk_token))



			def 	_SCREAMING_SNAKE_CASE      (			self  :					int       ,						lowerCAmelCase__  :					Tuple):
									return self.decoder.get(_A       ,						self.unk_token)



			def 	_SCREAMING_SNAKE_CASE      (			self  :					List[Any]       ,						lowerCAmelCase__  :					int       ,						lowerCAmelCase__  :					Optional[int] = None):



									if os.path.isdir(_A):
															SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	os.path.join(
															    _A       ,						(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"])
									else:
															SCREAMING_SNAKE_CASE_:			List[Any]  		=	(filename_prefix + "-" if filename_prefix else "") + save_directory
									SCREAMING_SNAKE_CASE_:			Dict  		=	0
									if " " in self.encoder:
															SCREAMING_SNAKE_CASE_:			Any  		=	self.encoder[" "]
															del self.encoder[" "]
									if "\n" in self.encoder:
															SCREAMING_SNAKE_CASE_:			List[Any]  		=	self.encoder["\n"]
															del self.encoder["\n"]
									SCREAMING_SNAKE_CASE_:			Dict  		=	collections.OrderedDict(sorted(self.encoder.items()       ,						key=lambda lowerCAmelCase__: x[1]))
									with open(_A       ,						"w"       ,						encoding="utf-8") as writer:
															for token, token_index in self.encoder.items():
																					if index != token_index:
																											logger.warning(
																											    F"Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."
																											    " Please check that the vocabulary is not corrupted!")
																											SCREAMING_SNAKE_CASE_:			str  		=	token_index
																					writer.write(token + "\n")
																					index += 1
									return (vocab_file,)



			def 	_SCREAMING_SNAKE_CASE      (			self  :					List[Any]       ,						lowerCAmelCase__  :					List[Any]       ,						lowerCAmelCase__  :					Optional[Any] = None):



									if token_ids_a is None:
															return [self.bos_token_id] + token_ids_a
									return [self.bos_token_id] + token_ids_a + [self.bos_token_id] + token_ids_a




			def 	_SCREAMING_SNAKE_CASE      (			self  :					Optional[int]       ,						lowerCAmelCase__  :					Optional[int]       ,						lowerCAmelCase__  :					Tuple = None       ,						lowerCAmelCase__  :					Tuple = False):



									if already_has_special_tokens:
															return super().get_special_tokens_mask(
															    token_ids_a=_A       ,						token_ids_a=_A       ,						already_has_special_tokens=_A)

									if token_ids_a is not None:
															return [1] + ([0] * len(_A)) + [1] + ([0] * len(_A))
									return [1] + ([0] * len(_A))









from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import convert_to_rgb, normalize, rescale, resize, to_channel_dimension_format
from ...image_utils import (
    OPENAI_CLIP_MEAN,
    OPENAI_CLIP_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


if is_vision_available():
						import PIL


lowerCAmelCase							:  Optional[Any]            =						logging.get_logger(__name__)



class   __lowercase    (			UpperCAmelCase_     ):




			"""simple docstring"""







			_UpperCAmelCase       :							Any      							=					['''pixel_values''']



			def __init__(			self  :					List[Any]       ,						lowerCAmelCase__  :					bool = True       ,						lowerCAmelCase__  :					Dict[str, int] = None       ,						lowerCAmelCase__  :					PILImageResampling = PILImageResampling.BICUBIC       ,						lowerCAmelCase__  :					bool = True       ,						lowerCAmelCase__  :					Union[int, float] = 1 / 255       ,						lowerCAmelCase__  :					bool = True       ,						lowerCAmelCase__  :					Optional[Union[float, List[float]]] = None       ,						lowerCAmelCase__  :					Optional[Union[float, List[float]]] = None       ,						lowerCAmelCase__  :					bool = True       ,						**lowerCAmelCase__  :					Tuple       ,						):
									super().__init__(**lowerCAmelCase__)
									SCREAMING_SNAKE_CASE_:			List[str]  		=	size if size is not None else {"height": 384, "width": 384}
									SCREAMING_SNAKE_CASE_:			Union[str, Any]  		=	get_size_dict(lowerCAmelCase__       ,						default_to_square=lowerCAmelCase__)

									SCREAMING_SNAKE_CASE_:			int  		=	do_resize
									SCREAMING_SNAKE_CASE_:			Dict  		=	size
									SCREAMING_SNAKE_CASE_:			int  		=	resample
									SCREAMING_SNAKE_CASE_:			str  		=	do_rescale
									SCREAMING_SNAKE_CASE_:			str  		=	rescale_factor
									SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	do_normalize
									SCREAMING_SNAKE_CASE_:			Union[str, Any]  		=	image_mean if image_mean is not None else OPENAI_CLIP_MEAN
									SCREAMING_SNAKE_CASE_:			List[str]  		=	image_std if image_std is not None else OPENAI_CLIP_STD
									SCREAMING_SNAKE_CASE_:			Optional[int]  		=	do_convert_rgb



			def 	_SCREAMING_SNAKE_CASE      (			self  :					str       ,						lowerCAmelCase__  :					np.ndarray       ,						lowerCAmelCase__  :					Dict[str, int]       ,						lowerCAmelCase__  :					PILImageResampling = PILImageResampling.BICUBIC       ,						lowerCAmelCase__  :					Optional[Union[str, ChannelDimension]] = None       ,						**lowerCAmelCase__  :					Tuple       ,						):
									SCREAMING_SNAKE_CASE_:			List[Any]  		=	get_size_dict(lowerCAmelCase__       ,						default_to_square=lowerCAmelCase__)
									if "height" not in size or "width" not in size:
															raise ValueError(F"The `size` dictionary must contain the keys `height` and `width`. Got {size.keys()}")
									SCREAMING_SNAKE_CASE_:			int  		=	(size["height"], size["width"])
									return resize(lowerCAmelCase__       ,						size=lowerCAmelCase__       ,						resample=lowerCAmelCase__       ,						data_format=lowerCAmelCase__       ,						**lowerCAmelCase__)



			def 	_SCREAMING_SNAKE_CASE      (			self  :					Tuple       ,						lowerCAmelCase__  :					np.ndarray       ,						lowerCAmelCase__  :					Union[int, float]       ,						lowerCAmelCase__  :					Optional[Union[str, ChannelDimension]] = None       ,						**lowerCAmelCase__  :					List[Any]       ,						):
									return rescale(lowerCAmelCase__       ,						scale=lowerCAmelCase__       ,						data_format=lowerCAmelCase__       ,						**lowerCAmelCase__)



			def 	_SCREAMING_SNAKE_CASE      (			self  :					List[str]       ,						lowerCAmelCase__  :					np.ndarray       ,						lowerCAmelCase__  :					Union[float, List[float]]       ,						lowerCAmelCase__  :					Union[float, List[float]]       ,						lowerCAmelCase__  :					Optional[Union[str, ChannelDimension]] = None       ,						**lowerCAmelCase__  :					List[Any]       ,						):
									return normalize(lowerCAmelCase__       ,						mean=lowerCAmelCase__       ,						std=lowerCAmelCase__       ,						data_format=lowerCAmelCase__       ,						**lowerCAmelCase__)




			def 	_SCREAMING_SNAKE_CASE      (			self  :					Union[str, Any]       ,						lowerCAmelCase__  :					ImageInput       ,						lowerCAmelCase__  :					Optional[bool] = None       ,						lowerCAmelCase__  :					Optional[Dict[str, int]] = None       ,						lowerCAmelCase__  :					PILImageResampling = None       ,						lowerCAmelCase__  :					Optional[bool] = None       ,						lowerCAmelCase__  :					Optional[float] = None       ,						lowerCAmelCase__  :					Optional[bool] = None       ,						lowerCAmelCase__  :					Optional[Union[float, List[float]]] = None       ,						lowerCAmelCase__  :					Optional[Union[float, List[float]]] = None       ,						lowerCAmelCase__  :					Optional[Union[str, TensorType]] = None       ,						lowerCAmelCase__  :					bool = None       ,						lowerCAmelCase__  :					ChannelDimension = ChannelDimension.FIRST       ,						**lowerCAmelCase__  :					Dict       ,						):
									SCREAMING_SNAKE_CASE_:			Any  		=	do_resize if do_resize is not None else self.do_resize
									SCREAMING_SNAKE_CASE_:			Optional[int]  		=	resample if resample is not None else self.resample
									SCREAMING_SNAKE_CASE_:			Dict  		=	do_rescale if do_rescale is not None else self.do_rescale
									SCREAMING_SNAKE_CASE_:			Any  		=	rescale_factor if rescale_factor is not None else self.rescale_factor
									SCREAMING_SNAKE_CASE_:			Optional[int]  		=	do_normalize if do_normalize is not None else self.do_normalize
									SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	image_mean if image_mean is not None else self.image_mean
									SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	image_std if image_std is not None else self.image_std
									SCREAMING_SNAKE_CASE_:			Optional[int]  		=	do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb

									SCREAMING_SNAKE_CASE_:			str  		=	size if size is not None else self.size
									SCREAMING_SNAKE_CASE_:			List[Any]  		=	get_size_dict(lowerCAmelCase__       ,						default_to_square=lowerCAmelCase__)

									SCREAMING_SNAKE_CASE_:			List[Any]  		=	make_list_of_images(lowerCAmelCase__)

									if not valid_images(lowerCAmelCase__):
															raise ValueError(
															    "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
															    "torch.Tensor, tf.Tensor or jax.ndarray.")

									if do_resize and size is None or resample is None:
															raise ValueError("Size and resample must be specified if do_resize is True.")

									if do_rescale and rescale_factor is None:
															raise ValueError("Rescale factor must be specified if do_rescale is True.")

									if do_normalize and (image_mean is None or image_std is None):
															raise ValueError("Image mean and std must be specified if do_normalize is True.")

									# PIL RGBA images are converted to RGB
									if do_convert_rgb:
															SCREAMING_SNAKE_CASE_:			List[Any]  		=	[convert_to_rgb(lowerCAmelCase__) for image in images]

									# All transformations expect numpy arrays.
									SCREAMING_SNAKE_CASE_:			List[Any]  		=	[to_numpy_array(lowerCAmelCase__) for image in images]

									if do_resize:
															SCREAMING_SNAKE_CASE_:			int  		=	[self.resize(image=lowerCAmelCase__       ,						size=lowerCAmelCase__       ,						resample=lowerCAmelCase__) for image in images]

									if do_rescale:
															SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	[self.rescale(image=lowerCAmelCase__       ,						scale=lowerCAmelCase__) for image in images]

									if do_normalize:
															SCREAMING_SNAKE_CASE_:			List[str]  		=	[self.normalize(image=lowerCAmelCase__       ,						mean=lowerCAmelCase__       ,						std=lowerCAmelCase__) for image in images]

									SCREAMING_SNAKE_CASE_:			str  		=	[to_channel_dimension_format(lowerCAmelCase__       ,						lowerCAmelCase__) for image in images]

									SCREAMING_SNAKE_CASE_:			List[str]  		=	BatchFeature(data={"pixel_values": images}       ,						tensor_type=lowerCAmelCase__)

									return encoded_outputs






