






import collections
import os
from typing import List, Optional, Tuple

from transformers.utils import is_jieba_available, requires_backends


if is_jieba_available():
					import jieba

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__snake_case      :      Optional[Any]             =       logging.get_logger(__name__)

__snake_case      :      Optional[Any]             =       {"""vocab_file""": """vocab.txt"""}

__snake_case      :      Optional[Any]             =       {
    """vocab_file""": {
        """openbmb/cpm-ant-10b""": """https://huggingface.co/openbmb/cpm-ant-10b/blob/main/vocab.txt""",
    },
}

__snake_case      :      List[str]             =       {
    """openbmb/cpm-ant-10b""": 10_24,
}







def 		_UpperCamelCase				(      UpperCamelCase_    :   List[Any]						)					->				str:

						"""simple docstring"""



						lowerCAmelCase__        =  collections.OrderedDict()
						with open(__lowerCAmelCase      ,    'r'      ,    encoding='utf-8'						) as reader:
												lowerCAmelCase__        =  reader.readlines()
						for index, token in enumerate(__lowerCAmelCase						):
												lowerCAmelCase__        =  token.rstrip('\n'						)
												lowerCAmelCase__        =  index
						return vocab






class       __SCREAMING_SNAKE_CASE   (				__lowerCamelCase):






						def __init__(		self       ,  _UpperCamelCase       ,  _UpperCamelCase="<unk>"       ,  _UpperCamelCase=2_00				):






												"""simple docstring"""
												lowerCAmelCase__        =  vocab
												lowerCAmelCase__        =  unk_token
												lowerCAmelCase__        =  max_input_chars_per_word







						def  UpperCamelCase__ (		self       ,  _UpperCamelCase				):






												"""simple docstring"""
												lowerCAmelCase__        =  list(__lowercase				)
												if len(__lowercase				) > self.max_input_chars_per_word:
																		return [self.unk_token]

												lowerCAmelCase__        =  0
												lowerCAmelCase__        =  []
												while start < len(__lowercase				):
																		lowerCAmelCase__        =  len(__lowercase				)
																		lowerCAmelCase__        =  None
																		while start < end:
																								lowerCAmelCase__        =  ''''''.join(chars[start:end]				)
																								if substr in self.vocab:
																														lowerCAmelCase__        =  substr
																														break
																								end -= 1
																		if cur_substr is None:
																								sub_tokens.append(self.unk_token				)
																								start += 1
																		else:
																								sub_tokens.append(__lowercase				)
																								lowerCAmelCase__        =  end

												return sub_tokens






class       __SCREAMING_SNAKE_CASE   (				__lowerCamelCase):
						_SCREAMING_SNAKE_CASE		:    List[str]						=   VOCAB_FILES_NAMES
						_SCREAMING_SNAKE_CASE		:    Any						=   PRETRAINED_VOCAB_FILES_MAP
						_SCREAMING_SNAKE_CASE		:    str						=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						_SCREAMING_SNAKE_CASE		:    Tuple						=   ["""input_ids""", """attention_mask"""]
						_SCREAMING_SNAKE_CASE		:    Optional[Any]						=   False






						def __init__(		self       ,  _UpperCamelCase       ,  _UpperCamelCase="<d>"       ,  _UpperCamelCase="</d>"       ,  _UpperCamelCase="<s>"       ,  _UpperCamelCase="</s>"       ,  _UpperCamelCase="<pad>"       ,  _UpperCamelCase="<unk>"       ,  _UpperCamelCase="</n>"       ,  _UpperCamelCase="</_>"       ,  _UpperCamelCase="left"       ,  **_UpperCamelCase       ,  ):






												"""simple docstring"""
												requires_backends(self       ,  ['jieba']				)
												super().__init__(
												    bod_token=__lowercase       ,  eod_token=__lowercase       ,  bos_token=__lowercase       ,  eos_token=__lowercase       ,  pad_token=__lowercase       ,  unk_token=__lowercase       ,  line_token=__lowercase       ,  space_token=__lowercase       ,  padding_side=__lowercase       ,  **__lowercase       ,  )
												lowerCAmelCase__        =  bod_token
												lowerCAmelCase__        =  eod_token
												lowerCAmelCase__        =  load_vocab(__lowercase				)
												lowerCAmelCase__        =  self.encoder[space_token]
												lowerCAmelCase__        =  self.encoder[line_token]

												del self.encoder[space_token]
												del self.encoder[line_token]

												lowerCAmelCase__        =  collections.OrderedDict(sorted(self.encoder.items()       ,  key=lambda _UpperCamelCase				: x[1]				)				)
												lowerCAmelCase__        =  {v: k for k, v in self.encoder.items()}

												lowerCAmelCase__        =  WordpieceTokenizer(vocab=self.encoder       ,  unk_token=self.unk_token				)






						@property
						def  UpperCamelCase__ (		self				):






												"""simple docstring"""
												return self.encoder[self.bod_token]






						@property
						def  UpperCamelCase__ (		self				):






												"""simple docstring"""
												return self.encoder[self.eod_token]






						@property
						def  UpperCamelCase__ (		self				):






												"""simple docstring"""
												return self.encoder["\n"]






						@property
						def  UpperCamelCase__ (		self				):






												"""simple docstring"""
												return len(self.encoder				)






						def  UpperCamelCase__ (		self				):






												"""simple docstring"""
												return dict(self.encoder       ,  **self.added_tokens_encoder				)






						def  UpperCamelCase__ (		self       ,  _UpperCamelCase				):






												"""simple docstring"""
												lowerCAmelCase__        =  []
												for x in jieba.cut(__lowercase       ,  cut_all=__lowercase				):
																		output_tokens.extend(self.wordpiece_tokenizer.tokenize(__lowercase				)				)
												return output_tokens






						def  UpperCamelCase__ (		self       ,  _UpperCamelCase       ,  **_UpperCamelCase				):






												"""simple docstring"""
												lowerCAmelCase__        =  [i for i in token_ids if i >= 0]
												lowerCAmelCase__        =  [
												    x for x in token_ids if x != self.pad_token_id and x != self.eos_token_id and x != self.bos_token_id
												]
												return super()._decode(__lowercase       ,  **__lowercase				)






						def  UpperCamelCase__ (		self       ,  _UpperCamelCase				):






												"""simple docstring"""
												return token in self.encoder






						def  UpperCamelCase__ (		self       ,  _UpperCamelCase				):






												"""simple docstring"""
												return "".join(__lowercase				)






						def  UpperCamelCase__ (		self       ,  _UpperCamelCase				):






												"""simple docstring"""
												return self.encoder.get(__lowercase       ,  self.encoder.get(self.unk_token				)				)






						def  UpperCamelCase__ (		self       ,  _UpperCamelCase				):






												"""simple docstring"""
												return self.decoder.get(__lowercase       ,  self.unk_token				)






						def  UpperCamelCase__ (		self       ,  _UpperCamelCase       ,  _UpperCamelCase = None				):






												"""simple docstring"""
												if os.path.isdir(__lowercase				):
																		lowerCAmelCase__        =  os.path.join(
																		    __lowercase       ,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']				)
												else:
																		lowerCAmelCase__        =  (filename_prefix + '''-''' if filename_prefix else '''''') + save_directory
												lowerCAmelCase__        =  0
												if " " in self.encoder:
																		lowerCAmelCase__        =  self.encoder[''' ''']
																		del self.encoder[" "]
												if "\n" in self.encoder:
																		lowerCAmelCase__        =  self.encoder['''\n''']
																		del self.encoder["\n"]
												lowerCAmelCase__        =  collections.OrderedDict(sorted(self.encoder.items()       ,  key=lambda _UpperCamelCase				: x[1]				)				)
												with open(__lowercase       ,  'w'       ,  encoding='utf-8'				) as writer:
																		for token, token_index in self.encoder.items():
																								if index != token_index:
																														logger.warning(
																														    F"Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."
																														    ' Please check that the vocabulary is not corrupted!'				)
																														lowerCAmelCase__        =  token_index
																								writer.write(token + '\n'				)
																								index += 1
												return (vocab_file,)






						def  UpperCamelCase__ (		self       ,  _UpperCamelCase       ,  _UpperCamelCase = None				):






												"""simple docstring"""
												if token_ids_a is None:
																		return [self.bos_token_id] + token_ids_a
												return [self.bos_token_id] + token_ids_a + [self.bos_token_id] + token_ids_a







						def  UpperCamelCase__ (		self       ,  _UpperCamelCase       ,  _UpperCamelCase = None       ,  _UpperCamelCase = False				):






												"""simple docstring"""

												if already_has_special_tokens:
																		return super().get_special_tokens_mask(
																		    token_ids_a=__lowercase       ,  token_ids_a=__lowercase       ,  already_has_special_tokens=__lowercase				)

												if token_ids_a is not None:
																		return [1] + ([0] * len(__lowercase				)) + [1] + ([0] * len(__lowercase				))
												return [1] + ([0] * len(__lowercase				))













from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, logging


__snake_case      :      Any             =       logging.get_logger(__name__)


class       __SCREAMING_SNAKE_CASE   (				__lowercase):
						_SCREAMING_SNAKE_CASE		:    List[Any]						=   ['''pixel_values''']






						def __init__(		self       ,  _UpperCamelCase = True       ,  _UpperCamelCase = None       ,  _UpperCamelCase = PILImageResampling.BILINEAR       ,  _UpperCamelCase = True       ,  _UpperCamelCase = None       ,  _UpperCamelCase = True       ,  _UpperCamelCase = 1 / 2_55       ,  _UpperCamelCase = True       ,  _UpperCamelCase = None       ,  _UpperCamelCase = None       ,  **_UpperCamelCase       ,  ):






												"""simple docstring"""
												super().__init__(**_UpperCamelCase				)
												lowerCAmelCase__        =  size if size is not None else {'shortest_edge': 2_56}
												lowerCAmelCase__        =  get_size_dict(_UpperCamelCase       ,  default_to_square=_UpperCamelCase				)
												lowerCAmelCase__        =  crop_size if crop_size is not None else {'height': 2_24, 'width': 2_24}
												lowerCAmelCase__        =  get_size_dict(_UpperCamelCase				)
												lowerCAmelCase__        =  do_resize
												lowerCAmelCase__        =  size
												lowerCAmelCase__        =  resample
												lowerCAmelCase__        =  do_center_crop
												lowerCAmelCase__        =  crop_size
												lowerCAmelCase__        =  do_rescale
												lowerCAmelCase__        =  rescale_factor
												lowerCAmelCase__        =  do_normalize
												lowerCAmelCase__        =  image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
												lowerCAmelCase__        =  image_std if image_std is not None else IMAGENET_STANDARD_STD






						def  UpperCamelCase__ (		self       ,  _UpperCamelCase       ,  _UpperCamelCase       ,  _UpperCamelCase = PILImageResampling.BICUBIC       ,  _UpperCamelCase = None       ,  **_UpperCamelCase       ,  ):






												"""simple docstring"""
												lowerCAmelCase__        =  get_size_dict(_UpperCamelCase       ,  default_to_square=_UpperCamelCase				)
												if "shortest_edge" not in size:
																		raise ValueError(F"The `size` parameter must contain the key `shortest_edge`. Got {size.keys()}"				)
												lowerCAmelCase__        =  get_resize_output_image_size(_UpperCamelCase       ,  size=size['shortest_edge']       ,  default_to_square=_UpperCamelCase				)
												return resize(_UpperCamelCase       ,  size=_UpperCamelCase       ,  resample=_UpperCamelCase       ,  data_format=_UpperCamelCase       ,  **_UpperCamelCase				)






						def  UpperCamelCase__ (		self       ,  _UpperCamelCase       ,  _UpperCamelCase       ,  _UpperCamelCase = None       ,  **_UpperCamelCase       ,  ):






												"""simple docstring"""
												lowerCAmelCase__        =  get_size_dict(_UpperCamelCase				)
												return center_crop(_UpperCamelCase       ,  size=(size['height'], size['width'])       ,  data_format=_UpperCamelCase       ,  **_UpperCamelCase				)






						def  UpperCamelCase__ (		self       ,  _UpperCamelCase       ,  _UpperCamelCase       ,  _UpperCamelCase = None       ,  **_UpperCamelCase				):






												"""simple docstring"""
												return rescale(_UpperCamelCase       ,  scale=_UpperCamelCase       ,  data_format=_UpperCamelCase       ,  **_UpperCamelCase				)






						def  UpperCamelCase__ (		self       ,  _UpperCamelCase       ,  _UpperCamelCase       ,  _UpperCamelCase       ,  _UpperCamelCase = None       ,  **_UpperCamelCase       ,  ):






												"""simple docstring"""
												return normalize(_UpperCamelCase       ,  mean=_UpperCamelCase       ,  std=_UpperCamelCase       ,  data_format=_UpperCamelCase       ,  **_UpperCamelCase				)







						def  UpperCamelCase__ (		self       ,  _UpperCamelCase       ,  _UpperCamelCase = None       ,  _UpperCamelCase = None       ,  _UpperCamelCase = None       ,  _UpperCamelCase = None       ,  _UpperCamelCase = None       ,  _UpperCamelCase = None       ,  _UpperCamelCase = None       ,  _UpperCamelCase = None       ,  _UpperCamelCase = None       ,  _UpperCamelCase = None       ,  _UpperCamelCase = None       ,  _UpperCamelCase = ChannelDimension.FIRST       ,  **_UpperCamelCase       ,  ):






												"""simple docstring"""
												lowerCAmelCase__        =  do_resize if do_resize is not None else self.do_resize
												lowerCAmelCase__        =  size if size is not None else self.size
												lowerCAmelCase__        =  get_size_dict(_UpperCamelCase       ,  default_to_square=_UpperCamelCase				)
												lowerCAmelCase__        =  resample if resample is not None else self.resample
												lowerCAmelCase__        =  do_center_crop if do_center_crop is not None else self.do_center_crop
												lowerCAmelCase__        =  crop_size if crop_size is not None else self.crop_size
												lowerCAmelCase__        =  get_size_dict(_UpperCamelCase				)
												lowerCAmelCase__        =  do_rescale if do_rescale is not None else self.do_rescale
												lowerCAmelCase__        =  rescale_factor if rescale_factor is not None else self.rescale_factor
												lowerCAmelCase__        =  do_normalize if do_normalize is not None else self.do_normalize
												lowerCAmelCase__        =  image_mean if image_mean is not None else self.image_mean
												lowerCAmelCase__        =  image_std if image_std is not None else self.image_std

												lowerCAmelCase__        =  make_list_of_images(_UpperCamelCase				)

												if not valid_images(_UpperCamelCase				):
																		raise ValueError(
																		    'Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '
																		    'torch.Tensor, tf.Tensor or jax.ndarray.'				)

												if do_resize and size is None:
																		raise ValueError('Size must be specified if do_resize is True.'				)

												if do_center_crop and crop_size is None:
																		raise ValueError('Crop size must be specified if do_center_crop is True.'				)

												if do_rescale and rescale_factor is None:
																		raise ValueError('Rescale factor must be specified if do_rescale is True.'				)

												if do_normalize and (image_mean is None or image_std is None):
																		raise ValueError('Image mean and std must be specified if do_normalize is True.'				)

												# All transformations expect numpy arrays.
												lowerCAmelCase__        =  [to_numpy_array(_UpperCamelCase				) for image in images]

												if do_resize:
																		lowerCAmelCase__        =  [self.resize(image=_UpperCamelCase       ,  size=_UpperCamelCase       ,  resample=_UpperCamelCase				) for image in images]

												if do_center_crop:
																		lowerCAmelCase__        =  [self.center_crop(image=_UpperCamelCase       ,  size=_UpperCamelCase				) for image in images]

												if do_rescale:
																		lowerCAmelCase__        =  [self.rescale(image=_UpperCamelCase       ,  scale=_UpperCamelCase				) for image in images]

												if do_normalize:
																		lowerCAmelCase__        =  [self.normalize(image=_UpperCamelCase       ,  mean=_UpperCamelCase       ,  std=_UpperCamelCase				) for image in images]

												lowerCAmelCase__        =  [to_channel_dimension_format(_UpperCamelCase       ,  _UpperCamelCase				) for image in images]

												lowerCAmelCase__        =  {'pixel_values': images}
												return BatchFeature(data=_UpperCamelCase       ,  tensor_type=_UpperCamelCase				)





