





import json
import os
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available


if is_vision_available():
					from PIL import Image

					from transformers import CLIPImageProcessor, CLIPProcessor

@require_vision
class     __SCREAMING_SNAKE_CASE   (       unittest.TestCase    ):






							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     tempfile.mkdtemp()

														# fmt: off
														UpperCamelCase__			       =     ['''l''', '''o''', '''w''', '''e''', '''r''', '''s''', '''t''', '''i''', '''d''', '''n''', '''lo''', '''l</w>''', '''w</w>''', '''r</w>''', '''t</w>''', '''low</w>''', '''er</w>''', '''lowest</w>''', '''newer</w>''', '''wider''', '''<unk>''', '''<|startoftext|>''', '''<|endoftext|>''']
														# fmt: on
														UpperCamelCase__			       =     dict(zip(_lowerCamelCase					,	range(len(_lowerCamelCase					)					)					)					)
														UpperCamelCase__			       =     ['''#version: 0.2''', '''l o''', '''lo w</w>''', '''e r</w>''', '''''']
														UpperCamelCase__			       =     {'''unk_token''': '''<unk>'''}

														UpperCamelCase__			       =     os.path.join(self.tmpdirname					,	VOCAB_FILES_NAMES["""vocab_file"""]					)
														UpperCamelCase__			       =     os.path.join(self.tmpdirname					,	VOCAB_FILES_NAMES["""merges_file"""]					)
														with open(self.vocab_file					,	"""w"""					,	encoding="""utf-8"""					) as fp:
																					fp.write(json.dumps(_lowerCamelCase					) + """\n"""					)
														with open(self.merges_file					,	"""w"""					,	encoding="""utf-8"""					) as fp:
																					fp.write("""\n""".join(_lowerCamelCase					)					)

														UpperCamelCase__			       =     {
														    '''do_resize''': True,
														    '''size''': 20,
														    '''do_center_crop''': True,
														    '''crop_size''': 18,
														    '''do_normalize''': True,
														    '''image_mean''': [0.4814_5466, 0.457_8275, 0.4082_1073],
														    '''image_std''': [0.2686_2954, 0.2613_0258, 0.2757_7711],
														}
														UpperCamelCase__			       =     os.path.join(self.tmpdirname					,	_lowerCamelCase					)
														with open(self.image_processor_file					,	"""w"""					,	encoding="""utf-8"""					) as fp:
																					json.dump(_lowerCamelCase					,	_lowerCamelCase					)






							def 			_lowerCamelCase						(  self					,	**__lowerCAmelCase					):
														return CLIPTokenizer.from_pretrained(self.tmpdirname					,	**_lowerCamelCase					)






							def 			_lowerCamelCase						(  self					,	**__lowerCAmelCase					):
														return CLIPTokenizerFast.from_pretrained(self.tmpdirname					,	**_lowerCamelCase					)






							def 			_lowerCamelCase						(  self					,	**__lowerCAmelCase					):
														return CLIPImageProcessor.from_pretrained(self.tmpdirname					,	**_lowerCamelCase					)






							def 			_lowerCamelCase						(  self					):
														shutil.rmtree(self.tmpdirname					)






							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     [np.random.randint(255					,	size=(3, 30, 400)					,	dtype=np.uinta					)]

														UpperCamelCase__			       =     [Image.fromarray(np.moveaxis(_lowerCamelCase					,	0					,	-1					)					) for x in image_inputs]

														return image_inputs






							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     self.get_tokenizer()
														UpperCamelCase__			       =     self.get_rust_tokenizer()
														UpperCamelCase__			       =     self.get_image_processor()

														UpperCamelCase__			       =     CLIPProcessor(tokenizer=_lowerCamelCase					,	image_processor=_lowerCamelCase					)
														processor_slow.save_pretrained(self.tmpdirname					)
														UpperCamelCase__			       =     CLIPProcessor.from_pretrained(self.tmpdirname					,	use_fast=_lowerCamelCase					)

														UpperCamelCase__			       =     CLIPProcessor(tokenizer=_lowerCamelCase					,	image_processor=_lowerCamelCase					)
														processor_fast.save_pretrained(self.tmpdirname					)
														UpperCamelCase__			       =     CLIPProcessor.from_pretrained(self.tmpdirname					)

														self.assertEqual(processor_slow.tokenizer.get_vocab()					,	tokenizer_slow.get_vocab()					)
														self.assertEqual(processor_fast.tokenizer.get_vocab()					,	tokenizer_fast.get_vocab()					)
														self.assertEqual(tokenizer_slow.get_vocab()					,	tokenizer_fast.get_vocab()					)
														self.assertIsInstance(processor_slow.tokenizer					,	_lowerCamelCase					)
														self.assertIsInstance(processor_fast.tokenizer					,	_lowerCamelCase					)

														self.assertEqual(processor_slow.image_processor.to_json_string()					,	image_processor.to_json_string()					)
														self.assertEqual(processor_fast.image_processor.to_json_string()					,	image_processor.to_json_string()					)
														self.assertIsInstance(processor_slow.image_processor					,	_lowerCamelCase					)
														self.assertIsInstance(processor_fast.image_processor					,	_lowerCamelCase					)






							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     CLIPProcessor(tokenizer=self.get_tokenizer()					,	image_processor=self.get_image_processor()					)
														processor.save_pretrained(self.tmpdirname					)

														UpperCamelCase__			       =     self.get_tokenizer(bos_token="""(BOS)"""					,	eos_token="""(EOS)"""					)
														UpperCamelCase__			       =     self.get_image_processor(do_normalize=_lowerCamelCase					,	padding_value=1.0					)

														UpperCamelCase__			       =     CLIPProcessor.from_pretrained(
														    self.tmpdirname					,	bos_token="""(BOS)"""					,	eos_token="""(EOS)"""					,	do_normalize=_lowerCamelCase					,	padding_value=1.0					)

														self.assertEqual(processor.tokenizer.get_vocab()					,	tokenizer_add_kwargs.get_vocab()					)
														self.assertIsInstance(processor.tokenizer					,	_lowerCamelCase					)

														self.assertEqual(processor.image_processor.to_json_string()					,	image_processor_add_kwargs.to_json_string()					)
														self.assertIsInstance(processor.image_processor					,	_lowerCamelCase					)






							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     self.get_image_processor()
														UpperCamelCase__			       =     self.get_tokenizer()

														UpperCamelCase__			       =     CLIPProcessor(tokenizer=_lowerCamelCase					,	image_processor=_lowerCamelCase					)

														UpperCamelCase__			       =     self.prepare_image_inputs()

														UpperCamelCase__			       =     image_processor(_lowerCamelCase					,	return_tensors="""np"""					)
														UpperCamelCase__			       =     processor(images=_lowerCamelCase					,	return_tensors="""np"""					)

														for key in input_image_proc.keys():
																					self.assertAlmostEqual(input_image_proc[key].sum()					,	input_processor[key].sum()					,	delta=1E-2					)






							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     self.get_image_processor()
														UpperCamelCase__			       =     self.get_tokenizer()

														UpperCamelCase__			       =     CLIPProcessor(tokenizer=_lowerCamelCase					,	image_processor=_lowerCamelCase					)

														UpperCamelCase__			       =     '''lower newer'''

														UpperCamelCase__			       =     processor(text=_lowerCamelCase					)

														UpperCamelCase__			       =     tokenizer(_lowerCamelCase					)

														for key in encoded_tok.keys():
																					self.assertListEqual(encoded_tok[key]					,	encoded_processor[key]					)






							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     self.get_image_processor()
														UpperCamelCase__			       =     self.get_tokenizer()

														UpperCamelCase__			       =     CLIPProcessor(tokenizer=_lowerCamelCase					,	image_processor=_lowerCamelCase					)

														UpperCamelCase__			       =     '''lower newer'''
														UpperCamelCase__			       =     self.prepare_image_inputs()

														UpperCamelCase__			       =     processor(text=_lowerCamelCase					,	images=_lowerCamelCase					)

														self.assertListEqual(list(inputs.keys()					)					,	["""input_ids""", """attention_mask""", """pixel_values"""]					)

														# test if it raises when no input is passed
														with pytest.raises(_lowerCamelCase					):
																					processor()






							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     self.get_image_processor()
														UpperCamelCase__			       =     self.get_tokenizer()

														UpperCamelCase__			       =     CLIPProcessor(tokenizer=_lowerCamelCase					,	image_processor=_lowerCamelCase					)

														UpperCamelCase__			       =     [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

														UpperCamelCase__			       =     processor.batch_decode(_lowerCamelCase					)
														UpperCamelCase__			       =     tokenizer.batch_decode(_lowerCamelCase					)

														self.assertListEqual(_lowerCamelCase					,	_lowerCamelCase					)







							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     self.get_image_processor()
														UpperCamelCase__			       =     self.get_tokenizer()

														UpperCamelCase__			       =     CLIPProcessor(tokenizer=_lowerCamelCase					,	image_processor=_lowerCamelCase					)

														UpperCamelCase__			       =     '''lower newer'''
														UpperCamelCase__			       =     self.prepare_image_inputs()

														UpperCamelCase__			       =     processor(text=_lowerCamelCase					,	images=_lowerCamelCase					)

														self.assertListEqual(list(inputs.keys()					)					,	processor.model_input_names					)











import collections
import os
from typing import List, Optional, Tuple

from transformers.utils import is_jieba_available, requires_backends


if is_jieba_available():
					import jieba

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCamelCase__     =	logging.get_logger(__name__)

UpperCamelCase__     =	{"vocab_file": "vocab.txt"}

UpperCamelCase__     =	{
    "vocab_file": {
        "openbmb/cpm-ant-10b": "https://huggingface.co/openbmb/cpm-ant-10b/blob/main/vocab.txt",
    },
}

UpperCamelCase__     =	{
    "openbmb/cpm-ant-10b": 1_024,
}





def       _UpperCamelCase       (a__				:Tuple    ):






							"""simple docstring"""





							UpperCamelCase__			       =     collections.OrderedDict()
							with open(a__				,     """r"""				,     encoding="""utf-8"""    ) as reader:
														UpperCamelCase__			       =     reader.readlines()
							for index, token in enumerate(a__    ):
														UpperCamelCase__			       =     token.rstrip("""\n"""    )
														UpperCamelCase__			       =     index
							return vocab

class     __SCREAMING_SNAKE_CASE   (       _a    ):






							def __init__(  self					,	__lowerCAmelCase					,	__lowerCAmelCase="<unk>"					,	__lowerCAmelCase=200					):
														UpperCamelCase__			       =     vocab
														UpperCamelCase__			       =     unk_token
														UpperCamelCase__			       =     max_input_chars_per_word







							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					):
														UpperCamelCase__			       =     list(__lowerCAmelCase					)
														if len(__lowerCAmelCase					) > self.max_input_chars_per_word:
																					return [self.unk_token]

														UpperCamelCase__			       =     0
														UpperCamelCase__			       =     []
														while start < len(__lowerCAmelCase					):
																					UpperCamelCase__			       =     len(__lowerCAmelCase					)
																					UpperCamelCase__			       =     None
																					while start < end:
																												UpperCamelCase__			       =     """""".join(chars[start:end]					)
																												if substr in self.vocab:
																																			UpperCamelCase__			       =     substr
																																			break
																												end -= 1
																					if cur_substr is None:
																												sub_tokens.append(self.unk_token					)
																												start += 1
																					else:
																												sub_tokens.append(__lowerCAmelCase					)
																												UpperCamelCase__			       =     end

														return sub_tokens






class     __SCREAMING_SNAKE_CASE   (       _a    ):
							snake_case       : Optional[int]         =    VOCAB_FILES_NAMES
							snake_case       : Optional[Any]         =    PRETRAINED_VOCAB_FILES_MAP
							snake_case       : List[Any]         =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							snake_case       : Optional[int]         =    ["""input_ids""", """attention_mask"""]
							snake_case       : int         =    False






							def __init__(  self					,	__lowerCAmelCase					,	__lowerCAmelCase="<d>"					,	__lowerCAmelCase="</d>"					,	__lowerCAmelCase="<s>"					,	__lowerCAmelCase="</s>"					,	__lowerCAmelCase="<pad>"					,	__lowerCAmelCase="<unk>"					,	__lowerCAmelCase="</n>"					,	__lowerCAmelCase="</_>"					,	__lowerCAmelCase="left"					,	**__lowerCAmelCase					,	):
														requires_backends(self					,	["""jieba"""]					)
														super().__init__(
														    bod_token=__lowerCAmelCase					,	eod_token=__lowerCAmelCase					,	bos_token=__lowerCAmelCase					,	eos_token=__lowerCAmelCase					,	pad_token=__lowerCAmelCase					,	unk_token=__lowerCAmelCase					,	line_token=__lowerCAmelCase					,	space_token=__lowerCAmelCase					,	padding_side=__lowerCAmelCase					,	**__lowerCAmelCase					,	)
														UpperCamelCase__			       =     bod_token
														UpperCamelCase__			       =     eod_token
														UpperCamelCase__			       =     load_vocab(__lowerCAmelCase					)
														UpperCamelCase__			       =     self.encoder[space_token]
														UpperCamelCase__			       =     self.encoder[line_token]

														del self.encoder[space_token]
														del self.encoder[line_token]

														UpperCamelCase__			       =     collections.OrderedDict(sorted(self.encoder.items()					,	key=lambda __lowerCAmelCase					: x[1]					)					)
														UpperCamelCase__			       =     {v: k for k, v in self.encoder.items()}

														UpperCamelCase__			       =     WordpieceTokenizer(vocab=self.encoder					,	unk_token=self.unk_token					)






							@property
							def 			_lowerCamelCase						(  self					):
														return self.encoder[self.bod_token]






							@property
							def 			_lowerCamelCase						(  self					):
														return self.encoder[self.eod_token]






							@property
							def 			_lowerCamelCase						(  self					):
														return self.encoder["\n"]






							@property
							def 			_lowerCamelCase						(  self					):
														return len(self.encoder					)






							def 			_lowerCamelCase						(  self					):
														return dict(self.encoder					,	**self.added_tokens_encoder					)






							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					):
														UpperCamelCase__			       =     []
														for x in jieba.cut(__lowerCAmelCase					,	cut_all=__lowerCAmelCase					):
																					output_tokens.extend(self.wordpiece_tokenizer.tokenize(__lowerCAmelCase					)					)
														return output_tokens






							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					,	**__lowerCAmelCase					):
														UpperCamelCase__			       =     [i for i in token_ids if i >= 0]
														UpperCamelCase__			       =     [
														    x for x in token_ids if x != self.pad_token_id and x != self.eos_token_id and x != self.bos_token_id
														]
														return super()._decode(__lowerCAmelCase					,	**__lowerCAmelCase					)






							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					):
														return token in self.encoder






							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					):
														return "".join(__lowerCAmelCase					)






							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					):
														return self.encoder.get(__lowerCAmelCase					,	self.encoder.get(self.unk_token					)					)






							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					):
														return self.decoder.get(__lowerCAmelCase					,	self.unk_token					)






							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					,	__lowerCAmelCase = None					):
														if os.path.isdir(__lowerCAmelCase					):
																					UpperCamelCase__			       =     os.path.join(
																					    __lowerCAmelCase					,	(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]					)
														else:
																					UpperCamelCase__			       =     (filename_prefix + """-""" if filename_prefix else """""") + save_directory
														UpperCamelCase__			       =     0
														if " " in self.encoder:
																					UpperCamelCase__			       =     self.encoder[""" """]
																					del self.encoder[" "]
														if "\n" in self.encoder:
																					UpperCamelCase__			       =     self.encoder["""\n"""]
																					del self.encoder["\n"]
														UpperCamelCase__			       =     collections.OrderedDict(sorted(self.encoder.items()					,	key=lambda __lowerCAmelCase					: x[1]					)					)
														with open(__lowerCAmelCase					,	"""w"""					,	encoding="""utf-8"""					) as writer:
																					for token, token_index in self.encoder.items():
																												if index != token_index:
																																			logger.warning(
																																			    f"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
																																			    """ Please check that the vocabulary is not corrupted!"""					)
																																			UpperCamelCase__			       =     token_index
																												writer.write(token + """\n"""					)
																												index += 1
														return (vocab_file,)






							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					,	__lowerCAmelCase = None					):
														if token_ids_a is None:
																					return [self.bos_token_id] + token_ids_a
														return [self.bos_token_id] + token_ids_a + [self.bos_token_id] + token_ids_a







							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					,	__lowerCAmelCase = None					,	__lowerCAmelCase = False					):

														if already_has_special_tokens:
																					return super().get_special_tokens_mask(
																					    token_ids_a=__lowerCAmelCase					,	token_ids_a=__lowerCAmelCase					,	already_has_special_tokens=__lowerCAmelCase					)

														if token_ids_a is not None:
																					return [1] + ([0] * len(__lowerCAmelCase					)) + [1] + ([0] * len(__lowerCAmelCase					))
														return [1] + ([0] * len(__lowerCAmelCase					))




