



import json
import os
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available


if is_vision_available():
 from PIL import Image

 from transformers import OwlViTImageProcessor, OwlViTProcessor

@require_vision
class    _A	(     unittest.TestCase   ):



       def 		_lowerCamelCase      (		self							:					Dict):


           '''simple docstring'''

           __a							=    tempfile.mkdtemp()

           # fmt: off
           __a							=    ['''''', '''l''', '''o''', '''w''', '''e''', '''r''', '''s''', '''t''', '''i''', '''d''', '''n''', '''lo''', '''l</w>''', '''w</w>''', '''r</w>''', '''t</w>''', '''low</w>''', '''er</w>''', '''lowest</w>''', '''newer</w>''', '''wider''', '''<unk>''', '''<|startoftext|>''', '''<|endoftext|>''']
           # fmt: on
           __a							=    dict(zip(__SCREAMING_SNAKE_CASE					,		range(len(__SCREAMING_SNAKE_CASE))))
           __a							=    ['''#version: 0.2''', '''l o''', '''lo w</w>''', '''e r</w>''', '''''']
           __a							=    {'''unk_token''': '''<unk>'''}

           __a							=    os.path.join(self.tmpdirname					,		VOCAB_FILES_NAMES['''vocab_file'''])
           __a							=    os.path.join(self.tmpdirname					,		VOCAB_FILES_NAMES['''merges_file'''])
           with open(self.vocab_file					,		'''w'''					,		encoding='''utf-8''') as fp:
               fp.write(json.dumps(__SCREAMING_SNAKE_CASE) + '''\n''')
           with open(self.merges_file					,		'''w'''					,		encoding='''utf-8''') as fp:
               fp.write('''\n'''.join(__SCREAMING_SNAKE_CASE))

           __a							=    {
               '''do_resize''': True,
               '''size''': 20,
               '''do_center_crop''': True,
               '''crop_size''': 18,
               '''do_normalize''': True,
               '''image_mean''': [0.48_14_54_66, 0.4_57_82_75, 0.40_82_10_73],
               '''image_std''': [0.26_86_29_54, 0.26_13_02_58, 0.27_57_77_11],
           }
           __a							=    os.path.join(self.tmpdirname					,		__SCREAMING_SNAKE_CASE)
           with open(self.image_processor_file					,		'''w'''					,		encoding='''utf-8''') as fp:
               json.dump(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE)



       def 		_lowerCamelCase      (		self							:					str					,		**__SCREAMING_SNAKE_CASE							:					Tuple):


           '''simple docstring'''

           return CLIPTokenizer.from_pretrained(self.tmpdirname					,		pad_token='''!'''					,		**__SCREAMING_SNAKE_CASE)



       def 		_lowerCamelCase      (		self							:					Any					,		**__SCREAMING_SNAKE_CASE							:					Any):


           '''simple docstring'''

           return CLIPTokenizerFast.from_pretrained(self.tmpdirname					,		pad_token='''!'''					,		**__SCREAMING_SNAKE_CASE)



       def 		_lowerCamelCase      (		self							:					Any					,		**__SCREAMING_SNAKE_CASE							:					Union[str, Any]):


           '''simple docstring'''

           return OwlViTImageProcessor.from_pretrained(self.tmpdirname					,		**__SCREAMING_SNAKE_CASE)



       def 		_lowerCamelCase      (		self							:					List[str]):


           '''simple docstring'''

           shutil.rmtree(self.tmpdirname)



       def 		_lowerCamelCase      (		self							:					Dict):


           '''simple docstring'''

           __a							=    [np.random.randint(255					,		size=(3, 30, 400)					,		dtype=np.uinta)]

           __a							=    [Image.fromarray(np.moveaxis(__SCREAMING_SNAKE_CASE					,		0					,		-1)) for x in image_inputs]

           return image_inputs



       def 		_lowerCamelCase      (		self							:					List[str]):


           '''simple docstring'''

           __a							=    self.get_tokenizer()
           __a							=    self.get_rust_tokenizer()
           __a							=    self.get_image_processor()

           __a							=    OwlViTProcessor(tokenizer=__SCREAMING_SNAKE_CASE					,		image_processor=__SCREAMING_SNAKE_CASE)
           processor_slow.save_pretrained(self.tmpdirname)
           __a							=    OwlViTProcessor.from_pretrained(self.tmpdirname					,		use_fast=__SCREAMING_SNAKE_CASE)

           __a							=    OwlViTProcessor(tokenizer=__SCREAMING_SNAKE_CASE					,		image_processor=__SCREAMING_SNAKE_CASE)
           processor_fast.save_pretrained(self.tmpdirname)
           __a							=    OwlViTProcessor.from_pretrained(self.tmpdirname)

           self.assertEqual(processor_slow.tokenizer.get_vocab()					,		tokenizer_slow.get_vocab())
           self.assertEqual(processor_fast.tokenizer.get_vocab()					,		tokenizer_fast.get_vocab())
           self.assertEqual(tokenizer_slow.get_vocab()					,		tokenizer_fast.get_vocab())
           self.assertIsInstance(processor_slow.tokenizer					,		__SCREAMING_SNAKE_CASE)
           self.assertIsInstance(processor_fast.tokenizer					,		__SCREAMING_SNAKE_CASE)

           self.assertEqual(processor_slow.image_processor.to_json_string()					,		image_processor.to_json_string())
           self.assertEqual(processor_fast.image_processor.to_json_string()					,		image_processor.to_json_string())
           self.assertIsInstance(processor_slow.image_processor					,		__SCREAMING_SNAKE_CASE)
           self.assertIsInstance(processor_fast.image_processor					,		__SCREAMING_SNAKE_CASE)



       def 		_lowerCamelCase      (		self							:					Tuple):


           '''simple docstring'''

           __a							=    OwlViTProcessor(tokenizer=self.get_tokenizer()					,		image_processor=self.get_image_processor())
           processor.save_pretrained(self.tmpdirname)

           __a							=    self.get_tokenizer(bos_token='''(BOS)'''					,		eos_token='''(EOS)''')
           __a							=    self.get_image_processor(do_normalize=__SCREAMING_SNAKE_CASE)

           __a							=    OwlViTProcessor.from_pretrained(
               self.tmpdirname					,		bos_token='''(BOS)'''					,		eos_token='''(EOS)'''					,		do_normalize=__SCREAMING_SNAKE_CASE)

           self.assertEqual(processor.tokenizer.get_vocab()					,		tokenizer_add_kwargs.get_vocab())
           self.assertIsInstance(processor.tokenizer					,		__SCREAMING_SNAKE_CASE)

           self.assertEqual(processor.image_processor.to_json_string()					,		image_processor_add_kwargs.to_json_string())
           self.assertIsInstance(processor.image_processor					,		__SCREAMING_SNAKE_CASE)



       def 		_lowerCamelCase      (		self							:					Tuple):


           '''simple docstring'''

           __a							=    self.get_image_processor()
           __a							=    self.get_tokenizer()

           __a							=    OwlViTProcessor(tokenizer=__SCREAMING_SNAKE_CASE					,		image_processor=__SCREAMING_SNAKE_CASE)

           __a							=    self.prepare_image_inputs()

           __a							=    image_processor(__SCREAMING_SNAKE_CASE					,		return_tensors='''np''')
           __a							=    processor(images=__SCREAMING_SNAKE_CASE					,		return_tensors='''np''')

           for key in input_image_proc.keys():
               self.assertAlmostEqual(input_image_proc[key].sum()					,		input_processor[key].sum()					,		delta=1E-2)



       def 		_lowerCamelCase      (		self							:					Optional[int]):


           '''simple docstring'''

           __a							=    self.get_image_processor()
           __a							=    self.get_tokenizer()

           __a							=    OwlViTProcessor(tokenizer=__SCREAMING_SNAKE_CASE					,		image_processor=__SCREAMING_SNAKE_CASE)

           __a							=    '''lower newer'''

           __a							=    processor(text=__SCREAMING_SNAKE_CASE					,		return_tensors='''np''')

           __a							=    tokenizer(__SCREAMING_SNAKE_CASE					,		return_tensors='''np''')

           for key in encoded_tok.keys():
               self.assertListEqual(encoded_tok[key][0].tolist()					,		encoded_processor[key][0].tolist())



       def 		_lowerCamelCase      (		self							:					Optional[Any]):


           '''simple docstring'''

           __a							=    self.get_image_processor()
           __a							=    self.get_tokenizer()

           __a							=    OwlViTProcessor(tokenizer=__SCREAMING_SNAKE_CASE					,		image_processor=__SCREAMING_SNAKE_CASE)

           __a							=    '''lower newer'''
           __a							=    self.prepare_image_inputs()

           __a							=    processor(text=__SCREAMING_SNAKE_CASE					,		images=__SCREAMING_SNAKE_CASE)

           self.assertListEqual(list(inputs.keys())					,		['''input_ids''', '''attention_mask''', '''pixel_values'''])

           # test if it raises when no input is passed
           with pytest.raises(__SCREAMING_SNAKE_CASE):
               processor()



       def 		_lowerCamelCase      (		self							:					Any):


           '''simple docstring'''

           __a							=    '''google/owlvit-base-patch32'''
           __a							=    OwlViTProcessor.from_pretrained(__SCREAMING_SNAKE_CASE)

           __a							=    ['''cat''', '''nasa badge''']
           __a							=    processor(text=__SCREAMING_SNAKE_CASE)

           __a							=    16
           self.assertListEqual(list(inputs.keys())					,		['''input_ids''', '''attention_mask'''])
           self.assertEqual(inputs['''input_ids'''].shape					,		(2, seq_length))

           # test if it raises when no input is passed
           with pytest.raises(__SCREAMING_SNAKE_CASE):
               processor()



       def 		_lowerCamelCase      (		self							:					Union[str, Any]):


           '''simple docstring'''

           __a							=    '''google/owlvit-base-patch32'''
           __a							=    OwlViTProcessor.from_pretrained(__SCREAMING_SNAKE_CASE)

           __a							=    [['''cat''', '''nasa badge'''], ['''person''']]
           __a							=    processor(text=__SCREAMING_SNAKE_CASE)

           __a							=    16
           __a							=    len(__SCREAMING_SNAKE_CASE)
           __a							=    max([len(__SCREAMING_SNAKE_CASE) for texts in input_texts])

           self.assertListEqual(list(inputs.keys())					,		['''input_ids''', '''attention_mask'''])
           self.assertEqual(inputs['''input_ids'''].shape					,		(batch_size * num_max_text_queries, seq_length))

           # test if it raises when no input is passed
           with pytest.raises(__SCREAMING_SNAKE_CASE):
               processor()



       def 		_lowerCamelCase      (		self							:					Optional[int]):


           '''simple docstring'''

           __a							=    '''google/owlvit-base-patch32'''
           __a							=    OwlViTProcessor.from_pretrained(__SCREAMING_SNAKE_CASE)

           __a							=    ['''cat''', '''nasa badge''']
           __a							=    processor(text=__SCREAMING_SNAKE_CASE)

           __a							=    16
           __a							=    inputs['''input_ids''']
           __a							=    [
               [49_406, 2_368, 49_407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
               [49_406, 6_841, 11_301, 49_407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
           ]

           self.assertListEqual(list(inputs.keys())					,		['''input_ids''', '''attention_mask'''])
           self.assertEqual(inputs['''input_ids'''].shape					,		(2, seq_length))
           self.assertListEqual(list(input_ids[0])					,		predicted_ids[0])
           self.assertListEqual(list(input_ids[1])					,		predicted_ids[1])



       def 		_lowerCamelCase      (		self							:					List[str]):


           '''simple docstring'''

           __a							=    self.get_image_processor()
           __a							=    self.get_tokenizer()

           __a							=    OwlViTProcessor(tokenizer=__SCREAMING_SNAKE_CASE					,		image_processor=__SCREAMING_SNAKE_CASE)

           __a							=    self.prepare_image_inputs()
           __a							=    self.prepare_image_inputs()

           __a							=    processor(images=__SCREAMING_SNAKE_CASE					,		query_images=__SCREAMING_SNAKE_CASE)

           self.assertListEqual(list(inputs.keys())					,		['''query_pixel_values''', '''pixel_values'''])

           # test if it raises when no input is passed
           with pytest.raises(__SCREAMING_SNAKE_CASE):
               processor()





       def 		_lowerCamelCase      (		self							:					List[Any]):


           '''simple docstring'''

           __a							=    self.get_image_processor()
           __a							=    self.get_tokenizer()

           __a							=    OwlViTProcessor(tokenizer=__SCREAMING_SNAKE_CASE					,		image_processor=__SCREAMING_SNAKE_CASE)

           __a							=    [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

           __a							=    processor.batch_decode(__SCREAMING_SNAKE_CASE)
           __a							=    tokenizer.batch_decode(__SCREAMING_SNAKE_CASE)

           self.assertListEqual(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE)










import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


__snake_case						:Dict						=					logging.get_logger(__name__)

__snake_case						:List[str]						=					{'''vocab_file''': '''spiece.model'''}

__snake_case						:Dict						=					{
    '''vocab_file''': {
        '''TsinghuaAI/CPM-Generate''': '''https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/spiece.model''',
    }
}

class    _A	(     __UpperCAmelCase   ):



       def __init__(		self							:					Union[str, Any]					,		__SCREAMING_SNAKE_CASE							:					Dict					,		__SCREAMING_SNAKE_CASE							:					Optional[Any]=False					,		__SCREAMING_SNAKE_CASE							:					List[Any]=True					,		__SCREAMING_SNAKE_CASE							:					List[Any]=False					,		__SCREAMING_SNAKE_CASE							:					Optional[int]="<s>"					,		__SCREAMING_SNAKE_CASE							:					Optional[Any]="</s>"					,		__SCREAMING_SNAKE_CASE							:					Union[str, Any]="<unk>"					,		__SCREAMING_SNAKE_CASE							:					Optional[int]="<sep>"					,		__SCREAMING_SNAKE_CASE							:					Optional[int]="<pad>"					,		__SCREAMING_SNAKE_CASE							:					List[Any]="<cls>"					,		__SCREAMING_SNAKE_CASE							:					Optional[int]="<mask>"					,		__SCREAMING_SNAKE_CASE							:					Any=["<eop>", "<eod>"]					,		__SCREAMING_SNAKE_CASE							:					Optional[Dict[str, Any]] = None					,		**__SCREAMING_SNAKE_CASE							:					Union[str, Any]					,		):


           '''simple docstring'''

           __a							=    AddedToken(__SCREAMING_SNAKE_CASE					,		lstrip=__SCREAMING_SNAKE_CASE					,		rstrip=__SCREAMING_SNAKE_CASE) if isinstance(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE) else mask_token

           __a							=    {} if sp_model_kwargs is None else sp_model_kwargs

           super().__init__(
               do_lower_case=__SCREAMING_SNAKE_CASE					,		remove_space=__SCREAMING_SNAKE_CASE					,		keep_accents=__SCREAMING_SNAKE_CASE					,		bos_token=__SCREAMING_SNAKE_CASE					,		eos_token=__SCREAMING_SNAKE_CASE					,		unk_token=__SCREAMING_SNAKE_CASE					,		sep_token=__SCREAMING_SNAKE_CASE					,		pad_token=__SCREAMING_SNAKE_CASE					,		cls_token=__SCREAMING_SNAKE_CASE					,		mask_token=__SCREAMING_SNAKE_CASE					,		additional_special_tokens=__SCREAMING_SNAKE_CASE					,		sp_model_kwargs=self.sp_model_kwargs					,		**__SCREAMING_SNAKE_CASE					,		)

           __a							=    3

           __a							=    do_lower_case
           __a							=    remove_space
           __a							=    keep_accents
           __a							=    vocab_file

           __a							=    spm.SentencePieceProcessor(**self.sp_model_kwargs)
           self.sp_model.Load(__SCREAMING_SNAKE_CASE)

           try:
               import jieba
           except ModuleNotFoundError as error:
               raise error.__class__(
                   '''You need to install jieba to use CpmTokenizer or CpmTokenizerFast. '''
                   '''See https://pypi.org/project/jieba/ for installation.''')
           __a							=    jieba
           __a							=    str.maketrans(''' \n'''					,		'''\u2582\u2583''')



       @property
       # Copied from transformers.models.xlnet.tokenization_xlnet.XLNetTokenizer.vocab_size
       def 		_lowerCamelCase      (		self							:					int):


           '''simple docstring'''

           return len(self.sp_model)



       def 		_lowerCamelCase      (		self							:					str):


           '''simple docstring'''

           __a							=    {self.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE): i for i in range(self.vocab_size)}
           vocab.update(self.added_tokens_encoder)
           return vocab



       def __getstate__(		self							:					Any):


           '''simple docstring'''

           __a							=    self.__dict__.copy()
           __a							=    None
           return state



       def __setstate__(		self							:					Tuple					,		__SCREAMING_SNAKE_CASE							:					List[str]):


           '''simple docstring'''

           __a							=    d

           # for backward compatibility
           if not hasattr(self					,		'''sp_model_kwargs'''):
               __a							=    {}

           __a							=    spm.SentencePieceProcessor(**self.sp_model_kwargs)
           self.sp_model.Load(self.vocab_file)



       def 		_lowerCamelCase      (		self							:					Dict					,		__SCREAMING_SNAKE_CASE							:					Any):


           '''simple docstring'''

           if self.remove_space:
               __a							=    ''' '''.join(inputs.strip().split())
           else:
               __a							=    inputs
           __a							=    outputs.replace('''``'''					,		'''"''').replace('''\'\''''					,		'''"''')

           if not self.keep_accents:
               __a							=    unicodedata.normalize('''NFKD'''					,		__SCREAMING_SNAKE_CASE)
               __a							=    ''''''.join([c for c in outputs if not unicodedata.combining(__SCREAMING_SNAKE_CASE)])
           if self.do_lower_case:
               __a							=    outputs.lower()

           return outputs



       def 		_lowerCamelCase      (		self							:					str					,		__SCREAMING_SNAKE_CASE							:					str):


           '''simple docstring'''

           __a							=    self.preprocess_text(__SCREAMING_SNAKE_CASE)
           __a							=    self.sp_model.encode(__SCREAMING_SNAKE_CASE					,		out_type=__SCREAMING_SNAKE_CASE)
           __a							=    []
           for piece in pieces:
               if len(__SCREAMING_SNAKE_CASE) > 1 and piece[-1] == str(''',''') and piece[-2].isdigit():
                   __a							=    self.sp_model.EncodeAsPieces(piece[:-1].replace(__SCREAMING_SNAKE_CASE					,		''''''))
                   if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                       if len(cur_pieces[0]) == 1:
                           __a							=    cur_pieces[1:]
                       else:
                           __a							=    cur_pieces[0][1:]
                   cur_pieces.append(piece[-1])
                   new_pieces.extend(__SCREAMING_SNAKE_CASE)
               else:
                   new_pieces.append(__SCREAMING_SNAKE_CASE)

           return new_pieces



       def 		_lowerCamelCase      (		self							:					str					,		__SCREAMING_SNAKE_CASE							:					List[Any]):


           '''simple docstring'''

           return self.sp_model.PieceToId(__SCREAMING_SNAKE_CASE)



       def 		_lowerCamelCase      (		self							:					Optional[Any]					,		__SCREAMING_SNAKE_CASE							:					Optional[int]):


           '''simple docstring'''

           return self.sp_model.IdToPiece(__SCREAMING_SNAKE_CASE)



       def 		_lowerCamelCase      (		self							:					List[str]					,		__SCREAMING_SNAKE_CASE							:					Tuple):


           '''simple docstring'''

           __a							=    ''''''.join(__SCREAMING_SNAKE_CASE).replace(__SCREAMING_SNAKE_CASE					,		''' ''').strip()
           return out_string



       def 		_lowerCamelCase      (		self							:					Optional[int]					,		__SCREAMING_SNAKE_CASE							:					List[int]					,		__SCREAMING_SNAKE_CASE							:					Optional[List[int]] = None):


           '''simple docstring'''

           __a							=    [self.sep_token_id]
           __a							=    [self.cls_token_id]
           if token_ids_a is None:
               return token_ids_a + sep + cls
           return token_ids_a + sep + token_ids_a + sep + cls



       def 		_lowerCamelCase      (		self							:					Optional[Any]					,		__SCREAMING_SNAKE_CASE							:					List[int]					,		__SCREAMING_SNAKE_CASE							:					Optional[List[int]] = None					,		__SCREAMING_SNAKE_CASE							:					bool = False):


           '''simple docstring'''

           if already_has_special_tokens:
               return super().get_special_tokens_mask(
                   token_ids_a=__SCREAMING_SNAKE_CASE					,		token_ids_a=__SCREAMING_SNAKE_CASE					,		already_has_special_tokens=__SCREAMING_SNAKE_CASE)

           if token_ids_a is not None:
               return ([0] * len(__SCREAMING_SNAKE_CASE)) + [1] + ([0] * len(__SCREAMING_SNAKE_CASE)) + [1, 1]
           return ([0] * len(__SCREAMING_SNAKE_CASE)) + [1, 1]



       def 		_lowerCamelCase      (		self							:					Dict					,		__SCREAMING_SNAKE_CASE							:					List[int]					,		__SCREAMING_SNAKE_CASE							:					Optional[List[int]] = None):


           '''simple docstring'''

           __a							=    [self.sep_token_id]
           __a							=    [2]

           if token_ids_a is None:
               return len(token_ids_a + sep) * [0] + cls_segment_id
           return len(token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1] + cls_segment_id



       def 		_lowerCamelCase      (		self							:					Optional[int]					,		__SCREAMING_SNAKE_CASE							:					str					,		__SCREAMING_SNAKE_CASE							:					Optional[str] = None):


           '''simple docstring'''

           if not os.path.isdir(__SCREAMING_SNAKE_CASE):
               logger.error(F'Vocabulary path ({save_directory}) should be a directory')
               return
           __a							=    os.path.join(
               __SCREAMING_SNAKE_CASE					,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''])

           if os.path.abspath(self.vocab_file) != os.path.abspath(__SCREAMING_SNAKE_CASE) and os.path.isfile(self.vocab_file):
               copyfile(self.vocab_file					,		__SCREAMING_SNAKE_CASE)
           elif not os.path.isfile(self.vocab_file):
               with open(__SCREAMING_SNAKE_CASE					,		'''wb''') as fi:
                   __a							=    self.sp_model.serialized_model_proto()
                   fi.write(__SCREAMING_SNAKE_CASE)

           return (out_vocab_file,)





       def 		_lowerCamelCase      (		self							:					Any					,		*__SCREAMING_SNAKE_CASE							:					List[str]					,		**__SCREAMING_SNAKE_CASE							:					Any):


           '''simple docstring'''

           __a							=    super()._decode(*__SCREAMING_SNAKE_CASE					,		**__SCREAMING_SNAKE_CASE)
           __a							=    text.replace(''' '''					,		'''''').replace('''\u2582'''					,		''' ''').replace('''\u2583'''					,		'''\n''')
           return text





