



import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


__snake_case						:Dict						=					logging.get_logger(__name__)

__snake_case						:List[str]						=					{'''vocab_file''': '''spiece.model'''}

__snake_case						:Dict						=					{
    '''vocab_file''': {
        '''TsinghuaAI/CPM-Generate''': '''https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/spiece.model''',
    }
}

class    _A	(     __UpperCAmelCase   ):



							def __init__(		self							:					Union[str, Any]					,		__SCREAMING_SNAKE_CASE							:					Dict					,		__SCREAMING_SNAKE_CASE							:					Optional[Any]=False					,		__SCREAMING_SNAKE_CASE							:					List[Any]=True					,		__SCREAMING_SNAKE_CASE							:					List[Any]=False					,		__SCREAMING_SNAKE_CASE							:					Optional[int]="<s>"					,		__SCREAMING_SNAKE_CASE							:					Optional[Any]="</s>"					,		__SCREAMING_SNAKE_CASE							:					Union[str, Any]="<unk>"					,		__SCREAMING_SNAKE_CASE							:					Optional[int]="<sep>"					,		__SCREAMING_SNAKE_CASE							:					Optional[int]="<pad>"					,		__SCREAMING_SNAKE_CASE							:					List[Any]="<cls>"					,		__SCREAMING_SNAKE_CASE							:					Optional[int]="<mask>"					,		__SCREAMING_SNAKE_CASE							:					Any=["<eop>", "<eod>"]					,		__SCREAMING_SNAKE_CASE							:					Optional[Dict[str, Any]] = None					,		**__SCREAMING_SNAKE_CASE							:					Union[str, Any]					,		):


											'''simple docstring'''

											__a							=    AddedToken(__SCREAMING_SNAKE_CASE					,		lstrip=__SCREAMING_SNAKE_CASE					,		rstrip=__SCREAMING_SNAKE_CASE) if isinstance(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE) else mask_token

											__a							=    {} if sp_model_kwargs is None else sp_model_kwargs

											super().__init__(
											    do_lower_case=__SCREAMING_SNAKE_CASE					,		remove_space=__SCREAMING_SNAKE_CASE					,		keep_accents=__SCREAMING_SNAKE_CASE					,		bos_token=__SCREAMING_SNAKE_CASE					,		eos_token=__SCREAMING_SNAKE_CASE					,		unk_token=__SCREAMING_SNAKE_CASE					,		sep_token=__SCREAMING_SNAKE_CASE					,		pad_token=__SCREAMING_SNAKE_CASE					,		cls_token=__SCREAMING_SNAKE_CASE					,		mask_token=__SCREAMING_SNAKE_CASE					,		additional_special_tokens=__SCREAMING_SNAKE_CASE					,		sp_model_kwargs=self.sp_model_kwargs					,		**__SCREAMING_SNAKE_CASE					,		)

											__a							=    3

											__a							=    do_lower_case
											__a							=    remove_space
											__a							=    keep_accents
											__a							=    vocab_file

											__a							=    spm.SentencePieceProcessor(**self.sp_model_kwargs)
											self.sp_model.Load(__SCREAMING_SNAKE_CASE)

											try:
															import jieba
											except ModuleNotFoundError as error:
															raise error.__class__(
															    '''You need to install jieba to use CpmTokenizer or CpmTokenizerFast. '''
															    '''See https://pypi.org/project/jieba/ for installation.''')
											__a							=    jieba
											__a							=    str.maketrans(''' \n'''					,		'''\u2582\u2583''')



							@property
							# Copied from transformers.models.xlnet.tokenization_xlnet.XLNetTokenizer.vocab_size
							def 		_lowerCamelCase      (		self							:					int):


											'''simple docstring'''

											return len(self.sp_model)



							def 		_lowerCamelCase      (		self							:					str):


											'''simple docstring'''

											__a							=    {self.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE): i for i in range(self.vocab_size)}
											vocab.update(self.added_tokens_encoder)
											return vocab



							def __getstate__(		self							:					Any):


											'''simple docstring'''

											__a							=    self.__dict__.copy()
											__a							=    None
											return state



							def __setstate__(		self							:					Tuple					,		__SCREAMING_SNAKE_CASE							:					List[str]):


											'''simple docstring'''

											__a							=    d

											# for backward compatibility
											if not hasattr(self					,		'''sp_model_kwargs'''):
															__a							=    {}

											__a							=    spm.SentencePieceProcessor(**self.sp_model_kwargs)
											self.sp_model.Load(self.vocab_file)



							def 		_lowerCamelCase      (		self							:					Dict					,		__SCREAMING_SNAKE_CASE							:					Any):


											'''simple docstring'''

											if self.remove_space:
															__a							=    ''' '''.join(inputs.strip().split())
											else:
															__a							=    inputs
											__a							=    outputs.replace('''``'''					,		'''"''').replace('''\'\''''					,		'''"''')

											if not self.keep_accents:
															__a							=    unicodedata.normalize('''NFKD'''					,		__SCREAMING_SNAKE_CASE)
															__a							=    ''''''.join([c for c in outputs if not unicodedata.combining(__SCREAMING_SNAKE_CASE)])
											if self.do_lower_case:
															__a							=    outputs.lower()

											return outputs



							def 		_lowerCamelCase      (		self							:					str					,		__SCREAMING_SNAKE_CASE							:					str):


											'''simple docstring'''

											__a							=    self.preprocess_text(__SCREAMING_SNAKE_CASE)
											__a							=    self.sp_model.encode(__SCREAMING_SNAKE_CASE					,		out_type=__SCREAMING_SNAKE_CASE)
											__a							=    []
											for piece in pieces:
															if len(__SCREAMING_SNAKE_CASE) > 1 and piece[-1] == str(''',''') and piece[-2].isdigit():
																			__a							=    self.sp_model.EncodeAsPieces(piece[:-1].replace(__SCREAMING_SNAKE_CASE					,		''''''))
																			if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																							if len(cur_pieces[0]) == 1:
																											__a							=    cur_pieces[1:]
																							else:
																											__a							=    cur_pieces[0][1:]
																			cur_pieces.append(piece[-1])
																			new_pieces.extend(__SCREAMING_SNAKE_CASE)
															else:
																			new_pieces.append(__SCREAMING_SNAKE_CASE)

											return new_pieces



							def 		_lowerCamelCase      (		self							:					str					,		__SCREAMING_SNAKE_CASE							:					List[Any]):


											'''simple docstring'''

											return self.sp_model.PieceToId(__SCREAMING_SNAKE_CASE)



							def 		_lowerCamelCase      (		self							:					Optional[Any]					,		__SCREAMING_SNAKE_CASE							:					Optional[int]):


											'''simple docstring'''

											return self.sp_model.IdToPiece(__SCREAMING_SNAKE_CASE)



							def 		_lowerCamelCase      (		self							:					List[str]					,		__SCREAMING_SNAKE_CASE							:					Tuple):


											'''simple docstring'''

											__a							=    ''''''.join(__SCREAMING_SNAKE_CASE).replace(__SCREAMING_SNAKE_CASE					,		''' ''').strip()
											return out_string



							def 		_lowerCamelCase      (		self							:					Optional[int]					,		__SCREAMING_SNAKE_CASE							:					List[int]					,		__SCREAMING_SNAKE_CASE							:					Optional[List[int]] = None):


											'''simple docstring'''

											__a							=    [self.sep_token_id]
											__a							=    [self.cls_token_id]
											if token_ids_a is None:
															return token_ids_a + sep + cls
											return token_ids_a + sep + token_ids_a + sep + cls



							def 		_lowerCamelCase      (		self							:					Optional[Any]					,		__SCREAMING_SNAKE_CASE							:					List[int]					,		__SCREAMING_SNAKE_CASE							:					Optional[List[int]] = None					,		__SCREAMING_SNAKE_CASE							:					bool = False):


											'''simple docstring'''

											if already_has_special_tokens:
															return super().get_special_tokens_mask(
															    token_ids_a=__SCREAMING_SNAKE_CASE					,		token_ids_a=__SCREAMING_SNAKE_CASE					,		already_has_special_tokens=__SCREAMING_SNAKE_CASE)

											if token_ids_a is not None:
															return ([0] * len(__SCREAMING_SNAKE_CASE)) + [1] + ([0] * len(__SCREAMING_SNAKE_CASE)) + [1, 1]
											return ([0] * len(__SCREAMING_SNAKE_CASE)) + [1, 1]



							def 		_lowerCamelCase      (		self							:					Dict					,		__SCREAMING_SNAKE_CASE							:					List[int]					,		__SCREAMING_SNAKE_CASE							:					Optional[List[int]] = None):


											'''simple docstring'''

											__a							=    [self.sep_token_id]
											__a							=    [2]

											if token_ids_a is None:
															return len(token_ids_a + sep) * [0] + cls_segment_id
											return len(token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1] + cls_segment_id



							def 		_lowerCamelCase      (		self							:					Optional[int]					,		__SCREAMING_SNAKE_CASE							:					str					,		__SCREAMING_SNAKE_CASE							:					Optional[str] = None):


											'''simple docstring'''

											if not os.path.isdir(__SCREAMING_SNAKE_CASE):
															logger.error(F'Vocabulary path ({save_directory}) should be a directory')
															return
											__a							=    os.path.join(
											    __SCREAMING_SNAKE_CASE					,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''])

											if os.path.abspath(self.vocab_file) != os.path.abspath(__SCREAMING_SNAKE_CASE) and os.path.isfile(self.vocab_file):
															copyfile(self.vocab_file					,		__SCREAMING_SNAKE_CASE)
											elif not os.path.isfile(self.vocab_file):
															with open(__SCREAMING_SNAKE_CASE					,		'''wb''') as fi:
																			__a							=    self.sp_model.serialized_model_proto()
																			fi.write(__SCREAMING_SNAKE_CASE)

											return (out_vocab_file,)





							def 		_lowerCamelCase      (		self							:					Any					,		*__SCREAMING_SNAKE_CASE							:					List[str]					,		**__SCREAMING_SNAKE_CASE							:					Any):


											'''simple docstring'''

											__a							=    super()._decode(*__SCREAMING_SNAKE_CASE					,		**__SCREAMING_SNAKE_CASE)
											__a							=    text.replace(''' '''					,		'''''').replace('''\u2582'''					,		''' ''').replace('''\u2583'''					,		'''\n''')
											return text










import gc
import random
import unittest

import numpy as np
import torch
from PIL import Image

from diffusers import (
    DDIMScheduler,
    KandinskyVaaImgaImgPipeline,
    KandinskyVaaPriorPipeline,
    UNetaDConditionModel,
    VQModel,
)
from diffusers.utils import floats_tensor, load_image, load_numpy, slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu

from ..test_pipelines_common import PipelineTesterMixin, assert_mean_pixel_difference


enable_full_determinism()

class    _A	(     __UpperCAmelCase				,unittest.TestCase   ):
       UpperCamelCase__  :   str						 =						KandinskyVaaImgaImgPipeline
       UpperCamelCase__  :   Optional[Any]						 =						['''image_embeds''', '''negative_image_embeds''', '''image''']
       UpperCamelCase__  :   Dict						 =						[
           '''image_embeds''',
           '''negative_image_embeds''',
           '''image''',
       ]
       UpperCamelCase__  :   Any						 =						[
           '''generator''',
           '''height''',
           '''width''',
           '''strength''',
           '''guidance_scale''',
           '''num_inference_steps''',
           '''return_dict''',
           '''guidance_scale''',
           '''num_images_per_prompt''',
           '''output_type''',
           '''return_dict''',
       ]
       UpperCamelCase__  :   List[Any]						 =						False



       @property
       def 		_lowerCamelCase      (		self							:					Optional[int]):


           '''simple docstring'''

           return 32



       @property
       def 		_lowerCamelCase      (		self							:					List[str]):


           '''simple docstring'''

           return 32



       @property
       def 		_lowerCamelCase      (		self							:					List[Any]):


           '''simple docstring'''

           return self.time_input_dim



       @property
       def 		_lowerCamelCase      (		self							:					int):


           '''simple docstring'''

           return self.time_input_dim * 4



       @property
       def 		_lowerCamelCase      (		self							:					Optional[Any]):


           '''simple docstring'''

           return 100



       @property
       def 		_lowerCamelCase      (		self							:					Any):


           '''simple docstring'''

           torch.manual_seed(0)

           __a							=    {
               '''in_channels''': 4,
               # Out channels is double in channels because predicts mean and variance
               '''out_channels''': 8,
               '''addition_embed_type''': '''image''',
               '''down_block_types''': ('''ResnetDownsampleBlock2D''', '''SimpleCrossAttnDownBlock2D'''),
               '''up_block_types''': ('''SimpleCrossAttnUpBlock2D''', '''ResnetUpsampleBlock2D'''),
               '''mid_block_type''': '''UNetMidBlock2DSimpleCrossAttn''',
               '''block_out_channels''': (self.block_out_channels_a, self.block_out_channels_a * 2),
               '''layers_per_block''': 1,
               '''encoder_hid_dim''': self.text_embedder_hidden_size,
               '''encoder_hid_dim_type''': '''image_proj''',
               '''cross_attention_dim''': self.cross_attention_dim,
               '''attention_head_dim''': 4,
               '''resnet_time_scale_shift''': '''scale_shift''',
               '''class_embed_type''': None,
           }

           __a							=    UNetaDConditionModel(**__SCREAMING_SNAKE_CASE)
           return model



       @property
       def 		_lowerCamelCase      (		self							:					Tuple):


           '''simple docstring'''

           return {
               "block_out_channels": [32, 64],
               "down_block_types": ["DownEncoderBlock2D", "AttnDownEncoderBlock2D"],
               "in_channels": 3,
               "latent_channels": 4,
               "layers_per_block": 1,
               "norm_num_groups": 8,
               "norm_type": "spatial",
               "num_vq_embeddings": 12,
               "out_channels": 3,
               "up_block_types": [
                   "AttnUpDecoderBlock2D",
                   "UpDecoderBlock2D",
               ],
               "vq_embed_dim": 4,
           }



       @property
       def 		_lowerCamelCase      (		self							:					Optional[int]):


           '''simple docstring'''

           torch.manual_seed(0)
           __a							=    VQModel(**self.dummy_movq_kwargs)
           return model



       def 		_lowerCamelCase      (		self							:					Optional[int]):


           '''simple docstring'''

           __a							=    self.dummy_unet
           __a							=    self.dummy_movq

           __a							=    {
               '''num_train_timesteps''': 1_000,
               '''beta_schedule''': '''linear''',
               '''beta_start''': 0.0_00_85,
               '''beta_end''': 0.0_12,
               '''clip_sample''': False,
               '''set_alpha_to_one''': False,
               '''steps_offset''': 0,
               '''prediction_type''': '''epsilon''',
               '''thresholding''': False,
           }

           __a							=    DDIMScheduler(**__SCREAMING_SNAKE_CASE)

           __a							=    {
               '''unet''': unet,
               '''scheduler''': scheduler,
               '''movq''': movq,
           }

           return components



       def 		_lowerCamelCase      (		self							:					Union[str, Any]					,		__SCREAMING_SNAKE_CASE							:					Optional[Any]					,		__SCREAMING_SNAKE_CASE							:					Union[str, Any]=0):


           '''simple docstring'''

           __a							=    floats_tensor((1, self.text_embedder_hidden_size)					,		rng=random.Random(__SCREAMING_SNAKE_CASE)).to(__SCREAMING_SNAKE_CASE)
           __a							=    floats_tensor((1, self.text_embedder_hidden_size)					,		rng=random.Random(seed + 1)).to(
               __SCREAMING_SNAKE_CASE)
           # create init_image
           __a							=    floats_tensor((1, 3, 64, 64)					,		rng=random.Random(__SCREAMING_SNAKE_CASE)).to(__SCREAMING_SNAKE_CASE)
           __a							=    image.cpu().permute(0					,		2					,		3					,		1)[0]
           __a							=    Image.fromarray(np.uinta(__SCREAMING_SNAKE_CASE)).convert('''RGB''').resize((256, 256))

           if str(__SCREAMING_SNAKE_CASE).startswith('''mps'''):
               __a							=    torch.manual_seed(__SCREAMING_SNAKE_CASE)
           else:
               __a							=    torch.Generator(device=__SCREAMING_SNAKE_CASE).manual_seed(__SCREAMING_SNAKE_CASE)
           __a							=    {
               '''image''': init_image,
               '''image_embeds''': image_embeds,
               '''negative_image_embeds''': negative_image_embeds,
               '''generator''': generator,
               '''height''': 64,
               '''width''': 64,
               '''num_inference_steps''': 10,
               '''guidance_scale''': 7.0,
               '''strength''': 0.2,
               '''output_type''': '''np''',
           }
           return inputs





       def 		_lowerCamelCase      (		self							:					Dict):


           '''simple docstring'''

           __a							=    '''cpu'''

           __a							=    self.get_dummy_components()

           __a							=    self.pipeline_class(**__SCREAMING_SNAKE_CASE)
           __a							=    pipe.to(__SCREAMING_SNAKE_CASE)

           pipe.set_progress_bar_config(disable=__SCREAMING_SNAKE_CASE)

           __a							=    pipe(**self.get_dummy_inputs(__SCREAMING_SNAKE_CASE))
           __a							=    output.images

           __a							=    pipe(
               **self.get_dummy_inputs(__SCREAMING_SNAKE_CASE)					,		return_dict=__SCREAMING_SNAKE_CASE					,		)[0]

           __a							=    image[0, -3:, -3:, -1]
           __a							=    image_from_tuple[0, -3:, -3:, -1]

           assert image.shape == (1, 64, 64, 3)

           __a							=    np.array(
               [0.6_19_97_78, 0.63_98_44_06, 0.46_14_57_85, 0.62_94_49_84, 0.5_62_22_15, 0.47_30_61_32, 0.47_44_14_56, 0.4_60_76_06, 0.48_71_92_63])
           assert (
               np.abs(image_slice.flatten() - expected_slice).max() < 1E-2
           ), F' expected_slice {expected_slice}, but got {image_slice.flatten()}'
           assert (
               np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1E-2
           ), F' expected_slice {expected_slice}, but got {image_from_tuple_slice.flatten()}'


@slow
@require_torch_gpu
class    _A	(     unittest.TestCase   ):



       def 		_lowerCamelCase      (		self							:					Dict):


           '''simple docstring'''

           super().tearDown()
           gc.collect()
           torch.cuda.empty_cache()





       def 		_lowerCamelCase      (		self							:					str):


           '''simple docstring'''

           __a							=    load_numpy(
               '''https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main'''
               '''/kandinskyv22/kandinskyv22_img2img_frog.npy''')

           __a							=    load_image(
               '''https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main''' '''/kandinsky/cat.png''')
           __a							=    '''A red cartoon frog, 4k'''

           __a							=    KandinskyVaaPriorPipeline.from_pretrained(
               '''kandinsky-community/kandinsky-2-2-prior'''					,		torch_dtype=torch.floataa)
           pipe_prior.to(__SCREAMING_SNAKE_CASE)

           __a							=    KandinskyVaaImgaImgPipeline.from_pretrained(
               '''kandinsky-community/kandinsky-2-2-decoder'''					,		torch_dtype=torch.floataa)
           __a							=    pipeline.to(__SCREAMING_SNAKE_CASE)

           pipeline.set_progress_bar_config(disable=__SCREAMING_SNAKE_CASE)

           __a							=    torch.Generator(device='''cpu''').manual_seed(0)
           __a	,       __a							=    pipe_prior(
               __SCREAMING_SNAKE_CASE					,		generator=__SCREAMING_SNAKE_CASE					,		num_inference_steps=5					,		negative_prompt=''''''					,		).to_tuple()

           __a							=    pipeline(
               image=__SCREAMING_SNAKE_CASE					,		image_embeds=__SCREAMING_SNAKE_CASE					,		negative_image_embeds=__SCREAMING_SNAKE_CASE					,		generator=__SCREAMING_SNAKE_CASE					,		num_inference_steps=100					,		height=768					,		width=768					,		strength=0.2					,		output_type='''np'''					,		)

           __a							=    output.images[0]

           assert image.shape == (768, 768, 3)

           assert_mean_pixel_difference(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE)





