



import gc
import unittest

import numpy as np
import torch
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import TransformeraDModel, VQDiffusionPipeline, VQDiffusionScheduler, VQModel
from diffusers.pipelines.vq_diffusion.pipeline_vq_diffusion import LearnedClassifierFreeSamplingEmbeddings
from diffusers.utils import load_numpy, slow, torch_device
from diffusers.utils.testing_utils import require_torch_gpu


__snake_case    :		List[str]  =False



class 	lowerCamelCase__      (							unittest.TestCase):




		'''simple docstring'''



		def 		lowerCAmelCase__				(self			)					->      str:

						"""simple docstring"""






						super().tearDown()
						gc.collect()
						torch.cuda.empty_cache()



		@property
		def 		lowerCAmelCase__				(self			)					->      Tuple:

						"""simple docstring"""






						return 12



		@property
		def 		lowerCAmelCase__				(self			)					->      Union[str, Any]:

						"""simple docstring"""






						return 12



		@property
		def 		lowerCAmelCase__				(self			)					->      Tuple:

						"""simple docstring"""






						return 32



		@property
		def 		lowerCAmelCase__				(self			)					->      Optional[Any]:

						"""simple docstring"""






						torch.manual_seed(0			)
						lowerCAmelCase__					:    Any												=					VQModel(
						    block_out_channels=[32, 64]		,in_channels=3		,out_channels=3		,down_block_types=['''DownEncoderBlock2D''', '''DownEncoderBlock2D''']		,up_block_types=['''UpDecoderBlock2D''', '''UpDecoderBlock2D''']		,latent_channels=3		,num_vq_embeddings=self.num_embed		,vq_embed_dim=3		,)
						return model



		@property
		def 		lowerCAmelCase__				(self			)					->      Dict:

						"""simple docstring"""






						lowerCAmelCase__					:    List[Any]												=					CLIPTokenizer.from_pretrained('''hf-internal-testing/tiny-random-clip'''			)
						return tokenizer



		@property
		def 		lowerCAmelCase__				(self			)					->      Optional[int]:

						"""simple docstring"""






						torch.manual_seed(0			)
						lowerCAmelCase__					:    Dict												=					CLIPTextConfig(
						    bos_token_id=0		,eos_token_id=2		,hidden_size=self.text_embedder_hidden_size		,intermediate_size=37		,layer_norm_eps=1e-05		,num_attention_heads=4		,num_hidden_layers=5		,pad_token_id=1		,vocab_size=10_00		,)
						return CLIPTextModel(__lowerCamelCase			)



		@property
		def 		lowerCAmelCase__				(self			)					->      List[Any]:

						"""simple docstring"""






						torch.manual_seed(0			)

						lowerCAmelCase__					:    Optional[Any]												=					12
						lowerCAmelCase__					:    Optional[Any]												=					12

						lowerCAmelCase__					:    Dict												=					{
						    '''attention_bias''': True,
						    '''cross_attention_dim''': 32,
						    '''attention_head_dim''': height * width,
						    '''num_attention_heads''': 1,
						    '''num_vector_embeds''': self.num_embed,
						    '''num_embeds_ada_norm''': self.num_embeds_ada_norm,
						    '''norm_num_groups''': 32,
						    '''sample_size''': width,
						    '''activation_fn''': '''geglu-approximate''',
						}

						lowerCAmelCase__					:    List[str]												=					TransformeraDModel(**__lowerCamelCase			)
						return model



		def 		lowerCAmelCase__				(self			)					->      Union[str, Any]:

						"""simple docstring"""






						lowerCAmelCase__					:    Optional[Any]												=					'''cpu'''

						lowerCAmelCase__					:    str												=					self.dummy_vqvae
						lowerCAmelCase__					:    Dict												=					self.dummy_text_encoder
						lowerCAmelCase__					:    Dict												=					self.dummy_tokenizer
						lowerCAmelCase__					:    List[str]												=					self.dummy_transformer
						lowerCAmelCase__					:    int												=					VQDiffusionScheduler(self.num_embed			)
						lowerCAmelCase__					:    List[str]												=					LearnedClassifierFreeSamplingEmbeddings(learnable=__lowerCamelCase			)

						lowerCAmelCase__					:    List[Any]												=					VQDiffusionPipeline(
						    vqvae=__lowerCamelCase		,text_encoder=__lowerCamelCase		,tokenizer=__lowerCamelCase		,transformer=__lowerCamelCase		,scheduler=__lowerCamelCase		,learned_classifier_free_sampling_embeddings=__lowerCamelCase		,)
						lowerCAmelCase__					:    Tuple												=					pipe.to(__lowerCamelCase			)
						pipe.set_progress_bar_config(disable=__lowerCamelCase			)

						lowerCAmelCase__					:    List[str]												=					'''teddy bear playing in the pool'''

						lowerCAmelCase__					:    Any												=					torch.Generator(device=__lowerCamelCase			).manual_seed(0			)
						lowerCAmelCase__					:    Optional[Any]												=					pipe([prompt]		,generator=__lowerCamelCase		,num_inference_steps=2		,output_type='''np'''			)
						lowerCAmelCase__					:    str												=					output.images

						lowerCAmelCase__					:    str												=					torch.Generator(device=__lowerCamelCase			).manual_seed(0			)
						lowerCAmelCase__					:    Optional[Any]												=					pipe(
						    [prompt]		,generator=__lowerCamelCase		,output_type='''np'''		,return_dict=__lowerCamelCase		,num_inference_steps=2			)[0]

						lowerCAmelCase__					:    Optional[int]												=					image[0, -3:, -3:, -1]
						lowerCAmelCase__					:    Optional[Any]												=					image_from_tuple[0, -3:, -3:, -1]

						assert image.shape == (1, 24, 24, 3)

						lowerCAmelCase__					:    Dict												=					np.array([0.6551, 0.6168, 0.5008, 0.5676, 0.5659, 0.4295, 0.6073, 0.5599, 0.4992]			)

						assert np.abs(image_slice.flatten() - expected_slice			).max() < 1e-2
						assert np.abs(image_from_tuple_slice.flatten() - expected_slice			).max() < 1e-2



		def 		lowerCAmelCase__				(self			)					->      int:

						"""simple docstring"""






						lowerCAmelCase__					:    Dict												=					'''cpu'''

						lowerCAmelCase__					:    List[str]												=					self.dummy_vqvae
						lowerCAmelCase__					:    Dict												=					self.dummy_text_encoder
						lowerCAmelCase__					:    Any												=					self.dummy_tokenizer
						lowerCAmelCase__					:    Dict												=					self.dummy_transformer
						lowerCAmelCase__					:    int												=					VQDiffusionScheduler(self.num_embed			)
						lowerCAmelCase__					:    List[str]												=					LearnedClassifierFreeSamplingEmbeddings(
						    learnable=__lowerCamelCase		,hidden_size=self.text_embedder_hidden_size		,length=tokenizer.model_max_length			)

						lowerCAmelCase__					:    Tuple												=					VQDiffusionPipeline(
						    vqvae=__lowerCamelCase		,text_encoder=__lowerCamelCase		,tokenizer=__lowerCamelCase		,transformer=__lowerCamelCase		,scheduler=__lowerCamelCase		,learned_classifier_free_sampling_embeddings=__lowerCamelCase		,)
						lowerCAmelCase__					:    Optional[Any]												=					pipe.to(__lowerCamelCase			)
						pipe.set_progress_bar_config(disable=__lowerCamelCase			)

						lowerCAmelCase__					:    Optional[int]												=					'''teddy bear playing in the pool'''

						lowerCAmelCase__					:    Dict												=					torch.Generator(device=__lowerCamelCase			).manual_seed(0			)
						lowerCAmelCase__					:    str												=					pipe([prompt]		,generator=__lowerCamelCase		,num_inference_steps=2		,output_type='''np'''			)
						lowerCAmelCase__					:    Optional[Any]												=					output.images

						lowerCAmelCase__					:    int												=					torch.Generator(device=__lowerCamelCase			).manual_seed(0			)
						lowerCAmelCase__					:    Tuple												=					pipe(
						    [prompt]		,generator=__lowerCamelCase		,output_type='''np'''		,return_dict=__lowerCamelCase		,num_inference_steps=2			)[0]

						lowerCAmelCase__					:    Tuple												=					image[0, -3:, -3:, -1]
						lowerCAmelCase__					:    Union[str, Any]												=					image_from_tuple[0, -3:, -3:, -1]

						assert image.shape == (1, 24, 24, 3)

						lowerCAmelCase__					:    Tuple												=					np.array([0.6693, 0.6075, 0.4959, 0.5701, 0.5583, 0.4333, 0.6171, 0.5684, 0.4988]			)

						assert np.abs(image_slice.flatten() - expected_slice			).max() < 2.0
						assert np.abs(image_from_tuple_slice.flatten() - expected_slice			).max() < 1e-2



@slow
@require_torch_gpu
class 	lowerCamelCase__      (							unittest.TestCase):




		'''simple docstring'''



		def 		lowerCAmelCase__				(self			)					->      int:

						"""simple docstring"""






						super().tearDown()
						gc.collect()
						torch.cuda.empty_cache()



		def 		lowerCAmelCase__				(self			)					->      Union[str, Any]:

						"""simple docstring"""






						lowerCAmelCase__					:    Union[str, Any]												=					load_numpy(
						    '''https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main'''
						    '''/vq_diffusion/teddy_bear_pool_classifier_free_sampling.npy'''			)

						lowerCAmelCase__					:    List[Any]												=					VQDiffusionPipeline.from_pretrained('''microsoft/vq-diffusion-ithq'''			)
						lowerCAmelCase__					:    str												=					pipeline.to(__lowerCamelCase			)
						pipeline.set_progress_bar_config(disable=__lowerCamelCase			)

						# requires GPU generator for gumbel softmax
						# don't use GPU generator in tests though
						lowerCAmelCase__					:    Tuple												=					torch.Generator(device=__lowerCamelCase			).manual_seed(0			)
						lowerCAmelCase__					:    int												=					pipeline(
						    '''teddy bear playing in the pool'''		,num_images_per_prompt=1		,generator=__lowerCamelCase		,output_type='''np'''		,)

						lowerCAmelCase__					:    List[Any]												=					output.images[0]

						assert image.shape == (2_56, 2_56, 3)
						assert np.abs(expected_image - image			).max() < 2.0






import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


__snake_case    :		int  =get_tests_dir('fixtures/test_sentencepiece.model')



@require_sentencepiece
class 	lowerCamelCase__      (							lowerCamelCase__				,    unittest.TestCase):




		'''simple docstring'''



		snake_case_     =XLMProphetNetTokenizer
		snake_case_     =False
		snake_case_     =True



		def 		lowerCAmelCase__				(self			)					->      Tuple:

						"""simple docstring"""






						super().setUp()

						# We have a SentencePiece fixture for testing
						lowerCAmelCase__					:    int												=					XLMProphetNetTokenizer(__lowerCamelCase		,keep_accents=__lowerCamelCase			)
						tokenizer.save_pretrained(self.tmpdirname			)



		def 		lowerCAmelCase__				(self			)					->      Union[str, Any]:

						"""simple docstring"""






						lowerCAmelCase__					:    str												=					'''[PAD]'''
						lowerCAmelCase__					:    Tuple												=					0

						self.assertEqual(self.get_tokenizer()._convert_token_to_id(__lowerCamelCase			)		,__lowerCamelCase			)
						self.assertEqual(self.get_tokenizer()._convert_id_to_token(__lowerCamelCase			)		,__lowerCamelCase			)



		def 		lowerCAmelCase__				(self			)					->      Optional[Any]:

						"""simple docstring"""






						lowerCAmelCase__					:    Any												=					list(self.get_tokenizer().get_vocab().keys()			)

						self.assertEqual(vocab_keys[0]		,'''[PAD]'''			)
						self.assertEqual(vocab_keys[1]		,'''[CLS]'''			)
						self.assertEqual(vocab_keys[-1]		,'''j'''			)
						self.assertEqual(len(__lowerCamelCase			)		,10_12			)



		def 		lowerCAmelCase__				(self			)					->      Union[str, Any]:

						"""simple docstring"""






						self.assertEqual(self.get_tokenizer().vocab_size		,10_12			)



		def 		lowerCAmelCase__				(self			)					->      Tuple:

						"""simple docstring"""






						lowerCAmelCase__					:    Dict												=					XLMProphetNetTokenizer(__lowerCamelCase		,keep_accents=__lowerCamelCase			)

						lowerCAmelCase__					:    Tuple												=					tokenizer.tokenize('''This is a test'''			)
						self.assertListEqual(__lowerCamelCase		,['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']			)

						self.assertListEqual(
						    tokenizer.convert_tokens_to_ids(__lowerCamelCase			)		,[value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]		,)

						lowerCAmelCase__					:    Dict												=					tokenizer.tokenize('''I was born in 92000, and this is falsé.'''			)
						self.assertListEqual(
						    __lowerCamelCase		,[
						        SPIECE_UNDERLINE + '''I''',
						        SPIECE_UNDERLINE + '''was''',
						        SPIECE_UNDERLINE + '''b''',
						        '''or''',
						        '''n''',
						        SPIECE_UNDERLINE + '''in''',
						        SPIECE_UNDERLINE + '''''',
						        '''9''',
						        '''2''',
						        '''0''',
						        '''0''',
						        '''0''',
						        ''',''',
						        SPIECE_UNDERLINE + '''and''',
						        SPIECE_UNDERLINE + '''this''',
						        SPIECE_UNDERLINE + '''is''',
						        SPIECE_UNDERLINE + '''f''',
						        '''al''',
						        '''s''',
						        '''é''',
						        '''.''',
						    ]		,)
						lowerCAmelCase__					:    Optional[Any]												=					tokenizer.convert_tokens_to_ids(__lowerCamelCase			)
						self.assertListEqual(
						    __lowerCamelCase		,[
						        value + tokenizer.fairseq_offset
						        for value in [8, 21, 84, 55, 24, 19, 7, -9, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, -9, 4]
						    ]		,)

						lowerCAmelCase__					:    Optional[Any]												=					tokenizer.convert_ids_to_tokens(__lowerCamelCase			)
						self.assertListEqual(
						    __lowerCamelCase		,[
						        SPIECE_UNDERLINE + '''I''',
						        SPIECE_UNDERLINE + '''was''',
						        SPIECE_UNDERLINE + '''b''',
						        '''or''',
						        '''n''',
						        SPIECE_UNDERLINE + '''in''',
						        SPIECE_UNDERLINE + '''''',
						        '''[UNK]''',
						        '''2''',
						        '''0''',
						        '''0''',
						        '''0''',
						        ''',''',
						        SPIECE_UNDERLINE + '''and''',
						        SPIECE_UNDERLINE + '''this''',
						        SPIECE_UNDERLINE + '''is''',
						        SPIECE_UNDERLINE + '''f''',
						        '''al''',
						        '''s''',
						        '''[UNK]''',
						        '''.''',
						    ]		,)



		@cached_property
		def 		lowerCAmelCase__				(self			)					->      List[str]:

						"""simple docstring"""






						return XLMProphetNetTokenizer.from_pretrained('''microsoft/xprophetnet-large-wiki100-cased'''			)



		@slow
		def 		lowerCAmelCase__				(self			)					->      Tuple:

						"""simple docstring"""






						lowerCAmelCase__					:    Optional[int]												=					'''Hello World!'''
						lowerCAmelCase__					:    str												=					[3_53_89, 66_72, 49, 2]
						self.assertListEqual(__lowerCamelCase		,self.big_tokenizer.encode(__lowerCamelCase			)			)



		@slow
		def 		lowerCAmelCase__				(self			)					->      List[str]:

						"""simple docstring"""






						lowerCAmelCase__					:    Any												=					{'''input_ids''': [[1_10_73, 8_27_83, 18, 26, 8_27_83, 5_49, 5_15_40, 2_48, 1_72_09, 13_01, 2_17, 20, 21_51_86, 13_25, 1_47, 1_72_09, 13_01, 2_17, 20, 5_63_70, 53, 12_20_20, 20, 1_64_77, 27, 8_73_55, 45_48, 20, 47_28, 7_83_92, 17, 15_99_69, 18, 26, 2_44_91, 6_29, 15, 5_38, 2_27_04, 54_39, 15, 27_88, 2_44_91, 98_85, 15, 4_35_34, 6_05, 15, 8_14, 1_84_03, 3_32_00, 29, 15, 4_35_34, 2_44_58, 1_24_10, 1_11, 2_49_66, 8_36_69, 96_37, 14_40_68, 26, 8_50, 2_23_46, 27, 1_47, 2_49_66, 8_36_69, 8_34_90, 26, 3_91_13, 7_35, 27, 6_89, 6_56, 28_00, 13_39, 46_00, 53, 12_20_20, 11_57_85, 34, 8_16, 13_39, 4_68_87, 18, 1_47, 5_39_05, 19_51, 4_22_38, 4_11_70, 1_77_32, 8_34, 4_36, 15, 2_75_23, 9_87_33, 2_17, 1_47, 55_42, 49_81, 9_30, 1_73_47, 16, 2], [2_00_91, 6_29, 94, 8_27_86, 58, 4_90, 20, 15_28, 84, 5_39_05, 3_44, 8_05_92, 11_01_28, 1_88_22, 52_67, 13_06, 62, 15_25_37, 3_08, 79_97, 4_01, 12_44_27, 5_49, 3_54_42, 2_25, 1_09, 1_50_55, 2_57_48, 1_47, 71_19, 4_37_12, 34, 7_67, 13_53_66, 18, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_92, 6_37_84, 11_94_66, 17, 14_78_08, 8_82_14, 18, 6_56, 81, 32, 32_96, 1_02_80, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
						# fmt: on

						self.tokenizer_integration_test_util(
						    expected_encoding=__lowerCamelCase		,model_name='''microsoft/xprophetnet-large-wiki100-cased'''		,revision='''1acad1643ddd54a44df6a1b797ada8373685d90e'''		,)

