



'''simple docstring'''




import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class      _lowerCAmelCase							(      __UpperCAmelCase				,      unittest.TestCase			):
	__SCREAMING_SNAKE_CASE       :		str		      = CodeGenTokenizer
	__SCREAMING_SNAKE_CASE       :		Optional[int]		      = CodeGenTokenizerFast
	__SCREAMING_SNAKE_CASE       :		int		      = True
	__SCREAMING_SNAKE_CASE       :		Optional[Any]		      = {'add_prefix_space': True}
	__SCREAMING_SNAKE_CASE       :		Union[str, Any]		      = False


	def    _a     (self							):
				super().setUp()

				# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
				A_     :						List[str]     =					[
				    """l""",
				    """o""",
				    """w""",
				    """e""",
				    """r""",
				    """s""",
				    """t""",
				    """i""",
				    """d""",
				    """n""",
				    """\u0120""",
				    """\u0120l""",
				    """\u0120n""",
				    """\u0120lo""",
				    """\u0120low""",
				    """er""",
				    """\u0120lowest""",
				    """\u0120newer""",
				    """\u0120wider""",
				    """<unk>""",
				    """<|endoftext|>""",
				]
				A_     :						int     =					dict(zip(lowercase	,				range(len(lowercase							)							)							)							)
				A_     :						Any     =					["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
				A_     :						Any     =					{"""unk_token""": """<unk>"""}

				A_     :						Union[str, Any]     =					os.path.join(self.tmpdirname	,				VOCAB_FILES_NAMES["""vocab_file"""]							)
				A_     :						int     =					os.path.join(self.tmpdirname	,				VOCAB_FILES_NAMES["""merges_file"""]							)
				with open(self.vocab_file	,				"""w"""	,				encoding="""utf-8"""							) as fp:
							fp.write(json.dumps(lowercase							) + """\n"""							)
				with open(self.merges_file	,				"""w"""	,				encoding="""utf-8"""							) as fp:
							fp.write("""\n""".join(lowercase							)							)


	def    _a     (self	,				**lowercase							):
				kwargs.update(self.special_tokens_map							)
				return CodeGenTokenizer.from_pretrained(self.tmpdirname	,				**lowercase							)


	def    _a     (self	,				**lowercase							):
				kwargs.update(self.special_tokens_map							)
				return CodeGenTokenizerFast.from_pretrained(self.tmpdirname	,				**lowercase							)


	def    _a     (self	,				lowercase							):
				A_     :						Optional[Any]     =					"""lower newer"""
				A_     :						str     =					"""lower newer"""
				return input_text, output_text


	def    _a     (self							):
				A_     :						int     =					CodeGenTokenizer(self.vocab_file	,				self.merges_file	,				**self.special_tokens_map							)
				A_     :						Union[str, Any]     =					"""lower newer"""
				A_     :						Optional[int]     =					["""\u0120low""", """er""", """\u0120""", """n""", """e""", """w""", """er"""]
				A_     :						Dict     =					tokenizer.tokenize(lowercase	,				add_prefix_space=lowercase							)
				self.assertListEqual(lowercase	,				lowercase							)

				A_     :						Optional[Any]     =					tokens + [tokenizer.unk_token]
				A_     :						Tuple     =					[14, 15, 10, 9, 3, 2, 15, 19]
				self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase							)	,				lowercase							)


	def    _a     (self							):
				if not self.test_rust_tokenizer:
							return

				A_     :						Any     =					self.get_tokenizer()
				A_     :						Tuple     =					self.get_rust_tokenizer(add_prefix_space=lowercase							)

				A_     :						List[str]     =					"""lower newer"""

				# Testing tokenization
				A_     :						Union[str, Any]     =					tokenizer.tokenize(lowercase	,				add_prefix_space=lowercase							)
				A_     :						Optional[Any]     =					rust_tokenizer.tokenize(lowercase							)
				self.assertListEqual(lowercase	,				lowercase							)

				# Testing conversion to ids without special tokens
				A_     :						Dict     =					tokenizer.encode(lowercase	,				add_special_tokens=lowercase	,				add_prefix_space=lowercase							)
				A_     :						Tuple     =					rust_tokenizer.encode(lowercase	,				add_special_tokens=lowercase							)
				self.assertListEqual(lowercase	,				lowercase							)

				# Testing conversion to ids with special tokens
				A_     :						Optional[Any]     =					self.get_rust_tokenizer(add_prefix_space=lowercase							)
				A_     :						str     =					tokenizer.encode(lowercase	,				add_prefix_space=lowercase							)
				A_     :						Optional[Any]     =					rust_tokenizer.encode(lowercase							)
				self.assertListEqual(lowercase	,				lowercase							)

				# Testing the unknown token
				A_     :						Dict     =					tokens + [rust_tokenizer.unk_token]
				A_     :						Optional[int]     =					[14, 15, 10, 9, 3, 2, 15, 19]
				self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(lowercase							)	,				lowercase							)


	def    _a     (self	,				*lowercase	,				**lowercase							):
				# It's very difficult to mix/test pretokenization with byte-level
				# And get both CodeGen and Roberta to work at the same time (mostly an issue of adding a space before the string)
				pass


	def    _a     (self	,				lowercase=15							):
				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'							):
										A_     :						str     =					self.rust_tokenizer_class.from_pretrained(lowercase	,				**lowercase							)

										# Simple input
										A_     :						Optional[Any]     =					"""This is a simple input"""
										A_     :						Optional[Any]     =					["""This is a simple input 1""", """This is a simple input 2"""]
										A_     :						List[str]     =					("""This is a simple input""", """This is a pair""")
										A_     :						Union[str, Any]     =					[
										    ("""This is a simple input 1""", """This is a simple input 2"""),
										    ("""This is a simple pair 1""", """This is a simple pair 2"""),
										]

										# Simple input tests
										self.assertRaises(lowercase	,				tokenizer_r.encode	,				lowercase	,				max_length=lowercase	,				padding="""max_length"""							)

										# Simple input
										self.assertRaises(lowercase	,				tokenizer_r.encode_plus	,				lowercase	,				max_length=lowercase	,				padding="""max_length"""							)

										# Simple input
										self.assertRaises(
										    lowercase	,				tokenizer_r.batch_encode_plus	,				lowercase	,				max_length=lowercase	,				padding="""max_length"""	,				)

										# Pair input
										self.assertRaises(lowercase	,				tokenizer_r.encode	,				lowercase	,				max_length=lowercase	,				padding="""max_length"""							)

										# Pair input
										self.assertRaises(lowercase	,				tokenizer_r.encode_plus	,				lowercase	,				max_length=lowercase	,				padding="""max_length"""							)

										# Pair input
										self.assertRaises(
										    lowercase	,				tokenizer_r.batch_encode_plus	,				lowercase	,				max_length=lowercase	,				padding="""max_length"""	,				)


	def    _a     (self							):
				A_     :						List[Any]     =					CodeGenTokenizer.from_pretrained(self.tmpdirname	,				pad_token="""<pad>"""							)

				# Simple input
				A_     :						Union[str, Any]     =					"""This is a simple input"""
				A_     :						str     =					["""This is a simple input looooooooong""", """This is a simple input"""]
				A_     :						Optional[int]     =					("""This is a simple input""", """This is a pair""")
				A_     :						Dict     =					[
				    ("""This is a simple input loooooong""", """This is a simple input"""),
				    ("""This is a simple pair loooooong""", """This is a simple pair"""),
				]

				A_     :						str     =					tokenizer.pad_token_id

				A_     :						str     =					tokenizer(lowercase	,				padding="""max_length"""	,				max_length=30	,				return_tensors="""np"""							)
				A_     :						Union[str, Any]     =					tokenizer(lowercase	,				padding=lowercase	,				truncate=lowercase	,				return_tensors="""np"""							)
				A_     :						Optional[Any]     =					tokenizer(*lowercase	,				padding="""max_length"""	,				max_length=60	,				return_tensors="""np"""							)
				A_     :						List[Any]     =					tokenizer(lowercase	,				padding=lowercase	,				truncate=lowercase	,				return_tensors="""np"""							)

				# s
				# test single string max_length padding
				self.assertEqual(out_s["""input_ids"""].shape[-1]	,				30							)
				self.assertTrue(pad_token_id in out_s["""input_ids"""]							)
				self.assertTrue(0 in out_s["""attention_mask"""]							)

				# s2
				# test automatic padding
				self.assertEqual(out_sa["""input_ids"""].shape[-1]	,				33							)
				# long slice doesn't have padding
				self.assertFalse(pad_token_id in out_sa["""input_ids"""][0]							)
				self.assertFalse(0 in out_sa["""attention_mask"""][0]							)
				# short slice does have padding
				self.assertTrue(pad_token_id in out_sa["""input_ids"""][1]							)
				self.assertTrue(0 in out_sa["""attention_mask"""][1]							)

				# p
				# test single pair max_length padding
				self.assertEqual(out_p["""input_ids"""].shape[-1]	,				60							)
				self.assertTrue(pad_token_id in out_p["""input_ids"""]							)
				self.assertTrue(0 in out_p["""attention_mask"""]							)

				# p2
				# test automatic padding pair
				self.assertEqual(out_pa["""input_ids"""].shape[-1]	,				52							)
				# long slice pair doesn't have padding
				self.assertFalse(pad_token_id in out_pa["""input_ids"""][0]							)
				self.assertFalse(0 in out_pa["""attention_mask"""][0]							)
				# short slice pair does have padding
				self.assertTrue(pad_token_id in out_pa["""input_ids"""][1]							)
				self.assertTrue(0 in out_pa["""attention_mask"""][1]							)


	def    _a     (self							):
				A_     :						Optional[Any]     =					"""$$$"""
				A_     :						List[Any]     =					CodeGenTokenizer.from_pretrained(self.tmpdirname	,				bos_token=lowercase	,				add_bos_token=lowercase							)

				A_     :						Optional[int]     =					"""This is a simple input"""
				A_     :						Union[str, Any]     =					["""This is a simple input 1""", """This is a simple input 2"""]

				A_     :						Any     =					tokenizer.bos_token_id

				A_     :						Optional[int]     =					tokenizer(lowercase							)
				A_     :						int     =					tokenizer(lowercase							)

				self.assertEqual(out_s.input_ids[0]	,				lowercase							)
				self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids							)							)

				A_     :						Optional[Any]     =					tokenizer.decode(out_s.input_ids							)
				A_     :						Optional[int]     =					tokenizer.batch_decode(out_sa.input_ids							)

				self.assertEqual(decode_s.split()[0]	,				lowercase							)
				self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa							)							)


	@slow
	def    _a     (self							):
				A_     :						int     =					CodeGenTokenizer.from_pretrained("""Salesforce/codegen-350M-mono"""							)

				A_     :						List[Any]     =					"""\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#"""
				A_     :						Dict     =					"""\nif len_a > len_b:      result = a\nelse:      result = b"""

				A_     :						Tuple     =					tokenizer.encode(lowercase							)
				A_     :						Union[str, Any]     =					["""^#""", re.escape("""<|endoftext|>"""							), """^'''""", """^\"\"\"""", """\n\n\n"""]
				A_     :						Union[str, Any]     =					tokenizer.decode(lowercase	,				truncate_before_pattern=lowercase							)
				self.assertEqual(lowercase	,				lowercase							)


	def    _a     (self							):
				pass




'''simple docstring'''




import gc
import unittest

import numpy as np
import torch
from transformers import CLIPTextConfig, CLIPTextModelWithProjection, CLIPTokenizer

from diffusers import HeunDiscreteScheduler, PriorTransformer, ShapEPipeline
from diffusers.pipelines.shap_e import ShapERenderer
from diffusers.utils import load_numpy, slow
from diffusers.utils.testing_utils import require_torch_gpu, torch_device

from ..test_pipelines_common import PipelineTesterMixin, assert_mean_pixel_difference







class      _lowerCAmelCase							(      __UpperCAmelCase				,      unittest.TestCase			):
	__SCREAMING_SNAKE_CASE       :		Union[str, Any]		      = ShapEPipeline
	__SCREAMING_SNAKE_CASE       :		Optional[int]		      = ['prompt']
	__SCREAMING_SNAKE_CASE       :		str		      = ['prompt']
	__SCREAMING_SNAKE_CASE       :		Dict		      = [
	    'num_images_per_prompt',
	    'num_inference_steps',
	    'generator',
	    'latents',
	    'guidance_scale',
	    'frame_size',
	    'output_type',
	    'return_dict',
	]
	__SCREAMING_SNAKE_CASE       :		Union[str, Any]		      = False


	@property
	def    _a     (self							):
				return 32


	@property
	def    _a     (self							):
				return 32


	@property
	def    _a     (self							):
				return self.time_input_dim * 4


	@property
	def    _a     (self							):
				return 8


	@property
	def    _a     (self							):
				A_     :						Dict     =					CLIPTokenizer.from_pretrained("""hf-internal-testing/tiny-random-clip"""							)
				return tokenizer


	@property
	def    _a     (self							):
				torch.manual_seed(0							)
				A_     :						Any     =					CLIPTextConfig(
				    bos_token_id=0	,				eos_token_id=2	,				hidden_size=self.text_embedder_hidden_size	,				projection_dim=self.text_embedder_hidden_size	,				intermediate_size=37	,				layer_norm_eps=1E-05	,				num_attention_heads=4	,				num_hidden_layers=5	,				pad_token_id=1	,				vocab_size=1000	,				)
				return CLIPTextModelWithProjection(lowercase							)


	@property
	def    _a     (self							):
				torch.manual_seed(0							)

				A_     :						Any     =					{
				    """num_attention_heads""": 2,
				    """attention_head_dim""": 16,
				    """embedding_dim""": self.time_input_dim,
				    """num_embeddings""": 32,
				    """embedding_proj_dim""": self.text_embedder_hidden_size,
				    """time_embed_dim""": self.time_embed_dim,
				    """num_layers""": 1,
				    """clip_embed_dim""": self.time_input_dim * 2,
				    """additional_embeddings""": 0,
				    """time_embed_act_fn""": """gelu""",
				    """norm_in_type""": """layer""",
				    """encoder_hid_proj_type""": None,
				    """added_emb_type""": None,
				}

				A_     :						List[str]     =					PriorTransformer(**lowercase							)
				return model


	@property
	def    _a     (self							):
				torch.manual_seed(0							)

				A_     :						str     =					{
				    """param_shapes""": (
				        (self.renderer_dim, 93),
				        (self.renderer_dim, 8),
				        (self.renderer_dim, 8),
				        (self.renderer_dim, 8),
				    ),
				    """d_latent""": self.time_input_dim,
				    """d_hidden""": self.renderer_dim,
				    """n_output""": 12,
				    """background""": (
				        0.1,
				        0.1,
				        0.1,
				    ),
				}
				A_     :						Dict     =					ShapERenderer(**lowercase							)
				return model


	def    _a     (self							):
				A_     :						Optional[int]     =					self.dummy_prior
				A_     :						Optional[int]     =					self.dummy_text_encoder
				A_     :						int     =					self.dummy_tokenizer
				A_     :						Dict     =					self.dummy_renderer

				A_     :						Tuple     =					HeunDiscreteScheduler(
				    beta_schedule="""exp"""	,				num_train_timesteps=1024	,				prediction_type="""sample"""	,				use_karras_sigmas=lowercase	,				clip_sample=lowercase	,				clip_sample_range=1.0	,				)
				A_     :						Union[str, Any]     =					{
				    """prior""": prior,
				    """text_encoder""": text_encoder,
				    """tokenizer""": tokenizer,
				    """renderer""": renderer,
				    """scheduler""": scheduler,
				}

				return components


	def    _a     (self	,				lowercase	,				lowercase=0							):
				if str(lowercase							).startswith("""mps"""							):
							A_     :						Any     =					torch.manual_seed(lowercase							)
				else:
							A_     :						str     =					torch.Generator(device=lowercase							).manual_seed(lowercase							)
				A_     :						List[str]     =					{
				    """prompt""": """horse""",
				    """generator""": generator,
				    """num_inference_steps""": 1,
				    """frame_size""": 32,
				    """output_type""": """np""",
				}
				return inputs


	def    _a     (self							):
				A_     :						str     =					"""cpu"""

				A_     :						Union[str, Any]     =					self.get_dummy_components()

				A_     :						Optional[int]     =					self.pipeline_class(**lowercase							)
				A_     :						str     =					pipe.to(lowercase							)

				pipe.set_progress_bar_config(disable=lowercase							)

				A_     :						Dict     =					pipe(**self.get_dummy_inputs(lowercase							)							)
				A_     :						Dict     =					output.images[0]
				A_     :						int     =					image[0, -3:, -3:, -1]

				assert image.shape == (20, 32, 32, 3)

				A_     :						Tuple     =					np.array(
				    [
				        0.00_03_92_16,
				        0.00_03_92_16,
				        0.00_03_92_16,
				        0.00_03_92_16,
				        0.00_03_92_16,
				        0.00_03_92_16,
				        0.00_03_92_16,
				        0.00_03_92_16,
				        0.00_03_92_16,
				    ]							)

				assert np.abs(image_slice.flatten() - expected_slice							).max() < 1E-2


	def    _a     (self							):
				# NOTE: Larger batch sizes cause this test to timeout, only test on smaller batches
				self._test_inference_batch_consistent(batch_sizes=[1, 2]							)


	def    _a     (self							):
				A_     :						Tuple     =					torch_device == """cpu"""
				A_     :						Union[str, Any]     =					True

				self._test_inference_batch_single_identical(
				    batch_size=2	,				test_max_difference=lowercase	,				relax_max_difference=lowercase	,				)


	def    _a     (self							):
				A_     :						List[Any]     =					self.get_dummy_components()
				A_     :						Any     =					self.pipeline_class(**lowercase							)
				A_     :						Any     =					pipe.to(lowercase							)
				pipe.set_progress_bar_config(disable=lowercase							)

				A_     :						int     =					1
				A_     :						Union[str, Any]     =					2

				A_     :						Dict     =					self.get_dummy_inputs(lowercase							)

				for key in inputs.keys():
							if key in self.batch_params:
										A_     :						Optional[Any]     =					batch_size * [inputs[key]]

				A_     :						List[Any]     =					pipe(**lowercase	,				num_images_per_prompt=lowercase							)[0]

				assert images.shape[0] == batch_size * num_images_per_prompt







@slow
@require_torch_gpu
class      _lowerCAmelCase							(      unittest.TestCase			):


	def    _a     (self							):
				# clean up the VRAM after each test
				super().tearDown()
				gc.collect()
				torch.cuda.empty_cache()


	def    _a     (self							):
				A_     :						Union[str, Any]     =					load_numpy(
				    """https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main"""
				    """/shap_e/test_shap_e_np_out.npy"""							)
				A_     :						Tuple     =					ShapEPipeline.from_pretrained("""openai/shap-e"""							)
				A_     :						int     =					pipe.to(lowercase							)
				pipe.set_progress_bar_config(disable=lowercase							)

				A_     :						int     =					torch.Generator(device=lowercase							).manual_seed(0							)

				A_     :						List[str]     =					pipe(
				    """a shark"""	,				generator=lowercase	,				guidance_scale=15.0	,				num_inference_steps=64	,				frame_size=64	,				output_type="""np"""	,				).images[0]

				assert images.shape == (20, 64, 64, 3)

				assert_mean_pixel_difference(lowercase	,				lowercase							)