



import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class 	lowerCamelCase__      (							lowerCamelCase__				,    unittest.TestCase):




		'''simple docstring'''



		snake_case_     =CodeGenTokenizer
		snake_case_     =CodeGenTokenizerFast
		snake_case_     =True
		snake_case_     ={"""add_prefix_space""": True}
		snake_case_     =False



		def 		lowerCAmelCase__				(self			)					->      Union[str, Any]:

						"""simple docstring"""






						super().setUp()

						# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
						lowerCAmelCase__					:    Optional[Any]												=					[
						    '''l''',
						    '''o''',
						    '''w''',
						    '''e''',
						    '''r''',
						    '''s''',
						    '''t''',
						    '''i''',
						    '''d''',
						    '''n''',
						    '''\u0120''',
						    '''\u0120l''',
						    '''\u0120n''',
						    '''\u0120lo''',
						    '''\u0120low''',
						    '''er''',
						    '''\u0120lowest''',
						    '''\u0120newer''',
						    '''\u0120wider''',
						    '''<unk>''',
						    '''<|endoftext|>''',
						]
						lowerCAmelCase__					:    Optional[Any]												=					dict(zip(__lowerCamelCase		,range(len(__lowerCamelCase			)			)			)			)
						lowerCAmelCase__					:    Optional[Any]												=					['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
						lowerCAmelCase__					:    Dict												=					{'''unk_token''': '''<unk>'''}

						lowerCAmelCase__					:    List[Any]												=					os.path.join(self.tmpdirname		,VOCAB_FILES_NAMES['''vocab_file''']			)
						lowerCAmelCase__					:    Tuple												=					os.path.join(self.tmpdirname		,VOCAB_FILES_NAMES['''merges_file''']			)
						with open(self.vocab_file		,'''w'''		,encoding='''utf-8'''			) as fp:
										fp.write(json.dumps(__lowerCamelCase			) + '''\n'''			)
						with open(self.merges_file		,'''w'''		,encoding='''utf-8'''			) as fp:
										fp.write('''\n'''.join(__lowerCamelCase			)			)



		def 		lowerCAmelCase__				(self		,**__lowerCamelCase			)					->      List[str]:

						"""simple docstring"""






						kwargs.update(self.special_tokens_map			)
						return CodeGenTokenizer.from_pretrained(self.tmpdirname		,**__lowerCamelCase			)



		def 		lowerCAmelCase__				(self		,**__lowerCamelCase			)					->      int:

						"""simple docstring"""






						kwargs.update(self.special_tokens_map			)
						return CodeGenTokenizerFast.from_pretrained(self.tmpdirname		,**__lowerCamelCase			)



		def 		lowerCAmelCase__				(self		,__lowerCamelCase			)					->      Optional[int]:

						"""simple docstring"""






						lowerCAmelCase__					:    Any												=					'''lower newer'''
						lowerCAmelCase__					:    Tuple												=					'''lower newer'''
						return input_text, output_text



		def 		lowerCAmelCase__				(self			)					->      Union[str, Any]:

						"""simple docstring"""






						lowerCAmelCase__					:    Tuple												=					CodeGenTokenizer(self.vocab_file		,self.merges_file		,**self.special_tokens_map			)
						lowerCAmelCase__					:    List[Any]												=					'''lower newer'''
						lowerCAmelCase__					:    Union[str, Any]												=					['''\u0120low''', '''er''', '''\u0120''', '''n''', '''e''', '''w''', '''er''']
						lowerCAmelCase__					:    Optional[Any]												=					tokenizer.tokenize(__lowerCamelCase		,add_prefix_space=__lowerCamelCase			)
						self.assertListEqual(__lowerCamelCase		,__lowerCamelCase			)

						lowerCAmelCase__					:    Optional[Any]												=					tokens + [tokenizer.unk_token]
						lowerCAmelCase__					:    Dict												=					[14, 15, 10, 9, 3, 2, 15, 19]
						self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase			)		,__lowerCamelCase			)



		def 		lowerCAmelCase__				(self			)					->      int:

						"""simple docstring"""






						if not self.test_rust_tokenizer:
										return

						lowerCAmelCase__					:    List[str]												=					self.get_tokenizer()
						lowerCAmelCase__					:    List[Any]												=					self.get_rust_tokenizer(add_prefix_space=__lowerCamelCase			)

						lowerCAmelCase__					:    int												=					'''lower newer'''

						# Testing tokenization
						lowerCAmelCase__					:    int												=					tokenizer.tokenize(__lowerCamelCase		,add_prefix_space=__lowerCamelCase			)
						lowerCAmelCase__					:    Optional[Any]												=					rust_tokenizer.tokenize(__lowerCamelCase			)
						self.assertListEqual(__lowerCamelCase		,__lowerCamelCase			)

						# Testing conversion to ids without special tokens
						lowerCAmelCase__					:    int												=					tokenizer.encode(__lowerCamelCase		,add_special_tokens=__lowerCamelCase		,add_prefix_space=__lowerCamelCase			)
						lowerCAmelCase__					:    List[str]												=					rust_tokenizer.encode(__lowerCamelCase		,add_special_tokens=__lowerCamelCase			)
						self.assertListEqual(__lowerCamelCase		,__lowerCamelCase			)

						# Testing conversion to ids with special tokens
						lowerCAmelCase__					:    List[str]												=					self.get_rust_tokenizer(add_prefix_space=__lowerCamelCase			)
						lowerCAmelCase__					:    Union[str, Any]												=					tokenizer.encode(__lowerCamelCase		,add_prefix_space=__lowerCamelCase			)
						lowerCAmelCase__					:    Tuple												=					rust_tokenizer.encode(__lowerCamelCase			)
						self.assertListEqual(__lowerCamelCase		,__lowerCamelCase			)

						# Testing the unknown token
						lowerCAmelCase__					:    str												=					tokens + [rust_tokenizer.unk_token]
						lowerCAmelCase__					:    Dict												=					[14, 15, 10, 9, 3, 2, 15, 19]
						self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(__lowerCamelCase			)		,__lowerCamelCase			)



		def 		lowerCAmelCase__				(self		,*__lowerCamelCase		,**__lowerCamelCase			)					->      Any:

						"""simple docstring"""






						pass



		def 		lowerCAmelCase__				(self		,__lowerCamelCase=15			)					->      Optional[Any]:

						"""simple docstring"""






						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""			):
														lowerCAmelCase__					:    str												=					self.rust_tokenizer_class.from_pretrained(__lowerCamelCase		,**__lowerCamelCase			)

														# Simple input
														lowerCAmelCase__					:    Dict												=					'''This is a simple input'''
														lowerCAmelCase__					:    Optional[Any]												=					['''This is a simple input 1''', '''This is a simple input 2''']
														lowerCAmelCase__					:    int												=					('''This is a simple input''', '''This is a pair''')
														lowerCAmelCase__					:    List[Any]												=					[
														    ('''This is a simple input 1''', '''This is a simple input 2'''),
														    ('''This is a simple pair 1''', '''This is a simple pair 2'''),
														]

														# Simple input tests
														self.assertRaises(__lowerCamelCase		,tokenizer_r.encode		,__lowerCamelCase		,max_length=__lowerCamelCase		,padding='''max_length'''			)

														# Simple input
														self.assertRaises(__lowerCamelCase		,tokenizer_r.encode_plus		,__lowerCamelCase		,max_length=__lowerCamelCase		,padding='''max_length'''			)

														# Simple input
														self.assertRaises(
														    __lowerCamelCase		,tokenizer_r.batch_encode_plus		,__lowerCamelCase		,max_length=__lowerCamelCase		,padding='''max_length'''		,)

														# Pair input
														self.assertRaises(__lowerCamelCase		,tokenizer_r.encode		,__lowerCamelCase		,max_length=__lowerCamelCase		,padding='''max_length'''			)

														# Pair input
														self.assertRaises(__lowerCamelCase		,tokenizer_r.encode_plus		,__lowerCamelCase		,max_length=__lowerCamelCase		,padding='''max_length'''			)

														# Pair input
														self.assertRaises(
														    __lowerCamelCase		,tokenizer_r.batch_encode_plus		,__lowerCamelCase		,max_length=__lowerCamelCase		,padding='''max_length'''		,)



		def 		lowerCAmelCase__				(self			)					->      List[str]:

						"""simple docstring"""






						lowerCAmelCase__					:    Tuple												=					CodeGenTokenizer.from_pretrained(self.tmpdirname		,pad_token='''<pad>'''			)

						# Simple input
						lowerCAmelCase__					:    str												=					'''This is a simple input'''
						lowerCAmelCase__					:    Dict												=					['''This is a simple input looooooooong''', '''This is a simple input''']
						lowerCAmelCase__					:    Dict												=					('''This is a simple input''', '''This is a pair''')
						lowerCAmelCase__					:    str												=					[
						    ('''This is a simple input loooooong''', '''This is a simple input'''),
						    ('''This is a simple pair loooooong''', '''This is a simple pair'''),
						]

						lowerCAmelCase__					:    List[Any]												=					tokenizer.pad_token_id

						lowerCAmelCase__					:    Optional[int]												=					tokenizer(__lowerCamelCase		,padding='''max_length'''		,max_length=30		,return_tensors='''np'''			)
						lowerCAmelCase__					:    List[Any]												=					tokenizer(__lowerCamelCase		,padding=__lowerCamelCase		,truncate=__lowerCamelCase		,return_tensors='''np'''			)
						lowerCAmelCase__					:    Union[str, Any]												=					tokenizer(*__lowerCamelCase		,padding='''max_length'''		,max_length=60		,return_tensors='''np'''			)
						lowerCAmelCase__					:    Any												=					tokenizer(__lowerCamelCase		,padding=__lowerCamelCase		,truncate=__lowerCamelCase		,return_tensors='''np'''			)

						# s
						# test single string max_length padding
						self.assertEqual(out_s['''input_ids'''].shape[-1]		,30			)
						self.assertTrue(pad_token_id in out_s['''input_ids''']			)
						self.assertTrue(0 in out_s['''attention_mask''']			)

						# s2
						# test automatic padding
						self.assertEqual(out_sa['''input_ids'''].shape[-1]		,33			)
						# long slice doesn't have padding
						self.assertFalse(pad_token_id in out_sa['''input_ids'''][0]			)
						self.assertFalse(0 in out_sa['''attention_mask'''][0]			)
						# short slice does have padding
						self.assertTrue(pad_token_id in out_sa['''input_ids'''][1]			)
						self.assertTrue(0 in out_sa['''attention_mask'''][1]			)

						# p
						# test single pair max_length padding
						self.assertEqual(out_p['''input_ids'''].shape[-1]		,60			)
						self.assertTrue(pad_token_id in out_p['''input_ids''']			)
						self.assertTrue(0 in out_p['''attention_mask''']			)

						# p2
						# test automatic padding pair
						self.assertEqual(out_pa['''input_ids'''].shape[-1]		,52			)
						# long slice pair doesn't have padding
						self.assertFalse(pad_token_id in out_pa['''input_ids'''][0]			)
						self.assertFalse(0 in out_pa['''attention_mask'''][0]			)
						# short slice pair does have padding
						self.assertTrue(pad_token_id in out_pa['''input_ids'''][1]			)
						self.assertTrue(0 in out_pa['''attention_mask'''][1]			)



		def 		lowerCAmelCase__				(self			)					->      Optional[Any]:

						"""simple docstring"""






						lowerCAmelCase__					:    Any												=					'''$$$'''
						lowerCAmelCase__					:    Dict												=					CodeGenTokenizer.from_pretrained(self.tmpdirname		,bos_token=__lowerCamelCase		,add_bos_token=__lowerCamelCase			)

						lowerCAmelCase__					:    List[Any]												=					'''This is a simple input'''
						lowerCAmelCase__					:    int												=					['''This is a simple input 1''', '''This is a simple input 2''']

						lowerCAmelCase__					:    str												=					tokenizer.bos_token_id

						lowerCAmelCase__					:    List[Any]												=					tokenizer(__lowerCamelCase			)
						lowerCAmelCase__					:    Optional[int]												=					tokenizer(__lowerCamelCase			)

						self.assertEqual(out_s.input_ids[0]		,__lowerCamelCase			)
						self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids			)			)

						lowerCAmelCase__					:    str												=					tokenizer.decode(out_s.input_ids			)
						lowerCAmelCase__					:    List[Any]												=					tokenizer.batch_decode(out_sa.input_ids			)

						self.assertEqual(decode_s.split()[0]		,__lowerCamelCase			)
						self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa			)			)



		@slow
		def 		lowerCAmelCase__				(self			)					->      str:

						"""simple docstring"""






						lowerCAmelCase__					:    List[Any]												=					CodeGenTokenizer.from_pretrained('''Salesforce/codegen-350M-mono'''			)

						lowerCAmelCase__					:    str												=					'''\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#'''
						lowerCAmelCase__					:    Dict												=					'''\nif len_a > len_b:      result = a\nelse:      result = b'''

						lowerCAmelCase__					:    Tuple												=					tokenizer.encode(__lowerCamelCase			)
						lowerCAmelCase__					:    int												=					['''^#''', re.escape('''<|endoftext|>'''			), '''^\'\'\'''', '''^"""''', '''\n\n\n''']
						lowerCAmelCase__					:    Dict												=					tokenizer.decode(__lowerCamelCase		,truncate_before_pattern=__lowerCamelCase			)
						self.assertEqual(__lowerCamelCase		,__lowerCamelCase			)



		def 		lowerCAmelCase__				(self			)					->      Tuple:

						"""simple docstring"""






						pass






import random
import unittest

import torch

from diffusers import IFInpaintingSuperResolutionPipeline
from diffusers.utils import floats_tensor
from diffusers.utils.import_utils import is_xformers_available
from diffusers.utils.testing_utils import skip_mps, torch_device

from ..pipeline_params import (
    TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS,
    TEXT_GUIDED_IMAGE_INPAINTING_PARAMS,
)
from ..test_pipelines_common import PipelineTesterMixin
from . import IFPipelineTesterMixin



@skip_mps
class 	lowerCamelCase__      (							lowerCamelCase__				,    lowerCamelCase__				,    unittest.TestCase):




		'''simple docstring'''



		snake_case_     =IFInpaintingSuperResolutionPipeline
		snake_case_     =TEXT_GUIDED_IMAGE_INPAINTING_PARAMS - {"""width""", """height"""}
		snake_case_     =TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS.union({"""original_image"""})
		snake_case_     =PipelineTesterMixin.required_optional_params - {"""latents"""}



		def 		lowerCAmelCase__				(self			)					->      Union[str, Any]:

						"""simple docstring"""






						return self._get_superresolution_dummy_components()



		def 		lowerCAmelCase__				(self		,__lowerCamelCase		,__lowerCamelCase=0			)					->      Dict:

						"""simple docstring"""






						if str(__lowerCamelCase			).startswith('''mps'''			):
										lowerCAmelCase__					:    Union[str, Any]												=					torch.manual_seed(__lowerCamelCase			)
						else:
										lowerCAmelCase__					:    int												=					torch.Generator(device=__lowerCamelCase			).manual_seed(__lowerCamelCase			)

						lowerCAmelCase__					:    str												=					floats_tensor((1, 3, 16, 16)		,rng=random.Random(__lowerCamelCase			)			).to(__lowerCamelCase			)
						lowerCAmelCase__					:    Optional[Any]												=					floats_tensor((1, 3, 32, 32)		,rng=random.Random(__lowerCamelCase			)			).to(__lowerCamelCase			)
						lowerCAmelCase__					:    Union[str, Any]												=					floats_tensor((1, 3, 32, 32)		,rng=random.Random(__lowerCamelCase			)			).to(__lowerCamelCase			)

						lowerCAmelCase__					:    Dict												=					{
						    '''prompt''': '''A painting of a squirrel eating a burger''',
						    '''image''': image,
						    '''original_image''': original_image,
						    '''mask_image''': mask_image,
						    '''generator''': generator,
						    '''num_inference_steps''': 2,
						    '''output_type''': '''numpy''',
						}

						return inputs



		@unittest.skipIf(
		    torch_device != '''cuda''' or not is_xformers_available()		,reason='''XFormers attention is only available with CUDA and `xformers` installed'''		,)
		def 		lowerCAmelCase__				(self			)					->      int:

						"""simple docstring"""






						self._test_xformers_attention_forwardGenerator_pass(expected_max_diff=1e-3			)



		def 		lowerCAmelCase__				(self			)					->      int:

						"""simple docstring"""






						self._test_save_load_optional_components()



		@unittest.skipIf(torch_device != '''cuda'''		,reason='''float16 requires CUDA'''			)
		def 		lowerCAmelCase__				(self			)					->      str:

						"""simple docstring"""






						super().test_save_load_floataa(expected_max_diff=1e-1			)



		def 		lowerCAmelCase__				(self			)					->      Any:

						"""simple docstring"""






						self._test_attention_slicing_forward_pass(expected_max_diff=1e-2			)



		def 		lowerCAmelCase__				(self			)					->      Optional[Any]:

						"""simple docstring"""






						self._test_save_load_local()



		def 		lowerCAmelCase__				(self			)					->      Any:

						"""simple docstring"""






						self._test_inference_batch_single_identical(
						    expected_max_diff=1e-2		,)

