






import json
import multiprocessing
import os
import re
from collections import defaultdict

import torch
from accelerate import Accelerator
from accelerate.utils import set_seed
from arguments import HumanEvalArguments
from datasets import load_dataset, load_metric
from torch.utils.data import IterableDataset
from torch.utils.data.dataloader import DataLoader
from tqdm import tqdm

import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer, HfArgumentParser, StoppingCriteria, StoppingCriteriaList


__UpperCAmelCase     							=						['\nclass', '\ndef', '\n#', '\n@', '\nprint', '\nif']



class   lowerCamelCase				(_snake_case      ):







							'''simple docstring'''



							def __init__(							self    ,		_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase=None    ,		_UpperCamelCase=1       )     ->							Optional[int]:
												UpperCAmelCase_   :				Any						=						tokenizer
												UpperCAmelCase_   :				Any						=						dataset
												UpperCAmelCase_   :				List[Any]						=						len(_UpperCamelCase       ) if n_tasks is None else n_tasks
												UpperCAmelCase_   :				Dict						=						n_copies



							def __iter__(							self       )     ->							str:
												UpperCAmelCase_   :				List[str]						=						[]
												for task in range(self.n_tasks       ):
																	# without strip, the model generate commented codes ...
																	prompts.append(self.tokenizer.eos_token + self.dataset[task]['prompt'].strip()       )
												UpperCAmelCase_   :				List[Any]						=						self.tokenizer(_UpperCamelCase    ,		padding=_UpperCamelCase    ,		return_tensors='pt'       )
												for task in range(self.n_tasks       ):
																	for _ in range(self.n_copies       ):
																						yield {
																						    "ids": outputs.input_ids[task],
																						    "task_id": task,
																						    "input_len": outputs.attention_mask[task].sum(),
																						}





class   lowerCamelCase				(_snake_case      ):







							'''simple docstring'''



							def __init__(							self    ,		_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase       )     ->							Optional[Any]:
												UpperCAmelCase_   :				Optional[int]						=						start_length
												UpperCAmelCase_   :				Tuple						=						eof_strings
												UpperCAmelCase_   :				Tuple						=						tokenizer



							def __call__(							self    ,		_UpperCamelCase    ,		_UpperCamelCase    ,		**_UpperCamelCase       )     ->							Tuple:
												UpperCAmelCase_   :				Dict						=						self.tokenizer.batch_decode(input_ids[:, self.start_length :]       )
												UpperCAmelCase_   :				Tuple						=						[]
												for decoded_generation in decoded_generations:
																	done.append(any(stop_string in decoded_generation for stop_string in self.eof_strings       )       )
												return all(_UpperCamelCase       )





def 					lowercase__				(					__snake_case   :							Any  ):


					'''simple docstring'''


					UpperCAmelCase_   :				List[str]						=						re.split('(%s)' % '|'.join(__snake_case  )					,	__snake_case  )
					# last string should be ""
					return "".join(string_list[:-2]  )




def 					lowercase__				(					__snake_case   :							Any					,	__snake_case   :							Union[str, Any]					,	__snake_case   :							Optional[int]					,	__snake_case   :							int					,	__snake_case   :							Union[str, Any]					,	__snake_case   :							List[str]=20					,	**__snake_case   :							Optional[int]  ):


					'''simple docstring'''


					UpperCAmelCase_   :				Union[str, Any]						=						defaultdict(__snake_case  )  # dict of list of generated tokens
					for step, batch in tqdm(enumerate(__snake_case  )  ):
										with torch.no_grad():
															UpperCAmelCase_   :				Any						=						batch['ids'].shape[-1]
															UpperCAmelCase_   :				Optional[int]						=						accelerator.unwrap_model(__snake_case  ).generate(
															    input_ids=batch['ids'][:, : batch['input_len']]					,	num_return_sequences=__snake_case					,	**__snake_case  )
															# each task is generated batch_size times
															UpperCAmelCase_   :				Any						=						batch['task_id'].repeat(__snake_case  )
															UpperCAmelCase_   :				Union[str, Any]						=						accelerator.pad_across_processes(
															    __snake_case					,	dim=1					,	pad_index=tokenizer.pad_token_id  )

															UpperCAmelCase_   :				Tuple						=						accelerator.gather((generated_tokens, generated_tasks)  )
															UpperCAmelCase_   :				Tuple						=						generated_tokens.cpu().numpy()
															UpperCAmelCase_   :				Any						=						generated_tasks.cpu().numpy()

															for task, generated_tokens in zip(__snake_case					,	__snake_case  ):
																				gen_token_dict[task].append(__snake_case  )

					UpperCAmelCase_   :				Union[str, Any]						=						[[] for _ in range(__snake_case  )]
					for task, generated_tokens in gen_token_dict.items():
										for s in generated_tokens:
															UpperCAmelCase_   :				Dict						=						tokenizer.decode(__snake_case					,	skip_special_tokens=__snake_case					,	clean_up_tokenization_spaces=__snake_case  )
															code_gens[task].append(remove_last_block(__snake_case  )  )
					return code_gens




def 					lowercase__				(					):


					'''simple docstring'''


					UpperCAmelCase_   :				List[Any]						=						HfArgumentParser(__snake_case  )
					UpperCAmelCase_   :				str						=						parser.parse_args()

					transformers.logging.set_verbosity_error()
					# enables code execution in code_eval metric
					UpperCAmelCase_   :				Optional[int]						=						args.HF_ALLOW_CODE_EVAL
					# make sure tokenizer plays nice with multiprocessing
					UpperCAmelCase_   :				int						=						'false'

					if args.num_workers is None:
										UpperCAmelCase_   :				Tuple						=						multiprocessing.cpu_count()

					# Use dataset load to feed to accelerate
					UpperCAmelCase_   :				Tuple						=						Accelerator()
					set_seed(args.seed					,	device_specific=__snake_case  )

					# Load model and tokenizer
					UpperCAmelCase_   :				str						=						AutoTokenizer.from_pretrained(args.model_ckpt  )
					UpperCAmelCase_   :				Dict						=						tokenizer.eos_token
					UpperCAmelCase_   :				int						=						AutoModelForCausalLM.from_pretrained(args.model_ckpt  )

					# Generation settings
					UpperCAmelCase_   :				Union[str, Any]						=						{
					    'do_sample': args.do_sample,
					    'temperature': args.temperature,
					    'max_new_tokens': args.max_new_tokens,
					    'top_p': args.top_p,
					    'top_k': args.top_k,
					    'stopping_criteria': StoppingCriteriaList([EndOfFunctionCriteria(0					,	__snake_case					,	__snake_case  )]  ),
					}

					# Load evaluation dataset and metric
					UpperCAmelCase_   :				Optional[Any]						=						load_dataset('openai_humaneval'  )
					UpperCAmelCase_   :				int						=						load_metric('code_eval'  )

					UpperCAmelCase_   :				List[Any]						=						args.num_tasks if args.num_tasks is not None else len(human_eval['test']  )
					UpperCAmelCase_   :				Optional[int]						=						args.n_samples // args.batch_size

					UpperCAmelCase_   :				List[str]						=						TokenizedDataset(__snake_case					,	human_eval['test']					,	n_copies=__snake_case					,	n_tasks=__snake_case  )
					# do not confuse args.batch_size, which is actually the num_return_sequences
					UpperCAmelCase_   :				Optional[int]						=						DataLoader(__snake_case					,	batch_size=1  )

					# Run a quick test to see if code evaluation is enabled
					try:
										UpperCAmelCase_   :				int						=						code_eval_metric.compute(references=['']					,	predictions=[['']]  )
					except ValueError as exception:
										print(
										    'Code evaluation not enabled. Read the warning below carefully and then use `--HF_ALLOW_CODE_EVAL="1"`'
										    ' flag to enable code evaluation.'  )
										raise exception

					UpperCAmelCase_   :				Union[str, Any]						=						accelerator.prepare(__snake_case					,	__snake_case  )

					UpperCAmelCase_   :				str						=						complete_code(
					    __snake_case					,	__snake_case					,	__snake_case					,	__snake_case					,	n_tasks=__snake_case					,	batch_size=args.batch_size					,	**__snake_case					,	)

					if accelerator.is_main_process:
										UpperCAmelCase_   :				Tuple						=						[]

										for task in tqdm(range(__snake_case  )  ):
															UpperCAmelCase_   :				List[Any]						=						human_eval['test'][task]['test']
															UpperCAmelCase_   :				int						=						F"check({human_eval['test'][task]['entry_point']})"
															references.append('\n' + test_func + '\n' + entry_point  )

										# Evaluate completions with "code_eval" metric
										UpperCAmelCase_   :				Union[str, Any]						=						code_eval_metric.compute(
										    references=__snake_case					,	predictions=__snake_case					,	num_workers=args.num_workers  )
										print(F"Results: {pass_at_k}"  )

										# Save results to json file
										with open(args.output_file					,	'w'  ) as fp:
															json.dump(__snake_case					,	__snake_case  )


# For some reason the folliwng seems to be necessary sometimes for code_eval to work nice with multiprocessing
# https://stackoverflow.com/questions/60804599/python-multiprocessing-keeps-spawning-the-whole-script
if __name__ == "__main__":
		main()














from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxSeqaSeqConfigWithPast
from ...utils import logging


__UpperCAmelCase     							=						logging.get_logger(__name__)

__UpperCAmelCase     							=						{
    'google/umt5-small': 'https://huggingface.co/google/umt5-small/resolve/main/config.json',
    # See all umt5 models at https://huggingface.co/models?filter=umt5
}



class   lowerCamelCase				(_snake_case      ):







		'''simple docstring'''



		_snake_case   :   Union[str, Any]							=  '''umt5'''
		_snake_case   :   Union[str, Any]							=  ['''past_key_values''']



		def __init__(							self    ,		_UpperCamelCase=2_5_0_1_1_2    ,		_UpperCamelCase=5_1_2    ,		_UpperCamelCase=6_4    ,		_UpperCamelCase=1_0_2_4    ,		_UpperCamelCase=8    ,		_UpperCamelCase=None    ,		_UpperCamelCase=6    ,		_UpperCamelCase=3_2    ,		_UpperCamelCase=1_2_8    ,		_UpperCamelCase=0.1    ,		_UpperCamelCase=1E-6    ,		_UpperCamelCase=1.0    ,		_UpperCamelCase="gated-gelu"    ,		_UpperCamelCase=True    ,		_UpperCamelCase=True    ,		_UpperCamelCase="T5Tokenizer"    ,		_UpperCamelCase=True    ,		_UpperCamelCase=0    ,		_UpperCamelCase=1    ,		_UpperCamelCase=0    ,		**_UpperCamelCase    ,		)     ->							List[Any]:
							super().__init__(
							    is_encoder_decoder=_UpperCamelCase    ,		tokenizer_class=_UpperCamelCase    ,		tie_word_embeddings=_UpperCamelCase    ,		pad_token_id=_UpperCamelCase    ,		eos_token_id=_UpperCamelCase    ,		decoder_start_token_id=_UpperCamelCase    ,		**_UpperCamelCase    ,		)
							UpperCAmelCase_   :				Union[str, Any]						=						vocab_size
							UpperCAmelCase_   :				List[str]						=						d_model
							UpperCAmelCase_   :				Any						=						d_kv
							UpperCAmelCase_   :				Optional[int]						=						d_ff
							UpperCAmelCase_   :				List[Any]						=						num_layers
							UpperCAmelCase_   :				Optional[Any]						=						(
							    num_decoder_layers if num_decoder_layers is not None else self.num_layers
							)  # default = symmetry
							UpperCAmelCase_   :				Optional[int]						=						num_heads
							UpperCAmelCase_   :				Optional[int]						=						relative_attention_num_buckets
							UpperCAmelCase_   :				Dict						=						relative_attention_max_distance
							UpperCAmelCase_   :				Tuple						=						dropout_rate
							UpperCAmelCase_   :				Union[str, Any]						=						layer_norm_epsilon
							UpperCAmelCase_   :				Optional[int]						=						initializer_factor
							UpperCAmelCase_   :				List[str]						=						feed_forward_proj
							UpperCAmelCase_   :				Any						=						use_cache

							UpperCAmelCase_   :				List[Any]						=						self.feed_forward_proj.split('-'       )
							UpperCAmelCase_   :				List[Any]						=						act_info[-1]
							UpperCAmelCase_   :				Union[str, Any]						=						act_info[0] == 'gated'

							if len(_UpperCamelCase       ) > 1 and act_info[0] != "gated" or len(_UpperCamelCase       ) > 2:
												raise ValueError(
												    f"`feed_forward_proj`: {feed_forward_proj} is not a valid activation function of the dense layer."
												    'Please make sure `feed_forward_proj` is of the format `gated-{ACT_FN}` or `{ACT_FN}`, e.g. '
												    '\'gated-gelu\' or \'relu\''       )

							if feed_forward_proj == "gated-gelu":
												UpperCAmelCase_   :				Optional[int]						=						'gelu_new'



		@property
		def  __UpperCAmelCase      (							self       )     ->							int:
							return self.d_model



		@property
		def  __UpperCAmelCase      (							self       )     ->							Any:
							return self.num_heads



		@property
		def  __UpperCAmelCase      (							self       )     ->							List[Any]:
							return self.num_layers





class   lowerCamelCase				(_snake_case      ):







		'''simple docstring'''



		@property
		# Copied from transformers.models.t5.configuration_t5.T5OnnxConfig.inputs
		def  __UpperCAmelCase      (							self       )     ->							Mapping[str, Mapping[int, str]]:
							UpperCAmelCase_   :				str						=						{
							    'input_ids': {0: 'batch', 1: 'encoder_sequence'},
							    'attention_mask': {0: 'batch', 1: 'encoder_sequence'},
							}
							if self.use_past:
												UpperCAmelCase_   :				Optional[int]						=						'past_encoder_sequence + sequence'
												UpperCAmelCase_   :				str						=						{0: 'batch'}
												UpperCAmelCase_   :				Optional[int]						=						{0: 'batch', 1: 'past_decoder_sequence + sequence'}
							else:
												UpperCAmelCase_   :				Optional[int]						=						{0: 'batch', 1: 'decoder_sequence'}
												UpperCAmelCase_   :				Union[str, Any]						=						{0: 'batch', 1: 'decoder_sequence'}

							if self.use_past:
												self.fill_with_past_key_values_(_UpperCamelCase    ,		direction='inputs'       )

							return common_inputs



		@property
		# Copied from transformers.models.t5.configuration_t5.T5OnnxConfig.default_onnx_opset
		def  __UpperCAmelCase      (							self       )     ->							int:
							return 1_3



		@property
		def  __UpperCAmelCase      (							self       )     ->							float:
							return 5E-4






