

import json
import multiprocessing
import os
import re
from collections import defaultdict

import torch
from accelerate import Accelerator
from accelerate.utils import set_seed
from arguments import HumanEvalArguments
from datasets import load_dataset, load_metric
from torch.utils.data import IterableDataset
from torch.utils.data.dataloader import DataLoader
from tqdm import tqdm

import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer, HfArgumentParser, StoppingCriteria, StoppingCriteriaList


lowerCAmelCase							:  List[Any]            =						["""\nclass""", """\ndef""", """\n#""", """\n@""", """\nprint""", """\nif"""]



class   __lowercase    (			lowerCamelCase__     ):




			"""simple docstring"""







			def __init__(			self  :					Optional[Any]       ,						lowerCAmelCase__  :					Optional[int]       ,						lowerCAmelCase__  :					str       ,						lowerCAmelCase__  :					List[str]=None       ,						lowerCAmelCase__  :					int=1):
									SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	tokenizer
									SCREAMING_SNAKE_CASE_:			List[Any]  		=	dataset
									SCREAMING_SNAKE_CASE_:			int  		=	len(lowerCAmelCase__) if n_tasks is None else n_tasks
									SCREAMING_SNAKE_CASE_:			Union[str, Any]  		=	n_copies




			def __iter__(			self  :					Dict):
									SCREAMING_SNAKE_CASE_:			Dict  		=	[]
									for task in range(self.n_tasks):
															# without strip, the model generate commented codes ...
															prompts.append(self.tokenizer.eos_token + self.dataset[task]["prompt"].strip())
									SCREAMING_SNAKE_CASE_:			Any  		=	self.tokenizer(lowerCAmelCase__       ,						padding=lowerCAmelCase__       ,						return_tensors="pt")
									for task in range(self.n_tasks):
															for _ in range(self.n_copies):
																					yield {
																					    "ids": outputs.input_ids[task],
																					    "task_id": task,
																					    "input_len": outputs.attention_mask[task].sum(),
																					}







class   __lowercase    (			lowerCamelCase__     ):




			"""simple docstring"""







			def __init__(			self  :					List[str]       ,						lowerCAmelCase__  :					Dict       ,						lowerCAmelCase__  :					str       ,						lowerCAmelCase__  :					List[str]):
									SCREAMING_SNAKE_CASE_:			int  		=	start_length
									SCREAMING_SNAKE_CASE_:			Union[str, Any]  		=	eof_strings
									SCREAMING_SNAKE_CASE_:			int  		=	tokenizer




			def __call__(			self  :					Optional[int]       ,						lowerCAmelCase__  :					Union[str, Any]       ,						lowerCAmelCase__  :					int       ,						**lowerCAmelCase__  :					Union[str, Any]):
									SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	self.tokenizer.batch_decode(input_ids[:, self.start_length :])
									SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	[]
									for decoded_generation in decoded_generations:
															done.append(any(stop_string in decoded_generation for stop_string in self.eof_strings))
									return all(lowerCAmelCase__)







def       A_				(   _UpperCAmelCase   ):
						SCREAMING_SNAKE_CASE_:			Tuple  		=	re.split("(%s)" % "|".join(__snake_case   )    ,     __snake_case   )
						# last string should be ""
						return "".join(string_list[:-2]   )







def       A_				(   _UpperCAmelCase    ,     _UpperCAmelCase    ,     _UpperCAmelCase    ,     _UpperCAmelCase    ,     _UpperCAmelCase    ,     _UpperCAmelCase=20    ,     **_UpperCAmelCase   ):
						SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	defaultdict(__snake_case   )  # dict of list of generated tokens
						for step, batch in tqdm(enumerate(__snake_case   )   ):
												with torch.no_grad():
																		SCREAMING_SNAKE_CASE_:			int  		=	batch["ids"].shape[-1]
																		SCREAMING_SNAKE_CASE_:			int  		=	accelerator.unwrap_model(__snake_case   ).generate(
																		    input_ids=batch["ids"][:, : batch["input_len"]]    ,     num_return_sequences=__snake_case    ,     **__snake_case   )
																		# each task is generated batch_size times
																		SCREAMING_SNAKE_CASE_:			Dict  		=	batch["task_id"].repeat(__snake_case   )
																		SCREAMING_SNAKE_CASE_:			Tuple  		=	accelerator.pad_across_processes(
																		    __snake_case    ,     dim=1    ,     pad_index=tokenizer.pad_token_id   )

																		SCREAMING_SNAKE_CASE_		,   SCREAMING_SNAKE_CASE_:			Tuple  		=	accelerator.gather((generated_tokens, generated_tasks)   )
																		SCREAMING_SNAKE_CASE_:			Any  		=	generated_tokens.cpu().numpy()
																		SCREAMING_SNAKE_CASE_:			Tuple  		=	generated_tasks.cpu().numpy()

																		for task, generated_tokens in zip(__snake_case    ,     __snake_case   ):
																								gen_token_dict[task].append(__snake_case   )

						SCREAMING_SNAKE_CASE_:			Union[str, Any]  		=	[[] for _ in range(__snake_case   )]
						for task, generated_tokens in gen_token_dict.items():
												for s in generated_tokens:
																		SCREAMING_SNAKE_CASE_:			Tuple  		=	tokenizer.decode(__snake_case    ,     skip_special_tokens=__snake_case    ,     clean_up_tokenization_spaces=__snake_case   )
																		code_gens[task].append(remove_last_block(__snake_case   )   )
						return code_gens







def       A_				(   ):
						SCREAMING_SNAKE_CASE_:			Tuple  		=	HfArgumentParser(__snake_case   )
						SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	parser.parse_args()

						transformers.logging.set_verbosity_error()
						# enables code execution in code_eval metric
						SCREAMING_SNAKE_CASE_:			str  		=	args.HF_ALLOW_CODE_EVAL
						# make sure tokenizer plays nice with multiprocessing
						SCREAMING_SNAKE_CASE_:			List[Any]  		=	"false"

						if args.num_workers is None:
												SCREAMING_SNAKE_CASE_:			List[Any]  		=	multiprocessing.cpu_count()

						# Use dataset load to feed to accelerate
						SCREAMING_SNAKE_CASE_:			List[Any]  		=	Accelerator()
						set_seed(args.seed    ,     device_specific=__snake_case   )

						# Load model and tokenizer
						SCREAMING_SNAKE_CASE_:			Any  		=	AutoTokenizer.from_pretrained(args.model_ckpt   )
						SCREAMING_SNAKE_CASE_:			Optional[int]  		=	tokenizer.eos_token
						SCREAMING_SNAKE_CASE_:			Any  		=	AutoModelForCausalLM.from_pretrained(args.model_ckpt   )

						# Generation settings
						SCREAMING_SNAKE_CASE_:			int  		=	{
						    "do_sample": args.do_sample,
						    "temperature": args.temperature,
						    "max_new_tokens": args.max_new_tokens,
						    "top_p": args.top_p,
						    "top_k": args.top_k,
						    "stopping_criteria": StoppingCriteriaList([EndOfFunctionCriteria(0    ,     __snake_case    ,     __snake_case   )]   ),
						}

						# Load evaluation dataset and metric
						SCREAMING_SNAKE_CASE_:			List[str]  		=	load_dataset("openai_humaneval"   )
						SCREAMING_SNAKE_CASE_:			List[Any]  		=	load_metric("code_eval"   )

						SCREAMING_SNAKE_CASE_:			Dict  		=	args.num_tasks if args.num_tasks is not None else len(human_eval["test"]   )
						SCREAMING_SNAKE_CASE_:			Union[str, Any]  		=	args.n_samples // args.batch_size

						SCREAMING_SNAKE_CASE_:			str  		=	TokenizedDataset(__snake_case    ,     human_eval["test"]    ,     n_copies=__snake_case    ,     n_tasks=__snake_case   )
						# do not confuse args.batch_size, which is actually the num_return_sequences
						SCREAMING_SNAKE_CASE_:			Dict  		=	DataLoader(__snake_case    ,     batch_size=1   )

						# Run a quick test to see if code evaluation is enabled
						try:
												SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	code_eval_metric.compute(references=[""]    ,     predictions=[[""]]   )
						except ValueError as exception:
												print(
												    "Code evaluation not enabled. Read the warning below carefully and then use `--HF_ALLOW_CODE_EVAL=\"1\"`"
												    " flag to enable code evaluation."   )
												raise exception

						SCREAMING_SNAKE_CASE_		,   SCREAMING_SNAKE_CASE_:			int  		=	accelerator.prepare(__snake_case    ,     __snake_case   )

						SCREAMING_SNAKE_CASE_:			Union[str, Any]  		=	complete_code(
						    __snake_case    ,     __snake_case    ,     __snake_case    ,     __snake_case    ,     n_tasks=__snake_case    ,     batch_size=args.batch_size    ,     **__snake_case    ,     )

						if accelerator.is_main_process:
												SCREAMING_SNAKE_CASE_:			Union[str, Any]  		=	[]

												for task in tqdm(range(__snake_case   )   ):
																		SCREAMING_SNAKE_CASE_:			List[str]  		=	human_eval["test"][task]["test"]
																		SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	f"check({human_eval['test'][task]['entry_point']})"
																		references.append("\n" + test_func + "\n" + entry_point   )

												# Evaluate completions with "code_eval" metric
												SCREAMING_SNAKE_CASE_		,   SCREAMING_SNAKE_CASE_:			int  		=	code_eval_metric.compute(
												    references=__snake_case    ,     predictions=__snake_case    ,     num_workers=args.num_workers   )
												print(f"Results: {pass_at_k}"   )

												# Save results to json file
												with open(args.output_file    ,     "w"   ) as fp:
																		json.dump(__snake_case    ,     __snake_case   )


# For some reason the folliwng seems to be necessary sometimes for code_eval to work nice with multiprocessing
# https://stackoverflow.com/questions/60804599/python-multiprocessing-keeps-spawning-the-whole-script
if __name__ == "__main__":
						main()









import re
from typing import Callable, List, Optional, Union

import tensorflow as tf


try:
						from tensorflow.keras.optimizers.legacy import Adam
except ImportError:
						from tensorflow.keras.optimizers import Adam



class   __lowercase    (			tf.keras.optimizers.schedules.LearningRateSchedule     ):




			"""simple docstring"""







			def __init__(			self  :					List[Any]       ,						lowerCAmelCase__  :					float       ,						lowerCAmelCase__  :					Callable       ,						lowerCAmelCase__  :					int       ,						lowerCAmelCase__  :					float = 1.0       ,						lowerCAmelCase__  :					str = None       ,						):
									super().__init__()
									SCREAMING_SNAKE_CASE_:			str  		=	initial_learning_rate
									SCREAMING_SNAKE_CASE_:			Dict  		=	warmup_steps
									SCREAMING_SNAKE_CASE_:			Any  		=	power
									SCREAMING_SNAKE_CASE_:			int  		=	decay_schedule_fn
									SCREAMING_SNAKE_CASE_:			Union[str, Any]  		=	name



			def __call__(			self  :					Optional[Any]       ,						lowerCAmelCase__  :					Any):
									with tf.name_scope(self.name or "WarmUp") as name:
															# Implements polynomial warmup. i.e., if global_step < warmup_steps, the
															# learning rate will be `global_step/num_warmup_steps * init_lr`.
															SCREAMING_SNAKE_CASE_:			Any  		=	tf.cast(lowerCAmelCase__       ,						tf.floataa)
															SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	tf.cast(self.warmup_steps       ,						tf.floataa)
															SCREAMING_SNAKE_CASE_:			Optional[int]  		=	global_step_float / warmup_steps_float
															SCREAMING_SNAKE_CASE_:			Union[str, Any]  		=	self.initial_learning_rate * tf.math.pow(lowerCAmelCase__       ,						self.power)
															return tf.cond(
															    global_step_float < warmup_steps_float       ,						lambda: warmup_learning_rate       ,						lambda: self.decay_schedule_fn(step - self.warmup_steps)       ,						name=lowerCAmelCase__       ,						)




			def 	_SCREAMING_SNAKE_CASE      (			self  :					Tuple):
									return {
									    "initial_learning_rate": self.initial_learning_rate,
									    "decay_schedule_fn": self.decay_schedule_fn,
									    "warmup_steps": self.warmup_steps,
									    "power": self.power,
									    "name": self.name,
									}







def       A_				(   _UpperCAmelCase    ,     _UpperCAmelCase    ,     _UpperCAmelCase    ,     _UpperCAmelCase = 0.0    ,     _UpperCAmelCase = 0.9    ,     _UpperCAmelCase = 0.9_9_9    ,     _UpperCAmelCase = 1e-8    ,     _UpperCAmelCase = None    ,     _UpperCAmelCase = None    ,     _UpperCAmelCase = 0.0    ,     _UpperCAmelCase = 1.0    ,     _UpperCAmelCase = None    ,     ):
						SCREAMING_SNAKE_CASE_:			Optional[int]  		=	tf.keras.optimizers.schedules.PolynomialDecay(
						    initial_learning_rate=_UpperCAmelCase    ,     decay_steps=num_train_steps - num_warmup_steps    ,     end_learning_rate=init_lr * min_lr_ratio    ,     power=_UpperCAmelCase    ,     )
						if num_warmup_steps:
												SCREAMING_SNAKE_CASE_:			Tuple  		=	WarmUp(
												    initial_learning_rate=_UpperCAmelCase    ,     decay_schedule_fn=_UpperCAmelCase    ,     warmup_steps=_UpperCAmelCase    ,     )
						if weight_decay_rate > 0.0:
												SCREAMING_SNAKE_CASE_:			List[str]  		=	AdamWeightDecay(
												    learning_rate=_UpperCAmelCase    ,     weight_decay_rate=_UpperCAmelCase    ,     beta_a=_UpperCAmelCase    ,     beta_a=_UpperCAmelCase    ,     epsilon=_UpperCAmelCase    ,     clipnorm=_UpperCAmelCase    ,     global_clipnorm=_UpperCAmelCase    ,     exclude_from_weight_decay=["LayerNorm", "layer_norm", "bias"]    ,     include_in_weight_decay=_UpperCAmelCase    ,     )
						else:
												SCREAMING_SNAKE_CASE_:			int  		=	tf.keras.optimizers.Adam(
												    learning_rate=_UpperCAmelCase    ,     beta_a=_UpperCAmelCase    ,     beta_a=_UpperCAmelCase    ,     epsilon=_UpperCAmelCase    ,     clipnorm=_UpperCAmelCase    ,     global_clipnorm=_UpperCAmelCase    ,     )
						# We return the optimizer and the LR scheduler in order to better track the
						# evolution of the LR independently of the optimizer.
						return optimizer, lr_schedule



class   __lowercase    (			UpperCAmelCase_     ):




			"""simple docstring"""







			def __init__(			self  :					List[Any]       ,						lowerCAmelCase__  :					Union[float, tf.keras.optimizers.schedules.LearningRateSchedule] = 0.001       ,						lowerCAmelCase__  :					float = 0.9       ,						lowerCAmelCase__  :					float = 0.999       ,						lowerCAmelCase__  :					float = 1E-7       ,						lowerCAmelCase__  :					bool = False       ,						lowerCAmelCase__  :					float = 0.0       ,						lowerCAmelCase__  :					Optional[List[str]] = None       ,						lowerCAmelCase__  :					Optional[List[str]] = None       ,						lowerCAmelCase__  :					str = "AdamWeightDecay"       ,						**lowerCAmelCase__  :					int       ,						):
									super().__init__(lowerCAmelCase__       ,						lowerCAmelCase__       ,						lowerCAmelCase__       ,						lowerCAmelCase__       ,						lowerCAmelCase__       ,						lowerCAmelCase__       ,						**lowerCAmelCase__)
									SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	weight_decay_rate
									SCREAMING_SNAKE_CASE_:			List[Any]  		=	include_in_weight_decay
									SCREAMING_SNAKE_CASE_:			List[Any]  		=	exclude_from_weight_decay



			@classmethod
			def 	_SCREAMING_SNAKE_CASE      (			cls  :					Dict       ,						lowerCAmelCase__  :					Dict):
									SCREAMING_SNAKE_CASE_:			List[str]  		=	{"WarmUp": WarmUp}
									return super(lowerCAmelCase__       ,						cls).from_config(lowerCAmelCase__       ,						custom_objects=lowerCAmelCase__)



			def 	_SCREAMING_SNAKE_CASE      (			self  :					List[str]       ,						lowerCAmelCase__  :					List[str]       ,						lowerCAmelCase__  :					List[Any]       ,						lowerCAmelCase__  :					Optional[int]):
									super(lowerCAmelCase__       ,						self)._prepare_local(lowerCAmelCase__       ,						lowerCAmelCase__       ,						lowerCAmelCase__)
									SCREAMING_SNAKE_CASE_:			Optional[int]  		=	tf.constant(
									    self.weight_decay_rate       ,						name="adam_weight_decay_rate")



			def 	_SCREAMING_SNAKE_CASE      (			self  :					Optional[Any]       ,						lowerCAmelCase__  :					str       ,						lowerCAmelCase__  :					Any       ,						lowerCAmelCase__  :					Tuple):
									SCREAMING_SNAKE_CASE_:			str  		=	self._do_use_weight_decay(var.name)
									if do_decay:
															return var.assign_sub(
															    learning_rate * var * apply_state[(var.device, var.dtype.base_dtype)]["weight_decay_rate"]       ,						use_locking=self._use_locking       ,						)
									return tf.no_op()



			def 	_SCREAMING_SNAKE_CASE      (			self  :					Dict       ,						lowerCAmelCase__  :					Optional[int]       ,						lowerCAmelCase__  :					Optional[Any]=None       ,						**lowerCAmelCase__  :					List[str]):
									SCREAMING_SNAKE_CASE_		,   SCREAMING_SNAKE_CASE_:			Any  		=	list(zip(*lowerCAmelCase__))
									return super(lowerCAmelCase__       ,						self).apply_gradients(zip(lowerCAmelCase__       ,						lowerCAmelCase__)       ,						name=lowerCAmelCase__       ,						**lowerCAmelCase__)



			def 	_SCREAMING_SNAKE_CASE      (			self  :					Optional[int]       ,						lowerCAmelCase__  :					Any       ,						lowerCAmelCase__  :					Tuple       ,						lowerCAmelCase__  :					Tuple):
									if apply_state is None:
															return self._decayed_lr_t[var_dtype], {}

									SCREAMING_SNAKE_CASE_:			Dict  		=	apply_state or {}
									SCREAMING_SNAKE_CASE_:			List[str]  		=	apply_state.get((var_device, var_dtype))
									if coefficients is None:
															SCREAMING_SNAKE_CASE_:			Optional[int]  		=	self._fallback_apply_state(lowerCAmelCase__       ,						lowerCAmelCase__)
															SCREAMING_SNAKE_CASE_:			Any  		=	coefficients

									return coefficients["lr_t"], {"apply_state": apply_state}



			def 	_SCREAMING_SNAKE_CASE      (			self  :					str       ,						lowerCAmelCase__  :					List[Any]       ,						lowerCAmelCase__  :					Dict       ,						lowerCAmelCase__  :					Tuple=None):
									SCREAMING_SNAKE_CASE_		,   SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	self._get_lr(var.device       ,						var.dtype.base_dtype       ,						lowerCAmelCase__)
									SCREAMING_SNAKE_CASE_:			Optional[int]  		=	self._decay_weights_op(lowerCAmelCase__       ,						lowerCAmelCase__       ,						lowerCAmelCase__)
									with tf.control_dependencies([decay]):
															return super(lowerCAmelCase__       ,						self)._resource_apply_dense(lowerCAmelCase__       ,						lowerCAmelCase__       ,						**lowerCAmelCase__)



			def 	_SCREAMING_SNAKE_CASE      (			self  :					Union[str, Any]       ,						lowerCAmelCase__  :					List[str]       ,						lowerCAmelCase__  :					Dict       ,						lowerCAmelCase__  :					Dict       ,						lowerCAmelCase__  :					Dict=None):
									SCREAMING_SNAKE_CASE_		,   SCREAMING_SNAKE_CASE_:			List[str]  		=	self._get_lr(var.device       ,						var.dtype.base_dtype       ,						lowerCAmelCase__)
									SCREAMING_SNAKE_CASE_:			Any  		=	self._decay_weights_op(lowerCAmelCase__       ,						lowerCAmelCase__       ,						lowerCAmelCase__)
									with tf.control_dependencies([decay]):
															return super(lowerCAmelCase__       ,						self)._resource_apply_sparse(lowerCAmelCase__       ,						lowerCAmelCase__       ,						lowerCAmelCase__       ,						**lowerCAmelCase__)



			def 	_SCREAMING_SNAKE_CASE      (			self  :					Optional[Any]):
									SCREAMING_SNAKE_CASE_:			List[str]  		=	super().get_config()
									config.update({"weight_decay_rate": self.weight_decay_rate})
									return config




			def 	_SCREAMING_SNAKE_CASE      (			self  :					Union[str, Any]       ,						lowerCAmelCase__  :					Tuple):
									if self.weight_decay_rate == 0:
															return False

									if self._include_in_weight_decay:
															for r in self._include_in_weight_decay:
																					if re.search(lowerCAmelCase__       ,						lowerCAmelCase__) is not None:
																											return True

									if self._exclude_from_weight_decay:
															for r in self._exclude_from_weight_decay:
																					if re.search(lowerCAmelCase__       ,						lowerCAmelCase__) is not None:
																											return False
									return True







class   __lowercase    (			UpperCAmelCase_     ):




			"""simple docstring"""







			def __init__(			self  :					Optional[Any]):
									SCREAMING_SNAKE_CASE_:			Any  		=	[]
									SCREAMING_SNAKE_CASE_:			Any  		=	None



			@property
			def 	_SCREAMING_SNAKE_CASE      (			self  :					int):
									if self._accum_steps is None:
															SCREAMING_SNAKE_CASE_:			Tuple  		=	tf.Variable(
															    tf.constant(0       ,						dtype=tf.intaa)       ,						trainable=lowerCAmelCase__       ,						synchronization=tf.VariableSynchronization.ON_READ       ,						aggregation=tf.VariableAggregation.ONLY_FIRST_REPLICA       ,						)

									return self._accum_steps.value()



			@property
			def 	_SCREAMING_SNAKE_CASE      (			self  :					Tuple):
									if not self._gradients:
															raise ValueError("The accumulator should be called first to initialize the gradients")
									return [gradient.value() if gradient is not None else gradient for gradient in self._gradients]



			def __call__(			self  :					str       ,						lowerCAmelCase__  :					Tuple):
									if not self._gradients:
															SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	self.step  # Create the step variable.
															self._gradients.extend(
															    [
															        tf.Variable(
															            tf.zeros_like(lowerCAmelCase__)       ,						trainable=lowerCAmelCase__       ,						synchronization=tf.VariableSynchronization.ON_READ       ,						aggregation=tf.VariableAggregation.ONLY_FIRST_REPLICA       ,						)
															        if gradient is not None
															        else gradient
															        for gradient in gradients
															    ])
									if len(lowerCAmelCase__) != len(self._gradients):
															raise ValueError(F"Expected {len(self._gradients)} gradients, but got {len(lowerCAmelCase__)}")

									for accum_gradient, gradient in zip(self._gradients       ,						lowerCAmelCase__):
															if accum_gradient is not None and gradient is not None:
																					accum_gradient.assign_add(lowerCAmelCase__)

									self._accum_steps.assign_add(1)




			def 	_SCREAMING_SNAKE_CASE      (			self  :					int):
									if not self._gradients:
															return
									self._accum_steps.assign(0)
									for gradient in self._gradients:
															if gradient is not None:
																					gradient.assign(tf.zeros_like(lowerCAmelCase__))






