





import argparse
import csv
import logging
import os
import random

import numpy as np
import torch
from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, TensorDataset
from tqdm import tqdm, trange

from transformers import (
    CONFIG_NAME,
    WEIGHTS_NAME,
    AdamW,
    OpenAIGPTDoubleHeadsModel,
    OpenAIGPTTokenizer,
    get_linear_schedule_with_warmup,
)


logging.basicConfig(
    format="""%(asctime)s - %(levelname)s - %(name)s - %(message)s""", datefmt="""%m/%d/%Y %H:%M:%S""", level=logging.INFO
)
_SCREAMING_SNAKE_CASE 		=						logging.getLogger(__name__)


def 							SCREAMING_SNAKE_CASE__  (      __a				,	__a				):
	snake_case_						:     str     =		np.argmax(__a				,	axis=1				)
	return np.sum(outputs == labels				)


def 							SCREAMING_SNAKE_CASE__  (      __a				):
	with open(__a				,	encoding='utf_8'				) as f:
		snake_case_						:     Any     =		csv.reader(__a				)
		snake_case_						:     List[Any]     =		[]
		next(__a				)  # skip the first line
		for line in tqdm(__a				):
			output.append((' '.join(line[1:5]				), line[5], line[6], int(line[-1]				) - 1)				)
	return output


def 							SCREAMING_SNAKE_CASE__  (      __a				,	__a				,	__a				,	__a				,	__a				,	__a				):
	snake_case_						:     List[Any]     =		[]
	for dataset in encoded_datasets:
		snake_case_						:     List[str]     =		len(__a				)
		snake_case_						:     Dict     =		np.zeros((n_batch, 2, input_len)				,	dtype=np.intaa				)
		snake_case_						:     Optional[Any]     =		np.zeros((n_batch, 2)				,	dtype=np.intaa				)
		snake_case_						:     Any     =		np.full((n_batch, 2, input_len)				,	fill_value=-1_00				,	dtype=np.intaa				)
		snake_case_						:     Union[str, Any]     =		np.zeros((n_batch,)				,	dtype=np.intaa				)
		for (
		    i,
		    (story, conta, conta, mc_label),
		) in enumerate(__a				):
			snake_case_						:     Tuple     =		[start_token] + story[:cap_length] + [delimiter_token] + conta[:cap_length] + [clf_token]
			snake_case_						:     Optional[Any]     =		[start_token] + story[:cap_length] + [delimiter_token] + conta[:cap_length] + [clf_token]
			snake_case_						:     Optional[Any]     =		with_conta
			snake_case_						:     Dict     =		with_conta
			snake_case_						:     List[str]     =		len(__a				) - 1
			snake_case_						:     List[Any]     =		len(__a				) - 1
			snake_case_						:     Union[str, Any]     =		with_conta
			snake_case_						:     Optional[Any]     =		with_conta
			snake_case_						:     Dict     =		mc_label
		snake_case_						:     Tuple     =		(input_ids, mc_token_ids, lm_labels, mc_labels)
		tensor_datasets.append(tuple(torch.tensor(__a				) for t in all_inputs				)				)
	return tensor_datasets


def 							SCREAMING_SNAKE_CASE__  (      ):
	snake_case_						:     Union[str, Any]     =		argparse.ArgumentParser()
	parser.add_argument('--model_name'				,	type=__a				,	default='openai-gpt'				,	help='pretrained model name'				)
	parser.add_argument('--do_train'				,	action='store_true'				,	help='Whether to run training.'				)
	parser.add_argument('--do_eval'				,	action='store_true'				,	help='Whether to run eval on the dev set.'				)
	parser.add_argument(
	    '--output_dir'				,	default=__a				,	type=__a				,	required=__a				,	help='The output directory where the model predictions and checkpoints will be written.'				,	)
	parser.add_argument('--train_dataset'				,	type=__a				,	default=''				)
	parser.add_argument('--eval_dataset'				,	type=__a				,	default=''				)
	parser.add_argument('--seed'				,	type=__a				,	default=42				)
	parser.add_argument('--num_train_epochs'				,	type=__a				,	default=3				)
	parser.add_argument('--train_batch_size'				,	type=__a				,	default=8				)
	parser.add_argument('--eval_batch_size'				,	type=__a				,	default=16				)
	parser.add_argument('--adam_epsilon'				,	default=1E-8				,	type=__a				,	help='Epsilon for Adam optimizer.'				)
	parser.add_argument('--max_grad_norm'				,	type=__a				,	default=1				)
	parser.add_argument(
	    '--max_steps'				,	default=-1				,	type=__a				,	help=(
	        'If > 0: set total number of training                         steps to perform. Override num_train_epochs.'
	    )				,	)
	parser.add_argument(
	    '--gradient_accumulation_steps'				,	type=__a				,	default=1				,	help='Number of updates steps to accumulate before                        performing a backward/update pass.'				,	)
	parser.add_argument('--learning_rate'				,	type=__a				,	default=6.25E-5				)
	parser.add_argument('--warmup_steps'				,	default=0				,	type=__a				,	help='Linear warmup over warmup_steps.'				)
	parser.add_argument('--lr_schedule'				,	type=__a				,	default='warmup_linear'				)
	parser.add_argument('--weight_decay'				,	type=__a				,	default=0.01				)
	parser.add_argument('--lm_coef'				,	type=__a				,	default=0.9				)
	parser.add_argument('--n_valid'				,	type=__a				,	default=3_74				)

	parser.add_argument('--server_ip'				,	type=__a				,	default=''				,	help='Can be used for distant debugging.'				)
	parser.add_argument('--server_port'				,	type=__a				,	default=''				,	help='Can be used for distant debugging.'				)
	snake_case_						:     Optional[int]     =		parser.parse_args()
	print(__a				)

	if args.server_ip and args.server_port:
		# Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
		import ptvsd

		print('Waiting for debugger attach'				)
		ptvsd.enable_attach(address=(args.server_ip, args.server_port)				,	redirect_output=__a				)
		ptvsd.wait_for_attach()

	random.seed(args.seed				)
	np.random.seed(args.seed				)
	torch.manual_seed(args.seed				)
	torch.cuda.manual_seed_all(args.seed				)

	snake_case_						:     Union[str, Any]     =		torch.device('cuda' if torch.cuda.is_available() else 'cpu'				)
	snake_case_						:     str     =		torch.cuda.device_count()
	logger.info('device: {}, n_gpu {}'.format(__a				,	__a				)				)

	if not args.do_train and not args.do_eval:
		raise ValueError('At least one of `do_train` or `do_eval` must be True.'				)

	if not os.path.exists(args.output_dir				):
		os.makedirs(args.output_dir				)

	# Load tokenizer and model
	# This loading functions also add new tokens and embeddings called `special tokens`
	# These new embeddings will be fine-tuned on the RocStories dataset
	snake_case_						:     Tuple     =		['_start_', '_delimiter_', '_classify_']
	snake_case_						:     Optional[int]     =		OpenAIGPTTokenizer.from_pretrained(args.model_name				)
	tokenizer.add_tokens(__a				)
	snake_case_						:     Dict     =		tokenizer.convert_tokens_to_ids(__a				)
	snake_case_						:     Union[str, Any]     =		OpenAIGPTDoubleHeadsModel.from_pretrained(args.model_name				)
	model.resize_token_embeddings(len(__a				)				)
	model.to(__a				)

	# Load and encode the datasets
	def tokenize_and_encode(__a				):
		if isinstance(__a				,	__a				):
			return tokenizer.convert_tokens_to_ids(tokenizer.tokenize(__a				)				)
		elif isinstance(__a				,	__a				):
			return obj
		return [tokenize_and_encode(__a				) for o in obj]

	logger.info('Encoding dataset...'				)
	snake_case_						:     Optional[Any]     =		load_rocstories_dataset(args.train_dataset				)
	snake_case_						:     Optional[int]     =		load_rocstories_dataset(args.eval_dataset				)
	snake_case_						:     List[str]     =		(train_dataset, eval_dataset)
	snake_case_						:     int     =		tokenize_and_encode(__a				)

	# Compute the max input length for the Transformer
	snake_case_						:     Optional[int]     =		model.config.n_positions // 2 - 2
	snake_case_						:     Optional[Any]     =		max(
	    len(story[:max_length]				) + max(len(conta[:max_length]				)				,	len(conta[:max_length]				)				) + 3
	    for dataset in encoded_datasets
	    for story, conta, conta, _ in dataset				)
	snake_case_						:     str     =		min(__a				,	model.config.n_positions				)  # Max size of input for the pre-trained model

	# Prepare inputs tensors and dataloaders
	snake_case_						:     Any     =		pre_process_datasets(__a				,	__a				,	__a				,	*__a				)
	snake_case_  ,snake_case_						:     Dict     =		tensor_datasets[0], tensor_datasets[1]

	snake_case_						:     Any     =		TensorDataset(*__a				)
	snake_case_						:     str     =		RandomSampler(__a				)
	snake_case_						:     Any     =		DataLoader(__a				,	sampler=__a				,	batch_size=args.train_batch_size				)

	snake_case_						:     Tuple     =		TensorDataset(*__a				)
	snake_case_						:     Optional[Any]     =		SequentialSampler(__a				)
	snake_case_						:     Optional[int]     =		DataLoader(__a				,	sampler=__a				,	batch_size=args.eval_batch_size				)

	# Prepare optimizer
	if args.do_train:
		if args.max_steps > 0:
			snake_case_						:     Optional[Any]     =		args.max_steps
			snake_case_						:     Tuple     =		args.max_steps // (len(__a				) // args.gradient_accumulation_steps) + 1
		else:
			snake_case_						:     int     =		len(__a				) // args.gradient_accumulation_steps * args.num_train_epochs

		snake_case_						:     Optional[Any]     =		list(model.named_parameters()				)
		snake_case_						:     Optional[Any]     =		['bias', 'LayerNorm.bias', 'LayerNorm.weight']
		snake_case_						:     Optional[Any]     =		[
		    {
		        'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay				)],
		        'weight_decay': args.weight_decay,
		    },
		    {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay				)], 'weight_decay': 0.0},
		]
		snake_case_						:     str     =		AdamW(__a				,	lr=args.learning_rate				,	eps=args.adam_epsilon				)
		snake_case_						:     Union[str, Any]     =		get_linear_schedule_with_warmup(
		    __a				,	num_warmup_steps=args.warmup_steps				,	num_training_steps=__a				)

	if args.do_train:
		snake_case_  ,snake_case_  ,snake_case_						:     Union[str, Any]     =		0, 0, None
		model.train()
		for _ in trange(int(args.num_train_epochs				)				,	desc='Epoch'				):
			snake_case_						:     int     =		0
			snake_case_						:     str     =		0
			snake_case_						:     Tuple     =		tqdm(__a				,	desc='Training'				)
			for step, batch in enumerate(__a				):
				snake_case_						:     Union[str, Any]     =		tuple(t.to(__a				) for t in batch				)
				snake_case_  ,snake_case_  ,snake_case_  ,snake_case_						:     Any     =		batch
				snake_case_						:     Tuple     =		model(__a				,	mc_token_ids=__a				,	lm_labels=__a				,	mc_labels=__a				)
				snake_case_						:     Optional[Any]     =		args.lm_coef * losses[0] + losses[1]
				loss.backward()
				optimizer.step()
				scheduler.step()
				optimizer.zero_grad()
				tr_loss += loss.item()
				snake_case_						:     List[str]     =		(
				    loss.item() if exp_average_loss is None else 0.7 * exp_average_loss + 0.3 * loss.item()
				)
				nb_tr_steps += 1
				snake_case_						:     List[str]     =		'Training loss: {:.2e} lr: {:.2e}'.format(__a				,	scheduler.get_lr()[0]				)

    # Save a trained model
	if args.do_train:
		# Save a trained model, configuration and tokenizer
		snake_case_						:     Optional[int]     =		model.module if hasattr(__a				,	'module'				) else model  # Only save the model itself

		# If we save using the predefined names, we can load using `from_pretrained`
		snake_case_						:     Optional[Any]     =		os.path.join(args.output_dir				,	__a				)
		snake_case_						:     Union[str, Any]     =		os.path.join(args.output_dir				,	__a				)

		torch.save(model_to_save.state_dict()				,	__a				)
		model_to_save.config.to_json_file(__a				)
		tokenizer.save_vocabulary(args.output_dir				)

		# Load a trained model and vocabulary that you have fine-tuned
		snake_case_						:     str     =		OpenAIGPTDoubleHeadsModel.from_pretrained(args.output_dir				)
		snake_case_						:     List[str]     =		OpenAIGPTTokenizer.from_pretrained(args.output_dir				)
		model.to(__a				)

	if args.do_eval:
		model.eval()
		snake_case_  ,snake_case_						:     List[Any]     =		0, 0
		snake_case_  ,snake_case_						:     Optional[int]     =		0, 0
		for batch in tqdm(__a				,	desc='Evaluating'				):
			snake_case_						:     Dict     =		tuple(t.to(__a				) for t in batch				)
			snake_case_  ,snake_case_  ,snake_case_  ,snake_case_						:     Optional[Any]     =		batch
			with torch.no_grad():
				snake_case_  ,snake_case_  ,snake_case_  ,snake_case_						:     Tuple     =		model(
				    __a				,	mc_token_ids=__a				,	lm_labels=__a				,	mc_labels=__a				)

			snake_case_						:     Optional[int]     =		mc_logits.detach().cpu().numpy()
			snake_case_						:     List[Any]     =		mc_labels.to('cpu'				).numpy()
			snake_case_						:     str     =		accuracy(__a				,	__a				)

			eval_loss += mc_loss.mean().item()
			eval_accuracy += tmp_eval_accuracy

			nb_eval_examples += input_ids.size(0				)
			nb_eval_steps += 1

		snake_case_						:     str     =		eval_loss / nb_eval_steps
		snake_case_						:     Tuple     =		eval_accuracy / nb_eval_examples
		snake_case_						:     Tuple     =		tr_loss / nb_tr_steps if args.do_train else None
		snake_case_						:     Optional[int]     =		{'eval_loss': eval_loss, 'eval_accuracy': eval_accuracy, 'train_loss': train_loss}

		snake_case_						:     List[str]     =		os.path.join(args.output_dir				,	'eval_results.txt'				)
		with open(__a				,	'w'				) as writer:
			logger.info('***** Eval results *****'				)
			for key in sorted(result.keys()				):
				logger.info('  %s = %s'				,	__a				,	str(result[key]				)				)
				writer.write('%s = %s\n' % (key, str(result[key]				))				)


if __name__ == "__main__":
						main()








import json
import multiprocessing
import os
import re
from collections import defaultdict

import torch
from accelerate import Accelerator
from accelerate.utils import set_seed
from arguments import HumanEvalArguments
from datasets import load_dataset, load_metric
from torch.utils.data import IterableDataset
from torch.utils.data.dataloader import DataLoader
from tqdm import tqdm

import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer, HfArgumentParser, StoppingCriteria, StoppingCriteriaList


_SCREAMING_SNAKE_CASE 		=						["""\nclass""", """\ndef""", """\n#""", """\n@""", """\nprint""", """\nif"""]
class SCREAMING_SNAKE_CASE_  (   snake_case_   ):






						def __init__( self : Tuple						,				_A : Any						,				_A : str						,				_A : int=None						,				_A : str=1		)							->      List[str]:







							"""simple docstring"""


							snake_case_						:     Union[str, Any]     =		tokenizer
							snake_case_						:     Optional[int]     =		dataset
							snake_case_						:     List[str]     =		len(_A		) if n_tasks is None else n_tasks
							snake_case_						:     List[Any]     =		n_copies






						def __iter__( self : Any		)							->      List[str]:







							"""simple docstring"""


							snake_case_						:     List[str]     =		[]
							for task in range(self.n_tasks		):
								# without strip, the model generate commented codes ...
								prompts.append(self.tokenizer.eos_token + self.dataset[task]['prompt'].strip()		)
							snake_case_						:     Optional[int]     =		self.tokenizer(_A						,				padding=_A						,				return_tensors='pt'		)
							for task in range(self.n_tasks		):
								for _ in range(self.n_copies		):
									yield {
									    "ids": outputs.input_ids[task],
									    "task_id": task,
									    "input_len": outputs.attention_mask[task].sum(),
									}
class SCREAMING_SNAKE_CASE_  (   snake_case_   ):






						def __init__( self : List[Any]						,				_A : Optional[int]						,				_A : str						,				_A : Dict		)							->      Any:







							"""simple docstring"""


							snake_case_						:     List[str]     =		start_length
							snake_case_						:     int     =		eof_strings
							snake_case_						:     Dict     =		tokenizer






						def __call__( self : Any						,				_A : Union[str, Any]						,				_A : Dict						,				**_A : List[Any]		)							->      List[str]:







							"""simple docstring"""


							snake_case_						:     Optional[int]     =		self.tokenizer.batch_decode(input_ids[:, self.start_length :]		)
							snake_case_						:     Optional[int]     =		[]
							for decoded_generation in decoded_generations:
								done.append(any(stop_string in decoded_generation for stop_string in self.eof_strings		)		)
							return all(_A		)


def 							SCREAMING_SNAKE_CASE__  (      __a				):
	snake_case_						:     List[str]     =		re.split('(%s)' % '|'.join(__a				)				,	__a				)
	# last string should be ""
	return "".join(string_list[:-2]				)


def 							SCREAMING_SNAKE_CASE__  (      __a				,	__a				,	__a				,	__a				,	__a				,	__a=20				,	**__a				):
	snake_case_						:     Tuple     =		defaultdict(__a				)  # dict of list of generated tokens
	for step, batch in tqdm(enumerate(__a				)				):
		with torch.no_grad():
			snake_case_						:     Optional[Any]     =		batch['ids'].shape[-1]
			snake_case_						:     List[str]     =		accelerator.unwrap_model(__a				).generate(
			    input_ids=batch['ids'][:, : batch['input_len']]				,	num_return_sequences=__a				,	**__a				)
			# each task is generated batch_size times
			snake_case_						:     List[str]     =		batch['task_id'].repeat(__a				)
			snake_case_						:     Union[str, Any]     =		accelerator.pad_across_processes(
			    __a				,	dim=1				,	pad_index=tokenizer.pad_token_id				)

			snake_case_  ,snake_case_						:     Union[str, Any]     =		accelerator.gather((generated_tokens, generated_tasks)				)
			snake_case_						:     Optional[Any]     =		generated_tokens.cpu().numpy()
			snake_case_						:     Dict     =		generated_tasks.cpu().numpy()

			for task, generated_tokens in zip(__a				,	__a				):
				gen_token_dict[task].append(__a				)

	snake_case_						:     Tuple     =		[[] for _ in range(__a				)]
	for task, generated_tokens in gen_token_dict.items():
		for s in generated_tokens:
			snake_case_						:     int     =		tokenizer.decode(__a				,	skip_special_tokens=__a				,	clean_up_tokenization_spaces=__a				)
			code_gens[task].append(remove_last_block(__a				)				)
	return code_gens


def 							SCREAMING_SNAKE_CASE__  (      ):
	# Setup configuration
	snake_case_						:     Optional[int]     =		HfArgumentParser(__a				)
	snake_case_						:     Tuple     =		parser.parse_args()

	transformers.logging.set_verbosity_error()
	# enables code execution in code_eval metric
	snake_case_						:     Optional[int]     =		args.HF_ALLOW_CODE_EVAL
	# make sure tokenizer plays nice with multiprocessing
	snake_case_						:     int     =		'false'

	if args.num_workers is None:
		snake_case_						:     Any     =		multiprocessing.cpu_count()

	# Use dataset load to feed to accelerate
	snake_case_						:     List[Any]     =		Accelerator()
	set_seed(args.seed				,	device_specific=__a				)

	# Load model and tokenizer
	snake_case_						:     Any     =		AutoTokenizer.from_pretrained(args.model_ckpt				)
	snake_case_						:     int     =		tokenizer.eos_token
	snake_case_						:     List[str]     =		AutoModelForCausalLM.from_pretrained(args.model_ckpt				)

	# Generation settings
	snake_case_						:     List[Any]     =		{
	    'do_sample': args.do_sample,
	    'temperature': args.temperature,
	    'max_new_tokens': args.max_new_tokens,
	    'top_p': args.top_p,
	    'top_k': args.top_k,
	    'stopping_criteria': StoppingCriteriaList([EndOfFunctionCriteria(0				,	__a				,	__a				)]				),
	}

	# Load evaluation dataset and metric
	snake_case_						:     Dict     =		load_dataset('openai_humaneval'				)
	snake_case_						:     Optional[Any]     =		load_metric('code_eval'				)

	snake_case_						:     Union[str, Any]     =		args.num_tasks if args.num_tasks is not None else len(human_eval['test']				)
	snake_case_						:     Optional[int]     =		args.n_samples // args.batch_size

	snake_case_						:     Dict     =		TokenizedDataset(__a				,	human_eval['test']				,	n_copies=__a				,	n_tasks=__a				)
	# do not confuse args.batch_size, which is actually the num_return_sequences
	snake_case_						:     Optional[Any]     =		DataLoader(__a				,	batch_size=1				)

	# Run a quick test to see if code evaluation is enabled
	try:
		snake_case_						:     Union[str, Any]     =		code_eval_metric.compute(references=['']				,	predictions=[['']]				)
	except ValueError as exception:
		print(
		    'Code evaluation not enabled. Read the warning below carefully and then use `--HF_ALLOW_CODE_EVAL="1"`'
		    ' flag to enable code evaluation.'				)
		raise exception

	snake_case_  ,snake_case_						:     Union[str, Any]     =		accelerator.prepare(__a				,	__a				)

	snake_case_						:     str     =		complete_code(
	    __a				,	__a				,	__a				,	__a				,	n_tasks=__a				,	batch_size=args.batch_size				,	**__a				,	)

	if accelerator.is_main_process:
		snake_case_						:     Tuple     =		[]

		for task in tqdm(range(__a				)				):
			snake_case_						:     Union[str, Any]     =		human_eval['test'][task]['test']
			snake_case_						:     Union[str, Any]     =		f"""check({human_eval['test'][task]['entry_point']})"""
			references.append('\n' + test_func + '\n' + entry_point				)

		# Evaluate completions with "code_eval" metric
		snake_case_  ,snake_case_						:     int     =		code_eval_metric.compute(
		    references=__a				,	predictions=__a				,	num_workers=args.num_workers				)
		print(f"""Results: {pass_at_k}"""				)

		# Save results to json file
		with open(args.output_file				,	'w'				) as fp:
			json.dump(__a				,	__a				)


# For some reason the folliwng seems to be necessary sometimes for code_eval to work nice with multiprocessing
# https://stackoverflow.com/questions/60804599/python-multiprocessing-keeps-spawning-the-whole-script
if __name__ == "__main__":
						main()

