



import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


lowerCAmelCase__					  =  re.compile(r'''\b(a|an|the)\b''', re.UNICODE)

lowerCAmelCase__					  =  None


def  __lowerCamelCase (       ):

				"""simple docstring"""


				lowercase__					:   Tuple													=					argparse.ArgumentParser("Official evaluation script for SQuAD version 2.0."						)
				parser.add_argument("data_file"					,			metavar="data.json"					,			help="Input data JSON file."						)
				parser.add_argument("pred_file"					,			metavar="pred.json"					,			help="Model predictions."						)
				parser.add_argument(
				    "--out-file"					,			"-o"					,			metavar="eval.json"					,			help="Write accuracy metrics to file (default is stdout)."						)
				parser.add_argument(
				    "--na-prob-file"					,			"-n"					,			metavar="na_prob.json"					,			help="Model estimates of probability of no answer."						)
				parser.add_argument(
				    "--na-prob-thresh"					,			"-t"					,			type=lowerCamelCase__					,			default=1.0					,			help="Predict \"\" if no-answer probability exceeds this (default = 1.0)."					,			)
				parser.add_argument(
				    "--out-image-dir"					,			"-p"					,			metavar="out_images"					,			default=lowerCamelCase__					,			help="Save precision-recall curves to directory."						)
				parser.add_argument("--verbose"					,			"-v"					,			action="store_true"						)
				if len(sys.argv						) == 1:
								parser.print_help()
								sys.exit(1						)
				return parser.parse_args()


def  __lowerCamelCase (       lowerCamelCase__						):

				"""simple docstring"""


				lowercase__					:   List[Any]													=					{}
				for article in dataset:
								for p in article["paragraphs"]:
												for qa in p["qas"]:
																lowercase__					:   Dict													=					bool(qa["answers"]["text"]						)
				return qid_to_has_ans


def  __lowerCamelCase (       lowerCamelCase__						):

				"""simple docstring"""


				def remove_articles(lowerCamelCase__						):
								return ARTICLES_REGEX.sub(" "					,			lowerCamelCase__						)

				def white_space_fix(lowerCamelCase__						):
								return " ".join(text.split()						)

				def remove_punc(lowerCamelCase__						):
								lowercase__					:   int													=					set(string.punctuation						)
								return "".join(ch for ch in text if ch not in exclude						)

				def lower(lowerCamelCase__						):
								return text.lower()

				return white_space_fix(remove_articles(remove_punc(lower(lowerCamelCase__						)						)						)						)


def  __lowerCamelCase (       lowerCamelCase__						):

				"""simple docstring"""


				if not s:
								return []
				return normalize_answer(lowerCamelCase__						).split()


def  __lowerCamelCase (       lowerCamelCase__					,			lowerCamelCase__						):

				"""simple docstring"""


				return int(normalize_answer(lowerCamelCase__						) == normalize_answer(lowerCamelCase__						)						)


def  __lowerCamelCase (       lowerCamelCase__					,			lowerCamelCase__						):

				"""simple docstring"""


				lowercase__					:   int													=					get_tokens(lowerCamelCase__						)
				lowercase__					:   List[Any]													=					get_tokens(lowerCamelCase__						)
				lowercase__					:   Dict													=					collections.Counter(lowerCamelCase__						) & collections.Counter(lowerCamelCase__						)
				lowercase__					:   Dict													=					sum(common.values()						)
				if len(lowerCamelCase__						) == 0 or len(lowerCamelCase__						) == 0:
								# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
								return int(gold_toks == pred_toks						)
				if num_same == 0:
								return 0
				lowercase__					:   int													=					1.0 * num_same / len(lowerCamelCase__						)
				lowercase__					:   List[str]													=					1.0 * num_same / len(lowerCamelCase__						)
				lowercase__					:   Union[str, Any]													=					(2 * precision * recall) / (precision + recall)
				return fa


def  __lowerCamelCase (       lowerCamelCase__					,			lowerCamelCase__						):

				"""simple docstring"""


				lowercase__					:   Union[str, Any]													=					{}
				lowercase__					:   str													=					{}
				for article in dataset:
								for p in article["paragraphs"]:
												for qa in p["qas"]:
																lowercase__					:   int													=					qa["id"]
																lowercase__					:   str													=					[t for t in qa["answers"]["text"] if normalize_answer(lowerCamelCase__						)]
																if not gold_answers:
																				# For unanswerable questions, only correct answer is empty string
																				lowercase__					:   Optional[int]													=					[""]
																if qid not in preds:
																				print(F"""Missing prediction for {qid}"""						)
																				continue
																lowercase__					:   Tuple													=					preds[qid]
																# Take max over all gold answers
																lowercase__					:   Tuple													=					max(compute_exact(lowerCamelCase__					,			lowerCamelCase__						) for a in gold_answers						)
																lowercase__					:   Optional[int]													=					max(compute_fa(lowerCamelCase__					,			lowerCamelCase__						) for a in gold_answers						)
				return exact_scores, fa_scores


def  __lowerCamelCase (       lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__						):

				"""simple docstring"""


				lowercase__					:   int													=					{}
				for qid, s in scores.items():
								lowercase__					:   Optional[Any]													=					na_probs[qid] > na_prob_thresh
								if pred_na:
												lowercase__					:   Optional[int]													=					float(not qid_to_has_ans[qid]						)
								else:
												lowercase__					:   List[Any]													=					s
				return new_scores


def  __lowerCamelCase (       lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__=None						):

				"""simple docstring"""


				if not qid_list:
								lowercase__					:   int													=					len(lowerCamelCase__						)
								return collections.OrderedDict(
								    [
								        ("exact", 100.0 * sum(exact_scores.values()						) / total),
								        ("f1", 100.0 * sum(fa_scores.values()						) / total),
								        ("total", total),
								    ]						)
				else:
								lowercase__					:   int													=					len(lowerCamelCase__						)
								return collections.OrderedDict(
								    [
								        ("exact", 100.0 * sum(exact_scores[k] for k in qid_list						) / total),
								        ("f1", 100.0 * sum(fa_scores[k] for k in qid_list						) / total),
								        ("total", total),
								    ]						)


def  __lowerCamelCase (       lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__						):

				"""simple docstring"""


				for k in new_eval:
								lowercase__					:   Optional[int]													=					new_eval[k]


def  __lowerCamelCase (       lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__						):

				"""simple docstring"""


				plt.step(lowerCamelCase__					,			lowerCamelCase__					,			color="b"					,			alpha=0.2					,			where="post"						)
				plt.fill_between(lowerCamelCase__					,			lowerCamelCase__					,			step="post"					,			alpha=0.2					,			color="b"						)
				plt.xlabel("Recall"						)
				plt.ylabel("Precision"						)
				plt.xlim([0.0, 1.05]						)
				plt.ylim([0.0, 1.05]						)
				plt.title(lowerCamelCase__						)
				plt.savefig(lowerCamelCase__						)
				plt.clf()


def  __lowerCamelCase (       lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__=None					,			lowerCamelCase__=None						):

				"""simple docstring"""


				lowercase__					:   Union[str, Any]													=					sorted(lowerCamelCase__					,			key=lambda lowerCamelCase__						: na_probs[k]						)
				lowercase__					:   List[Any]													=					0.0
				lowercase__					:   Union[str, Any]													=					1.0
				lowercase__					:   List[str]													=					0.0
				lowercase__					:   Tuple													=					[1.0]
				lowercase__					:   Optional[Any]													=					[0.0]
				lowercase__					:   Any													=					0.0
				for i, qid in enumerate(lowerCamelCase__						):
								if qid_to_has_ans[qid]:
												true_pos += scores[qid]
								lowercase__					:   Dict													=					true_pos / float(i + 1						)
								lowercase__					:   List[Any]													=					true_pos / float(lowerCamelCase__						)
								if i == len(lowerCamelCase__						) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
												# i.e., if we can put a threshold after this point
												avg_prec += cur_p * (cur_r - recalls[-1])
												precisions.append(lowerCamelCase__						)
												recalls.append(lowerCamelCase__						)
				if out_image:
								plot_pr_curve(lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__						)
				return {"ap": 100.0 * avg_prec}


def  __lowerCamelCase (       lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__						):

				"""simple docstring"""


				if out_image_dir and not os.path.exists(lowerCamelCase__						):
								os.makedirs(lowerCamelCase__						)
				lowercase__					:   Optional[int]													=					sum(1 for v in qid_to_has_ans.values() if v						)
				if num_true_pos == 0:
								return
				lowercase__					:   Any													=					make_precision_recall_eval(
				    lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__					,			out_image=os.path.join(lowerCamelCase__					,			"pr_exact.png"						)					,			title="Precision-Recall curve for Exact Match score"					,			)
				lowercase__					:   str													=					make_precision_recall_eval(
				    lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__					,			out_image=os.path.join(lowerCamelCase__					,			"pr_f1.png"						)					,			title="Precision-Recall curve for F1 score"					,			)
				lowercase__					:   List[Any]													=					{k: float(lowerCamelCase__						) for k, v in qid_to_has_ans.items()}
				lowercase__					:   int													=					make_precision_recall_eval(
				    lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__					,			out_image=os.path.join(lowerCamelCase__					,			"pr_oracle.png"						)					,			title="Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)"					,			)
				merge_eval(lowerCamelCase__					,			lowerCamelCase__					,			"pr_exact"						)
				merge_eval(lowerCamelCase__					,			lowerCamelCase__					,			"pr_f1"						)
				merge_eval(lowerCamelCase__					,			lowerCamelCase__					,			"pr_oracle"						)


def  __lowerCamelCase (       lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__						):

				"""simple docstring"""


				if not qid_list:
								return
				lowercase__					:   List[Any]													=					[na_probs[k] for k in qid_list]
				lowercase__					:   int													=					np.ones_like(lowerCamelCase__						) / float(len(lowerCamelCase__						)						)
				plt.hist(lowerCamelCase__					,			weights=lowerCamelCase__					,			bins=20					,			range=(0.0, 1.0)						)
				plt.xlabel("Model probability of no-answer"						)
				plt.ylabel("Proportion of dataset"						)
				plt.title(F"""Histogram of no-answer probability: {name}"""						)
				plt.savefig(os.path.join(lowerCamelCase__					,			F"""na_prob_hist_{name}.png"""						)						)
				plt.clf()


def  __lowerCamelCase (       lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__						):

				"""simple docstring"""


				lowercase__					:   List[str]													=					sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]						)
				lowercase__					:   List[str]													=					num_no_ans
				lowercase__					:   Union[str, Any]													=					cur_score
				lowercase__					:   Optional[Any]													=					0.0
				lowercase__					:   int													=					sorted(lowerCamelCase__					,			key=lambda lowerCamelCase__						: na_probs[k]						)
				for i, qid in enumerate(lowerCamelCase__						):
								if qid not in scores:
												continue
								if qid_to_has_ans[qid]:
												lowercase__					:   Optional[Any]													=					scores[qid]
								else:
												if preds[qid]:
																lowercase__					:   Optional[int]													=					-1
												else:
																lowercase__					:   int													=					0
								cur_score += diff
								if cur_score > best_score:
												lowercase__					:   Tuple													=					cur_score
												lowercase__					:   List[Any]													=					na_probs[qid]
				return 100.0 * best_score / len(lowerCamelCase__						), best_thresh


def  __lowerCamelCase (       lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__						):

				"""simple docstring"""


				lowercase__							,							lowercase__					:   Optional[Any]													=					find_best_thresh(lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__						)
				lowercase__							,							lowercase__					:   str													=					find_best_thresh(lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__						)
				lowercase__					:   int													=					best_exact
				lowercase__					:   Optional[Any]													=					exact_thresh
				lowercase__					:   Union[str, Any]													=					best_fa
				lowercase__					:   int													=					fa_thresh


def  __lowerCamelCase (       ):

				"""simple docstring"""


				with open(OPTS.data_file						) as f:
								lowercase__					:   Optional[int]													=					json.load(lowerCamelCase__						)
								lowercase__					:   Any													=					dataset_json["data"]
				with open(OPTS.pred_file						) as f:
								lowercase__					:   Any													=					json.load(lowerCamelCase__						)
				if OPTS.na_prob_file:
								with open(OPTS.na_prob_file						) as f:
												lowercase__					:   Optional[Any]													=					json.load(lowerCamelCase__						)
				else:
								lowercase__					:   Union[str, Any]													=					{k: 0.0 for k in preds}
				lowercase__					:   List[str]													=					make_qid_to_has_ans(lowerCamelCase__						)  # maps qid to True/False
				lowercase__					:   Tuple													=					[k for k, v in qid_to_has_ans.items() if v]
				lowercase__					:   Any													=					[k for k, v in qid_to_has_ans.items() if not v]
				lowercase__							,							lowercase__					:   List[str]													=					get_raw_scores(lowerCamelCase__					,			lowerCamelCase__						)
				lowercase__					:   List[str]													=					apply_no_ans_threshold(lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__					,			OPTS.na_prob_thresh						)
				lowercase__					:   Dict													=					apply_no_ans_threshold(lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__					,			OPTS.na_prob_thresh						)
				lowercase__					:   List[str]													=					make_eval_dict(lowerCamelCase__					,			lowerCamelCase__						)
				if has_ans_qids:
								lowercase__					:   List[Any]													=					make_eval_dict(lowerCamelCase__					,			lowerCamelCase__					,			qid_list=lowerCamelCase__						)
								merge_eval(lowerCamelCase__					,			lowerCamelCase__					,			"HasAns"						)
				if no_ans_qids:
								lowercase__					:   Any													=					make_eval_dict(lowerCamelCase__					,			lowerCamelCase__					,			qid_list=lowerCamelCase__						)
								merge_eval(lowerCamelCase__					,			lowerCamelCase__					,			"NoAns"						)
				if OPTS.na_prob_file:
								find_all_best_thresh(lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__						)
				if OPTS.na_prob_file and OPTS.out_image_dir:
								run_precision_recall_analysis(lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__					,			lowerCamelCase__					,			OPTS.out_image_dir						)
								histogram_na_prob(lowerCamelCase__					,			lowerCamelCase__					,			OPTS.out_image_dir					,			"hasAns"						)
								histogram_na_prob(lowerCamelCase__					,			lowerCamelCase__					,			OPTS.out_image_dir					,			"noAns"						)
				if OPTS.out_file:
								with open(OPTS.out_file					,			"w"						) as f:
												json.dump(lowerCamelCase__					,			lowerCamelCase__						)
				else:
								print(json.dumps(lowerCamelCase__					,			indent=2						)						)


if __name__ == "__main__":
				lowerCAmelCase__					  =  parse_args()
				if OPTS.out_image_dir:
								import matplotlib

								matplotlib.use('''Agg''')
								import matplotlib.pyplot as plt
				main()









import os
from argparse import ArgumentParser, Namespace

from ..data import SingleSentenceClassificationProcessor as Processor
from ..pipelines import TextClassificationPipeline
from ..utils import is_tf_available, is_torch_available, logging
from . import BaseTransformersCLICommand


if not is_tf_available() and not is_torch_available():
				raise RuntimeError('''At least one of PyTorch or TensorFlow 2.0+ should be installed to use CLI training''')

# TF training parameters
lowerCAmelCase__					  =  False
lowerCAmelCase__					  =  False


def  __lowerCamelCase (       lowerCamelCase__						):

				"""simple docstring"""


				return TrainCommand(lowerCamelCase__						)

class 				snake_case__(_UpperCamelCase      ):

					"""simple docstring"""






					@staticmethod
					def 			snake_case					(							SCREAMING_SNAKE_CASE :     ArgumentParser					):
									lowercase__					:   Optional[int]													=					parser.add_parser("train"				,       help="CLI tool to train a model on a task."					)

									train_parser.add_argument(
									    "--train_data"				,       type=SCREAMING_SNAKE_CASE				,       required=SCREAMING_SNAKE_CASE				,       help="path to train (and optionally evaluation) dataset as a csv with tab separated labels and sentences."				,       )
									train_parser.add_argument(
									    "--column_label"				,       type=SCREAMING_SNAKE_CASE				,       default=0				,       help="Column of the dataset csv file with example labels."					)
									train_parser.add_argument(
									    "--column_text"				,       type=SCREAMING_SNAKE_CASE				,       default=1				,       help="Column of the dataset csv file with example texts."					)
									train_parser.add_argument(
									    "--column_id"				,       type=SCREAMING_SNAKE_CASE				,       default=2				,       help="Column of the dataset csv file with example ids."					)
									train_parser.add_argument(
									    "--skip_first_row"				,       action="store_true"				,       help="Skip the first row of the csv file (headers)."					)

									train_parser.add_argument("--validation_data"				,       type=SCREAMING_SNAKE_CASE				,       default=""				,       help="path to validation dataset."					)
									train_parser.add_argument(
									    "--validation_split"				,       type=SCREAMING_SNAKE_CASE				,       default=0.1				,       help="if validation dataset is not provided, fraction of train dataset to use as validation dataset."				,       )

									train_parser.add_argument("--output"				,       type=SCREAMING_SNAKE_CASE				,       default="./"				,       help="path to saved the trained model."					)

									train_parser.add_argument(
									    "--task"				,       type=SCREAMING_SNAKE_CASE				,       default="text_classification"				,       help="Task to train the model on."					)
									train_parser.add_argument(
									    "--model"				,       type=SCREAMING_SNAKE_CASE				,       default="bert-base-uncased"				,       help="Model's name or path to stored model."					)
									train_parser.add_argument("--train_batch_size"				,       type=SCREAMING_SNAKE_CASE				,       default=32				,       help="Batch size for training."					)
									train_parser.add_argument("--valid_batch_size"				,       type=SCREAMING_SNAKE_CASE				,       default=64				,       help="Batch size for validation."					)
									train_parser.add_argument("--learning_rate"				,       type=SCREAMING_SNAKE_CASE				,       default=3E-5				,       help="Learning rate."					)
									train_parser.add_argument("--adam_epsilon"				,       type=SCREAMING_SNAKE_CASE				,       default=1E-0_8				,       help="Epsilon for Adam optimizer."					)
									train_parser.set_defaults(func=SCREAMING_SNAKE_CASE					)







					def __init__(							self :     int				,       SCREAMING_SNAKE_CASE :     Namespace					):
									lowercase__					:   int													=					logging.get_logger("transformers-cli/training"					)

									lowercase__					:   List[Any]													=					"tf" if is_tf_available() else "torch"

									os.makedirs(args.output				,       exist_ok=SCREAMING_SNAKE_CASE					)
									lowercase__					:   Optional[Any]													=					args.output

									lowercase__					:   Union[str, Any]													=					args.column_label
									lowercase__					:   Optional[int]													=					args.column_text
									lowercase__					:   Optional[int]													=					args.column_id

									self.logger.info(f"""Loading {args.task} pipeline for {args.model}"""					)
									if args.task == "text_classification":
													lowercase__					:   int													=					TextClassificationPipeline.from_pretrained(args.model					)
									elif args.task == "token_classification":
													raise NotImplementedError
									elif args.task == "question_answering":
													raise NotImplementedError

									self.logger.info(f"""Loading dataset from {args.train_data}"""					)
									lowercase__					:   List[str]													=					Processor.create_from_csv(
									    args.train_data				,       column_label=args.column_label				,       column_text=args.column_text				,       column_id=args.column_id				,       skip_first_row=args.skip_first_row				,       )
									lowercase__					:   Union[str, Any]													=					None
									if args.validation_data:
													self.logger.info(f"""Loading validation dataset from {args.validation_data}"""					)
													lowercase__					:   Optional[int]													=					Processor.create_from_csv(
													    args.validation_data				,       column_label=args.column_label				,       column_text=args.column_text				,       column_id=args.column_id				,       skip_first_row=args.skip_first_row				,       )

									lowercase__					:   Dict													=					args.validation_split
									lowercase__					:   List[str]													=					args.train_batch_size
									lowercase__					:   Any													=					args.valid_batch_size
									lowercase__					:   Optional[int]													=					args.learning_rate
									lowercase__					:   int													=					args.adam_epsilon







					def 			snake_case					(							self :     Dict					):
									if self.framework == "tf":
													return self.run_tf()
									return self.run_torch()







					def 			snake_case					(							self :     Union[str, Any]					):
									raise NotImplementedError







					def 			snake_case					(							self :     Union[str, Any]					):
									self.pipeline.fit(
									    self.train_dataset				,       validation_data=self.valid_dataset				,       validation_split=self.validation_split				,       learning_rate=self.learning_rate				,       adam_epsilon=self.adam_epsilon				,       train_batch_size=self.train_batch_size				,       valid_batch_size=self.valid_batch_size				,       )

									# Save trained pipeline
									self.pipeline.save_pretrained(self.output					)




