

'''simple docstring'''
import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


lowerCamelCase			:				Optional[Any] 			=					re.compile(r"\b(a|an|the)\b", re.UNICODE)

lowerCamelCase			:				Optional[int] 			=					None


def 	_lowerCAmelCase (						)				->      str:
					"""simple docstring"""




					_SCREAMING_SNAKE_CASE							   =argparse.ArgumentParser('Official evaluation script for SQuAD version 2.0.'    )
					parser.add_argument('data_file'							,			metavar='data.json'							,			help='Input data JSON file.'    )
					parser.add_argument('pred_file'							,			metavar='pred.json'							,			help='Model predictions.'    )
					parser.add_argument(
					    '--out-file'							,			'-o'							,			metavar='eval.json'							,			help='Write accuracy metrics to file (default is stdout).'    )
					parser.add_argument(
					    '--na-prob-file'							,			'-n'							,			metavar='na_prob.json'							,			help='Model estimates of probability of no answer.'    )
					parser.add_argument(
					    '--na-prob-thresh'							,			'-t'							,			type=a__							,			default=1.0							,			help='Predict \"\" if no-answer probability exceeds this (default = 1.0).'							,			)
					parser.add_argument(
					    '--out-image-dir'							,			'-p'							,			metavar='out_images'							,			default=a__							,			help='Save precision-recall curves to directory.'    )
					parser.add_argument('--verbose'							,			'-v'							,			action='store_true'    )
					if len(sys.argv    ) == 1:
										parser.print_help()
										sys.exit(1    )
					return parser.parse_args()


def 	_lowerCAmelCase (						_UpperCamelCase						:					Tuple    )				->      Union[str, Any]:
					"""simple docstring"""




					_SCREAMING_SNAKE_CASE							   ={}
					for article in dataset:
										for p in article["paragraphs"]:
															for qa in p["qas"]:
																				_SCREAMING_SNAKE_CASE							   =bool(qa['answers']['text']    )
					return qid_to_has_ans


def 	_lowerCAmelCase (						_UpperCamelCase						:					List[Any]    )				->      Optional[Any]:
					"""simple docstring"""




					def remove_articles(_UpperCamelCase						:					str    ):
										return ARTICLES_REGEX.sub(' '							,			a__    )

					def white_space_fix(_UpperCamelCase						:					Optional[Any]    ):
										return " ".join(text.split()    )

					def remove_punc(_UpperCamelCase						:					Union[str, Any]    ):
										_SCREAMING_SNAKE_CASE							   =set(string.punctuation    )
										return "".join(ch for ch in text if ch not in exclude    )

					def lower(_UpperCamelCase						:					List[Any]    ):
										return text.lower()

					return white_space_fix(remove_articles(remove_punc(lower(a__    )    )    )    )


def 	_lowerCAmelCase (						_UpperCamelCase						:					List[str]    )				->      Union[str, Any]:
					"""simple docstring"""




					if not s:
										return []
					return normalize_answer(a__    ).split()


def 	_lowerCAmelCase (						_UpperCamelCase						:					List[Any]							,			_UpperCamelCase						:					int    )				->      List[str]:
					"""simple docstring"""




					return int(normalize_answer(a__    ) == normalize_answer(a__    )    )


def 	_lowerCAmelCase (						_UpperCamelCase						:					Any							,			_UpperCamelCase						:					Optional[Any]    )				->      List[str]:
					"""simple docstring"""




					_SCREAMING_SNAKE_CASE							   =get_tokens(a__    )
					_SCREAMING_SNAKE_CASE							   =get_tokens(a__    )
					_SCREAMING_SNAKE_CASE							   =collections.Counter(a__    ) & collections.Counter(a__    )
					_SCREAMING_SNAKE_CASE							   =sum(common.values()    )
					if len(a__    ) == 0 or len(a__    ) == 0:
										# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
										return int(gold_toks == pred_toks    )
					if num_same == 0:
										return 0
					_SCREAMING_SNAKE_CASE							   =1.0 * num_same / len(a__    )
					_SCREAMING_SNAKE_CASE							   =1.0 * num_same / len(a__    )
					_SCREAMING_SNAKE_CASE							   =(2 * precision * recall) / (precision + recall)
					return fa


def 	_lowerCAmelCase (						_UpperCamelCase						:					Tuple							,			_UpperCamelCase						:					List[Any]    )				->      List[Any]:
					"""simple docstring"""




					_SCREAMING_SNAKE_CASE							   ={}
					_SCREAMING_SNAKE_CASE							   ={}
					for article in dataset:
										for p in article["paragraphs"]:
															for qa in p["qas"]:
																				_SCREAMING_SNAKE_CASE							   =qa['id']
																				_SCREAMING_SNAKE_CASE							   =[t for t in qa['answers']['text'] if normalize_answer(a__    )]
																				if not gold_answers:
																									# For unanswerable questions, only correct answer is empty string
																									_SCREAMING_SNAKE_CASE							   =['']
																				if qid not in preds:
																									print(f"Missing prediction for {qid}"    )
																									continue
																				_SCREAMING_SNAKE_CASE							   =preds[qid]
																				# Take max over all gold answers
																				_SCREAMING_SNAKE_CASE							   =max(compute_exact(a__							,			a__    ) for a in gold_answers    )
																				_SCREAMING_SNAKE_CASE							   =max(compute_fa(a__							,			a__    ) for a in gold_answers    )
					return exact_scores, fa_scores


def 	_lowerCAmelCase (						_UpperCamelCase						:					int							,			_UpperCamelCase						:					Optional[int]							,			_UpperCamelCase						:					int							,			_UpperCamelCase						:					int    )				->      List[Any]:
					"""simple docstring"""




					_SCREAMING_SNAKE_CASE							   ={}
					for qid, s in scores.items():
										_SCREAMING_SNAKE_CASE							   =na_probs[qid] > na_prob_thresh
										if pred_na:
															_SCREAMING_SNAKE_CASE							   =float(not qid_to_has_ans[qid]    )
										else:
															_SCREAMING_SNAKE_CASE							   =s
					return new_scores


def 	_lowerCAmelCase (						_UpperCamelCase						:					Any							,			_UpperCamelCase						:					Optional[Any]							,			_UpperCamelCase						:					Optional[Any]=None    )				->      int:
					"""simple docstring"""




					if not qid_list:
										_SCREAMING_SNAKE_CASE							   =len(a__    )
										return collections.OrderedDict(
										    [
										        ('exact', 1_00.0 * sum(exact_scores.values()    ) / total),
										        ('f1', 1_00.0 * sum(fa_scores.values()    ) / total),
										        ('total', total),
										    ]    )
					else:
										_SCREAMING_SNAKE_CASE							   =len(a__    )
										return collections.OrderedDict(
										    [
										        ('exact', 1_00.0 * sum(exact_scores[k] for k in qid_list    ) / total),
										        ('f1', 1_00.0 * sum(fa_scores[k] for k in qid_list    ) / total),
										        ('total', total),
										    ]    )


def 	_lowerCAmelCase (						_UpperCamelCase						:					Union[str, Any]							,			_UpperCamelCase						:					Union[str, Any]							,			_UpperCamelCase						:					List[Any]    )				->      Tuple:
					"""simple docstring"""




					for k in new_eval:
										_SCREAMING_SNAKE_CASE							   =new_eval[k]


def 	_lowerCAmelCase (						_UpperCamelCase						:					int							,			_UpperCamelCase						:					int							,			_UpperCamelCase						:					int							,			_UpperCamelCase						:					List[Any]    )				->      Optional[int]:
					"""simple docstring"""




					plt.step(a__							,			a__							,			color='b'							,			alpha=0.2							,			where='post'    )
					plt.fill_between(a__							,			a__							,			step='post'							,			alpha=0.2							,			color='b'    )
					plt.xlabel('Recall'    )
					plt.ylabel('Precision'    )
					plt.xlim([0.0, 1.05]    )
					plt.ylim([0.0, 1.05]    )
					plt.title(a__    )
					plt.savefig(a__    )
					plt.clf()


def 	_lowerCAmelCase (						_UpperCamelCase						:					Optional[Any]							,			_UpperCamelCase						:					Tuple							,			_UpperCamelCase						:					Tuple							,			_UpperCamelCase						:					List[Any]							,			_UpperCamelCase						:					str=None							,			_UpperCamelCase						:					List[Any]=None    )				->      Tuple:
					"""simple docstring"""




					_SCREAMING_SNAKE_CASE							   =sorted(a__							,			key=lambda _UpperCamelCase    : na_probs[k]    )
					_SCREAMING_SNAKE_CASE							   =0.0
					_SCREAMING_SNAKE_CASE							   =1.0
					_SCREAMING_SNAKE_CASE							   =0.0
					_SCREAMING_SNAKE_CASE							   =[1.0]
					_SCREAMING_SNAKE_CASE							   =[0.0]
					_SCREAMING_SNAKE_CASE							   =0.0
					for i, qid in enumerate(a__    ):
										if qid_to_has_ans[qid]:
															true_pos += scores[qid]
										_SCREAMING_SNAKE_CASE							   =true_pos / float(i + 1    )
										_SCREAMING_SNAKE_CASE							   =true_pos / float(a__    )
										if i == len(a__    ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
															# i.e., if we can put a threshold after this point
															avg_prec += cur_p * (cur_r - recalls[-1])
															precisions.append(a__    )
															recalls.append(a__    )
					if out_image:
										plot_pr_curve(a__							,			a__							,			a__							,			a__    )
					return {"ap": 1_00.0 * avg_prec}


def 	_lowerCAmelCase (						_UpperCamelCase						:					List[Any]							,			_UpperCamelCase						:					Optional[Any]							,			_UpperCamelCase						:					Any							,			_UpperCamelCase						:					Tuple							,			_UpperCamelCase						:					Optional[Any]							,			_UpperCamelCase						:					Dict    )				->      Optional[int]:
					"""simple docstring"""




					if out_image_dir and not os.path.exists(a__    ):
										os.makedirs(a__    )
					_SCREAMING_SNAKE_CASE							   =sum(1 for v in qid_to_has_ans.values() if v    )
					if num_true_pos == 0:
										return
					_SCREAMING_SNAKE_CASE							   =make_precision_recall_eval(
					    a__							,			a__							,			a__							,			a__							,			out_image=os.path.join(a__							,			'pr_exact.png'    )							,			title='Precision-Recall curve for Exact Match score'							,			)
					_SCREAMING_SNAKE_CASE							   =make_precision_recall_eval(
					    a__							,			a__							,			a__							,			a__							,			out_image=os.path.join(a__							,			'pr_f1.png'    )							,			title='Precision-Recall curve for F1 score'							,			)
					_SCREAMING_SNAKE_CASE							   ={k: float(a__    ) for k, v in qid_to_has_ans.items()}
					_SCREAMING_SNAKE_CASE							   =make_precision_recall_eval(
					    a__							,			a__							,			a__							,			a__							,			out_image=os.path.join(a__							,			'pr_oracle.png'    )							,			title='Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'							,			)
					merge_eval(a__							,			a__							,			'pr_exact'    )
					merge_eval(a__							,			a__							,			'pr_f1'    )
					merge_eval(a__							,			a__							,			'pr_oracle'    )


def 	_lowerCAmelCase (						_UpperCamelCase						:					Any							,			_UpperCamelCase						:					List[Any]							,			_UpperCamelCase						:					Tuple							,			_UpperCamelCase						:					Optional[Any]    )				->      Tuple:
					"""simple docstring"""




					if not qid_list:
										return
					_SCREAMING_SNAKE_CASE							   =[na_probs[k] for k in qid_list]
					_SCREAMING_SNAKE_CASE							   =np.ones_like(a__    ) / float(len(a__    )    )
					plt.hist(a__							,			weights=a__							,			bins=20							,			range=(0.0, 1.0)    )
					plt.xlabel('Model probability of no-answer'    )
					plt.ylabel('Proportion of dataset'    )
					plt.title(f"Histogram of no-answer probability: {name}"    )
					plt.savefig(os.path.join(a__							,			f"na_prob_hist_{name}.png"    )    )
					plt.clf()


def 	_lowerCAmelCase (						_UpperCamelCase						:					List[str]							,			_UpperCamelCase						:					str							,			_UpperCamelCase						:					Union[str, Any]							,			_UpperCamelCase						:					Union[str, Any]    )				->      Optional[Any]:
					"""simple docstring"""




					_SCREAMING_SNAKE_CASE							   =sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]    )
					_SCREAMING_SNAKE_CASE							   =num_no_ans
					_SCREAMING_SNAKE_CASE							   =cur_score
					_SCREAMING_SNAKE_CASE							   =0.0
					_SCREAMING_SNAKE_CASE							   =sorted(a__							,			key=lambda _UpperCamelCase    : na_probs[k]    )
					for i, qid in enumerate(a__    ):
										if qid not in scores:
															continue
										if qid_to_has_ans[qid]:
															_SCREAMING_SNAKE_CASE							   =scores[qid]
										else:
															if preds[qid]:
																				_SCREAMING_SNAKE_CASE							   =-1
															else:
																				_SCREAMING_SNAKE_CASE							   =0
										cur_score += diff
										if cur_score > best_score:
															_SCREAMING_SNAKE_CASE							   =cur_score
															_SCREAMING_SNAKE_CASE							   =na_probs[qid]
					return 1_00.0 * best_score / len(a__    ), best_thresh


def 	_lowerCAmelCase (						_UpperCamelCase						:					Union[str, Any]							,			_UpperCamelCase						:					Optional[Any]							,			_UpperCamelCase						:					List[Any]							,			_UpperCamelCase						:					Tuple							,			_UpperCamelCase						:					Tuple							,			_UpperCamelCase						:					int    )				->      List[Any]:
					"""simple docstring"""




					_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE							   =find_best_thresh(a__							,			a__							,			a__							,			a__    )
					_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE							   =find_best_thresh(a__							,			a__							,			a__							,			a__    )
					_SCREAMING_SNAKE_CASE							   =best_exact
					_SCREAMING_SNAKE_CASE							   =exact_thresh
					_SCREAMING_SNAKE_CASE							   =best_fa
					_SCREAMING_SNAKE_CASE							   =fa_thresh


def 	_lowerCAmelCase (						)				->      List[Any]:
					"""simple docstring"""




					with open(OPTS.data_file    ) as f:
										_SCREAMING_SNAKE_CASE							   =json.load(a__    )
										_SCREAMING_SNAKE_CASE							   =dataset_json['data']
					with open(OPTS.pred_file    ) as f:
										_SCREAMING_SNAKE_CASE							   =json.load(a__    )
					if OPTS.na_prob_file:
										with open(OPTS.na_prob_file    ) as f:
															_SCREAMING_SNAKE_CASE							   =json.load(a__    )
					else:
										_SCREAMING_SNAKE_CASE							   ={k: 0.0 for k in preds}
					_SCREAMING_SNAKE_CASE							   =make_qid_to_has_ans(a__    )  # maps qid to True/False
					_SCREAMING_SNAKE_CASE							   =[k for k, v in qid_to_has_ans.items() if v]
					_SCREAMING_SNAKE_CASE							   =[k for k, v in qid_to_has_ans.items() if not v]
					_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE							   =get_raw_scores(a__							,			a__    )
					_SCREAMING_SNAKE_CASE							   =apply_no_ans_threshold(a__							,			a__							,			a__							,			OPTS.na_prob_thresh    )
					_SCREAMING_SNAKE_CASE							   =apply_no_ans_threshold(a__							,			a__							,			a__							,			OPTS.na_prob_thresh    )
					_SCREAMING_SNAKE_CASE							   =make_eval_dict(a__							,			a__    )
					if has_ans_qids:
										_SCREAMING_SNAKE_CASE							   =make_eval_dict(a__							,			a__							,			qid_list=a__    )
										merge_eval(a__							,			a__							,			'HasAns'    )
					if no_ans_qids:
										_SCREAMING_SNAKE_CASE							   =make_eval_dict(a__							,			a__							,			qid_list=a__    )
										merge_eval(a__							,			a__							,			'NoAns'    )
					if OPTS.na_prob_file:
										find_all_best_thresh(a__							,			a__							,			a__							,			a__							,			a__							,			a__    )
					if OPTS.na_prob_file and OPTS.out_image_dir:
										run_precision_recall_analysis(a__							,			a__							,			a__							,			a__							,			a__							,			OPTS.out_image_dir    )
										histogram_na_prob(a__							,			a__							,			OPTS.out_image_dir							,			'hasAns'    )
										histogram_na_prob(a__							,			a__							,			OPTS.out_image_dir							,			'noAns'    )
					if OPTS.out_file:
										with open(OPTS.out_file							,			'w'    ) as f:
															json.dump(a__							,			a__    )
					else:
										print(json.dumps(a__							,			indent=2    )    )


if __name__ == "__main__":
					lowerCamelCase			:				Optional[int] 			=					parse_args()
					if OPTS.out_image_dir:
										import matplotlib

										matplotlib.use("Agg")
										import matplotlib.pyplot as plt
					main()



'''simple docstring'''
from scipy.stats import pearsonr, spearmanr
from sklearn.metrics import fa_score, matthews_corrcoef

import datasets


lowerCamelCase			:				List[Any] 			=					"\\n@inproceedings{wang2019glue,\n  title={{GLUE}: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding},\n  author={Wang, Alex and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R.},\n  note={In the Proceedings of ICLR.},\n  year={2019}\n}\n"

lowerCamelCase			:				List[str] 			=					"\\nGLUE, the General Language Understanding Evaluation benchmark\n(https://gluebenchmark.com/) is a collection of resources for training,\nevaluating, and analyzing natural language understanding systems.\n"

lowerCamelCase			:				Dict 			=					"\nCompute GLUE evaluation metric associated to each GLUE dataset.\nArgs:\n    predictions: list of predictions to score.\n        Each translation should be tokenized into a list of tokens.\n    references: list of lists of references for each translation.\n        Each reference should be tokenized into a list of tokens.\nReturns: depending on the GLUE subset, one or several of:\n    \"accuracy\": Accuracy\n    \"f1\": F1 score\n    \"pearson\": Pearson Correlation\n    \"spearmanr\": Spearman Correlation\n    \"matthews_correlation\": Matthew Correlation\nExamples:\n\n    >>> glue_metric = datasets.load_metric('glue', 'sst2')  # 'sst2' or any of [\"mnli\", \"mnli_mismatched\", \"mnli_matched\", \"qnli\", \"rte\", \"wnli\", \"hans\"]\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'accuracy': 1.0}\n\n    >>> glue_metric = datasets.load_metric('glue', 'mrpc')  # 'mrpc' or 'qqp'\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'accuracy': 1.0, 'f1': 1.0}\n\n    >>> glue_metric = datasets.load_metric('glue', 'stsb')\n    >>> references = [0., 1., 2., 3., 4., 5.]\n    >>> predictions = [0., 1., 2., 3., 4., 5.]\n    >>> results = glue_metric.compute(predictions=predictions, references=references)\n    >>> print({\"pearson\": round(results[\"pearson\"], 2), \"spearmanr\": round(results[\"spearmanr\"], 2)})\n    {'pearson': 1.0, 'spearmanr': 1.0}\n\n    >>> glue_metric = datasets.load_metric('glue', 'cola')\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {'matthews_correlation': 1.0}\n"


def 	_lowerCAmelCase (						_UpperCamelCase						:					List[Any]							,			_UpperCamelCase						:					Dict    )				->      List[str]:
     """simple docstring"""




     return float((preds == labels).mean()    )


def 	_lowerCAmelCase (						_UpperCamelCase						:					Dict							,			_UpperCamelCase						:					Any    )				->      Tuple:
     """simple docstring"""




     _SCREAMING_SNAKE_CASE							   =simple_accuracy(_UpperCamelCase							,			_UpperCamelCase    )
     _SCREAMING_SNAKE_CASE							   =float(fa_score(y_true=_UpperCamelCase							,			y_pred=_UpperCamelCase    )    )
     return {
         "accuracy": acc,
         "f1": fa,
     }


def 	_lowerCAmelCase (						_UpperCamelCase						:					Optional[int]							,			_UpperCamelCase						:					Optional[int]    )				->      Dict:
     """simple docstring"""




     _SCREAMING_SNAKE_CASE							   =float(pearsonr(_UpperCamelCase							,			_UpperCamelCase    )[0]    )
     _SCREAMING_SNAKE_CASE							   =float(spearmanr(_UpperCamelCase							,			_UpperCamelCase    )[0]    )
     return {
         "pearson": pearson_corr,
         "spearmanr": spearman_corr,
     }







@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION					,       _KWARGS_DESCRIPTION  )
class 							A__   (	datasets.Metric  ):





  def 						A  (		self				:     Optional[int]			)     ->   Any:





       '''simple docstring'''






       if self.config_name not in [
           "sst2",
           "mnli",
           "mnli_mismatched",
           "mnli_matched",
           "cola",
           "stsb",
           "mrpc",
           "qqp",
           "qnli",
           "rte",
           "wnli",
           "hans",
       ]:
            raise KeyError(
                'You should supply a configuration name selected in '
                '["sst2", "mnli", "mnli_mismatched", "mnli_matched", '
                '"cola", "stsb", "mrpc", "qqp", "qnli", "rte", "wnli", "hans"]'			)
       return datasets.MetricInfo(
           description=_DESCRIPTION							, citation=_CITATION							, inputs_description=_KWARGS_DESCRIPTION							, features=datasets.Features(
               {
                   'predictions': datasets.Value('int64' if self.config_name != 'stsb' else 'float32'			),
                   'references': datasets.Value('int64' if self.config_name != 'stsb' else 'float32'			),
               }			)							, codebase_urls=[]							, reference_urls=[]							, format='numpy'							, )





  def 						A  (		self				:     str							, _a				:     str							, _a				:     List[Any]			)     ->   str:





       '''simple docstring'''






       if self.config_name == "cola":
            return {"matthews_correlation": matthews_corrcoef(_a							, _a			)}
       elif self.config_name == "stsb":
            return pearson_and_spearman(_a							, _a			)
       elif self.config_name in ["mrpc", "qqp"]:
            return acc_and_fa(_a							, _a			)
       elif self.config_name in ["sst2", "mnli", "mnli_mismatched", "mnli_matched", "qnli", "rte", "wnli", "hans"]:
            return {"accuracy": simple_accuracy(_a							, _a			)}
       else:
            raise KeyError(
                'You should supply a configuration name selected in '
                '["sst2", "mnli", "mnli_mismatched", "mnli_matched", '
                '"cola", "stsb", "mrpc", "qqp", "qnli", "rte", "wnli", "hans"]'			)
