

'''simple docstring'''
import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


lowerCamelCase			:				Optional[Any] 			=					re.compile(r"\b(a|an|the)\b", re.UNICODE)

lowerCamelCase			:				Optional[Any] 			=					None


def 	_lowerCAmelCase (						)				->      int:
     """simple docstring"""




     _SCREAMING_SNAKE_CASE							   =argparse.ArgumentParser('Official evaluation script for SQuAD version 2.0.'    )
     parser.add_argument('data_file'							,			metavar='data.json'							,			help='Input data JSON file.'    )
     parser.add_argument('pred_file'							,			metavar='pred.json'							,			help='Model predictions.'    )
     parser.add_argument(
         '--out-file'							,			'-o'							,			metavar='eval.json'							,			help='Write accuracy metrics to file (default is stdout).'    )
     parser.add_argument(
         '--na-prob-file'							,			'-n'							,			metavar='na_prob.json'							,			help='Model estimates of probability of no answer.'    )
     parser.add_argument(
         '--na-prob-thresh'							,			'-t'							,			type=_UpperCamelCase							,			default=1.0							,			help='Predict "" if no-answer probability exceeds this (default = 1.0).'							,			)
     parser.add_argument(
         '--out-image-dir'							,			'-p'							,			metavar='out_images'							,			default=_UpperCamelCase							,			help='Save precision-recall curves to directory.'    )
     parser.add_argument('--verbose'							,			'-v'							,			action='store_true'    )
     if len(sys.argv    ) == 1:
          parser.print_help()
          sys.exit(1    )
     return parser.parse_args()


def 	_lowerCAmelCase (						_UpperCamelCase						:					List[str]    )				->      Any:
     """simple docstring"""




     _SCREAMING_SNAKE_CASE							   ={}
     for article in dataset:
          for p in article["paragraphs"]:
               for qa in p["qas"]:
                    _SCREAMING_SNAKE_CASE							   =bool(qa['answers']['text']    )
     return qid_to_has_ans


def 	_lowerCAmelCase (						_UpperCamelCase						:					Any    )				->      Any:
     """simple docstring"""




     def remove_articles(_UpperCamelCase						:					List[str]    ):
          return ARTICLES_REGEX.sub(' '							,			_UpperCamelCase    )

     def white_space_fix(_UpperCamelCase						:					Optional[int]    ):
          return " ".join(text.split()    )

     def remove_punc(_UpperCamelCase						:					Union[str, Any]    ):
          _SCREAMING_SNAKE_CASE							   =set(string.punctuation    )
          return "".join(ch for ch in text if ch not in exclude    )

     def lower(_UpperCamelCase						:					Dict    ):
          return text.lower()

     return white_space_fix(remove_articles(remove_punc(lower(_UpperCamelCase    )    )    )    )


def 	_lowerCAmelCase (						_UpperCamelCase						:					Any    )				->      Tuple:
     """simple docstring"""




     if not s:
          return []
     return normalize_answer(_UpperCamelCase    ).split()


def 	_lowerCAmelCase (						_UpperCamelCase						:					str							,			_UpperCamelCase						:					str    )				->      int:
     """simple docstring"""




     return int(normalize_answer(_UpperCamelCase    ) == normalize_answer(_UpperCamelCase    )    )


def 	_lowerCAmelCase (						_UpperCamelCase						:					List[str]							,			_UpperCamelCase						:					List[Any]    )				->      Optional[Any]:
     """simple docstring"""




     _SCREAMING_SNAKE_CASE							   =get_tokens(_UpperCamelCase    )
     _SCREAMING_SNAKE_CASE							   =get_tokens(_UpperCamelCase    )
     _SCREAMING_SNAKE_CASE							   =collections.Counter(_UpperCamelCase    ) & collections.Counter(_UpperCamelCase    )
     _SCREAMING_SNAKE_CASE							   =sum(common.values()    )
     if len(_UpperCamelCase    ) == 0 or len(_UpperCamelCase    ) == 0:
          # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
          return int(gold_toks == pred_toks    )
     if num_same == 0:
          return 0
     _SCREAMING_SNAKE_CASE							   =1.0 * num_same / len(_UpperCamelCase    )
     _SCREAMING_SNAKE_CASE							   =1.0 * num_same / len(_UpperCamelCase    )
     _SCREAMING_SNAKE_CASE							   =(2 * precision * recall) / (precision + recall)
     return fa


def 	_lowerCAmelCase (						_UpperCamelCase						:					List[Any]							,			_UpperCamelCase						:					List[str]    )				->      str:
     """simple docstring"""




     _SCREAMING_SNAKE_CASE							   ={}
     _SCREAMING_SNAKE_CASE							   ={}
     for article in dataset:
          for p in article["paragraphs"]:
               for qa in p["qas"]:
                    _SCREAMING_SNAKE_CASE							   =qa['id']
                    _SCREAMING_SNAKE_CASE							   =[t for t in qa['answers']['text'] if normalize_answer(_UpperCamelCase    )]
                    if not gold_answers:
                         # For unanswerable questions, only correct answer is empty string
                         _SCREAMING_SNAKE_CASE							   =['']
                    if qid not in preds:
                         print(f"Missing prediction for {qid}"    )
                         continue
                    _SCREAMING_SNAKE_CASE							   =preds[qid]
                    # Take max over all gold answers
                    _SCREAMING_SNAKE_CASE							   =max(compute_exact(_UpperCamelCase							,			_UpperCamelCase    ) for a in gold_answers    )
                    _SCREAMING_SNAKE_CASE							   =max(compute_fa(_UpperCamelCase							,			_UpperCamelCase    ) for a in gold_answers    )
     return exact_scores, fa_scores


def 	_lowerCAmelCase (						_UpperCamelCase						:					int							,			_UpperCamelCase						:					Dict							,			_UpperCamelCase						:					Tuple							,			_UpperCamelCase						:					str    )				->      List[Any]:
     """simple docstring"""




     _SCREAMING_SNAKE_CASE							   ={}
     for qid, s in scores.items():
          _SCREAMING_SNAKE_CASE							   =na_probs[qid] > na_prob_thresh
          if pred_na:
               _SCREAMING_SNAKE_CASE							   =float(not qid_to_has_ans[qid]    )
          else:
               _SCREAMING_SNAKE_CASE							   =s
     return new_scores


def 	_lowerCAmelCase (						_UpperCamelCase						:					Optional[int]							,			_UpperCamelCase						:					Any							,			_UpperCamelCase						:					List[str]=None    )				->      List[str]:
     """simple docstring"""




     if not qid_list:
          _SCREAMING_SNAKE_CASE							   =len(_UpperCamelCase    )
          return collections.OrderedDict(
              [
                  ('exact', 1_00.0 * sum(exact_scores.values()    ) / total),
                  ('f1', 1_00.0 * sum(fa_scores.values()    ) / total),
                  ('total', total),
              ]    )
     else:
          _SCREAMING_SNAKE_CASE							   =len(_UpperCamelCase    )
          return collections.OrderedDict(
              [
                  ('exact', 1_00.0 * sum(exact_scores[k] for k in qid_list    ) / total),
                  ('f1', 1_00.0 * sum(fa_scores[k] for k in qid_list    ) / total),
                  ('total', total),
              ]    )


def 	_lowerCAmelCase (						_UpperCamelCase						:					List[str]							,			_UpperCamelCase						:					int							,			_UpperCamelCase						:					Union[str, Any]    )				->      Optional[int]:
     """simple docstring"""




     for k in new_eval:
          _SCREAMING_SNAKE_CASE							   =new_eval[k]


def 	_lowerCAmelCase (						_UpperCamelCase						:					Union[str, Any]							,			_UpperCamelCase						:					List[str]							,			_UpperCamelCase						:					Tuple							,			_UpperCamelCase						:					Optional[Any]    )				->      Optional[int]:
     """simple docstring"""




     plt.step(_UpperCamelCase							,			_UpperCamelCase							,			color='b'							,			alpha=0.2							,			where='post'    )
     plt.fill_between(_UpperCamelCase							,			_UpperCamelCase							,			step='post'							,			alpha=0.2							,			color='b'    )
     plt.xlabel('Recall'    )
     plt.ylabel('Precision'    )
     plt.xlim([0.0, 1.05]    )
     plt.ylim([0.0, 1.05]    )
     plt.title(_UpperCamelCase    )
     plt.savefig(_UpperCamelCase    )
     plt.clf()


def 	_lowerCAmelCase (						_UpperCamelCase						:					Any							,			_UpperCamelCase						:					Union[str, Any]							,			_UpperCamelCase						:					int							,			_UpperCamelCase						:					Tuple							,			_UpperCamelCase						:					str=None							,			_UpperCamelCase						:					Dict=None    )				->      Any:
     """simple docstring"""




     _SCREAMING_SNAKE_CASE							   =sorted(_UpperCamelCase							,			key=lambda _UpperCamelCase    : na_probs[k]    )
     _SCREAMING_SNAKE_CASE							   =0.0
     _SCREAMING_SNAKE_CASE							   =1.0
     _SCREAMING_SNAKE_CASE							   =0.0
     _SCREAMING_SNAKE_CASE							   =[1.0]
     _SCREAMING_SNAKE_CASE							   =[0.0]
     _SCREAMING_SNAKE_CASE							   =0.0
     for i, qid in enumerate(_UpperCamelCase    ):
          if qid_to_has_ans[qid]:
               true_pos += scores[qid]
          _SCREAMING_SNAKE_CASE							   =true_pos / float(i + 1    )
          _SCREAMING_SNAKE_CASE							   =true_pos / float(_UpperCamelCase    )
          if i == len(_UpperCamelCase    ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
               # i.e., if we can put a threshold after this point
               avg_prec += cur_p * (cur_r - recalls[-1])
               precisions.append(_UpperCamelCase    )
               recalls.append(_UpperCamelCase    )
     if out_image:
          plot_pr_curve(_UpperCamelCase							,			_UpperCamelCase							,			_UpperCamelCase							,			_UpperCamelCase    )
     return {"ap": 1_00.0 * avg_prec}


def 	_lowerCAmelCase (						_UpperCamelCase						:					Optional[Any]							,			_UpperCamelCase						:					Tuple							,			_UpperCamelCase						:					Any							,			_UpperCamelCase						:					str							,			_UpperCamelCase						:					Union[str, Any]							,			_UpperCamelCase						:					Optional[int]    )				->      Dict:
     """simple docstring"""




     if out_image_dir and not os.path.exists(_UpperCamelCase    ):
          os.makedirs(_UpperCamelCase    )
     _SCREAMING_SNAKE_CASE							   =sum(1 for v in qid_to_has_ans.values() if v    )
     if num_true_pos == 0:
          return
     _SCREAMING_SNAKE_CASE							   =make_precision_recall_eval(
         _UpperCamelCase							,			_UpperCamelCase							,			_UpperCamelCase							,			_UpperCamelCase							,			out_image=os.path.join(_UpperCamelCase							,			'pr_exact.png'    )							,			title='Precision-Recall curve for Exact Match score'							,			)
     _SCREAMING_SNAKE_CASE							   =make_precision_recall_eval(
         _UpperCamelCase							,			_UpperCamelCase							,			_UpperCamelCase							,			_UpperCamelCase							,			out_image=os.path.join(_UpperCamelCase							,			'pr_f1.png'    )							,			title='Precision-Recall curve for F1 score'							,			)
     _SCREAMING_SNAKE_CASE							   ={k: float(_UpperCamelCase    ) for k, v in qid_to_has_ans.items()}
     _SCREAMING_SNAKE_CASE							   =make_precision_recall_eval(
         _UpperCamelCase							,			_UpperCamelCase							,			_UpperCamelCase							,			_UpperCamelCase							,			out_image=os.path.join(_UpperCamelCase							,			'pr_oracle.png'    )							,			title='Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'							,			)
     merge_eval(_UpperCamelCase							,			_UpperCamelCase							,			'pr_exact'    )
     merge_eval(_UpperCamelCase							,			_UpperCamelCase							,			'pr_f1'    )
     merge_eval(_UpperCamelCase							,			_UpperCamelCase							,			'pr_oracle'    )


def 	_lowerCAmelCase (						_UpperCamelCase						:					int							,			_UpperCamelCase						:					Tuple							,			_UpperCamelCase						:					str							,			_UpperCamelCase						:					Optional[Any]    )				->      List[Any]:
     """simple docstring"""




     if not qid_list:
          return
     _SCREAMING_SNAKE_CASE							   =[na_probs[k] for k in qid_list]
     _SCREAMING_SNAKE_CASE							   =np.ones_like(_UpperCamelCase    ) / float(len(_UpperCamelCase    )    )
     plt.hist(_UpperCamelCase							,			weights=_UpperCamelCase							,			bins=20							,			range=(0.0, 1.0)    )
     plt.xlabel('Model probability of no-answer'    )
     plt.ylabel('Proportion of dataset'    )
     plt.title(f"Histogram of no-answer probability: {name}"    )
     plt.savefig(os.path.join(_UpperCamelCase							,			f"na_prob_hist_{name}.png"    )    )
     plt.clf()


def 	_lowerCAmelCase (						_UpperCamelCase						:					Optional[Any]							,			_UpperCamelCase						:					Tuple							,			_UpperCamelCase						:					str							,			_UpperCamelCase						:					Optional[int]    )				->      int:
     """simple docstring"""




     _SCREAMING_SNAKE_CASE							   =sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]    )
     _SCREAMING_SNAKE_CASE							   =num_no_ans
     _SCREAMING_SNAKE_CASE							   =cur_score
     _SCREAMING_SNAKE_CASE							   =0.0
     _SCREAMING_SNAKE_CASE							   =sorted(_UpperCamelCase							,			key=lambda _UpperCamelCase    : na_probs[k]    )
     for i, qid in enumerate(_UpperCamelCase    ):
          if qid not in scores:
               continue
          if qid_to_has_ans[qid]:
               _SCREAMING_SNAKE_CASE							   =scores[qid]
          else:
               if preds[qid]:
                    _SCREAMING_SNAKE_CASE							   =-1
               else:
                    _SCREAMING_SNAKE_CASE							   =0
          cur_score += diff
          if cur_score > best_score:
               _SCREAMING_SNAKE_CASE							   =cur_score
               _SCREAMING_SNAKE_CASE							   =na_probs[qid]
     return 1_00.0 * best_score / len(_UpperCamelCase    ), best_thresh


def 	_lowerCAmelCase (						_UpperCamelCase						:					Union[str, Any]							,			_UpperCamelCase						:					Any							,			_UpperCamelCase						:					Tuple							,			_UpperCamelCase						:					Any							,			_UpperCamelCase						:					Optional[Any]							,			_UpperCamelCase						:					List[Any]    )				->      List[Any]:
     """simple docstring"""




     _SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE							   =find_best_thresh(_UpperCamelCase							,			_UpperCamelCase							,			_UpperCamelCase							,			_UpperCamelCase    )
     _SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE							   =find_best_thresh(_UpperCamelCase							,			_UpperCamelCase							,			_UpperCamelCase							,			_UpperCamelCase    )
     _SCREAMING_SNAKE_CASE							   =best_exact
     _SCREAMING_SNAKE_CASE							   =exact_thresh
     _SCREAMING_SNAKE_CASE							   =best_fa
     _SCREAMING_SNAKE_CASE							   =fa_thresh


def 	_lowerCAmelCase (						)				->      int:
     """simple docstring"""




     with open(OPTS.data_file    ) as f:
          _SCREAMING_SNAKE_CASE							   =json.load(_UpperCamelCase    )
          _SCREAMING_SNAKE_CASE							   =dataset_json['data']
     with open(OPTS.pred_file    ) as f:
          _SCREAMING_SNAKE_CASE							   =json.load(_UpperCamelCase    )
     if OPTS.na_prob_file:
          with open(OPTS.na_prob_file    ) as f:
               _SCREAMING_SNAKE_CASE							   =json.load(_UpperCamelCase    )
     else:
          _SCREAMING_SNAKE_CASE							   ={k: 0.0 for k in preds}
     _SCREAMING_SNAKE_CASE							   =make_qid_to_has_ans(_UpperCamelCase    )  # maps qid to True/False
     _SCREAMING_SNAKE_CASE							   =[k for k, v in qid_to_has_ans.items() if v]
     _SCREAMING_SNAKE_CASE							   =[k for k, v in qid_to_has_ans.items() if not v]
     _SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE							   =get_raw_scores(_UpperCamelCase							,			_UpperCamelCase    )
     _SCREAMING_SNAKE_CASE							   =apply_no_ans_threshold(_UpperCamelCase							,			_UpperCamelCase							,			_UpperCamelCase							,			OPTS.na_prob_thresh    )
     _SCREAMING_SNAKE_CASE							   =apply_no_ans_threshold(_UpperCamelCase							,			_UpperCamelCase							,			_UpperCamelCase							,			OPTS.na_prob_thresh    )
     _SCREAMING_SNAKE_CASE							   =make_eval_dict(_UpperCamelCase							,			_UpperCamelCase    )
     if has_ans_qids:
          _SCREAMING_SNAKE_CASE							   =make_eval_dict(_UpperCamelCase							,			_UpperCamelCase							,			qid_list=_UpperCamelCase    )
          merge_eval(_UpperCamelCase							,			_UpperCamelCase							,			'HasAns'    )
     if no_ans_qids:
          _SCREAMING_SNAKE_CASE							   =make_eval_dict(_UpperCamelCase							,			_UpperCamelCase							,			qid_list=_UpperCamelCase    )
          merge_eval(_UpperCamelCase							,			_UpperCamelCase							,			'NoAns'    )
     if OPTS.na_prob_file:
          find_all_best_thresh(_UpperCamelCase							,			_UpperCamelCase							,			_UpperCamelCase							,			_UpperCamelCase							,			_UpperCamelCase							,			_UpperCamelCase    )
     if OPTS.na_prob_file and OPTS.out_image_dir:
          run_precision_recall_analysis(_UpperCamelCase							,			_UpperCamelCase							,			_UpperCamelCase							,			_UpperCamelCase							,			_UpperCamelCase							,			OPTS.out_image_dir    )
          histogram_na_prob(_UpperCamelCase							,			_UpperCamelCase							,			OPTS.out_image_dir							,			'hasAns'    )
          histogram_na_prob(_UpperCamelCase							,			_UpperCamelCase							,			OPTS.out_image_dir							,			'noAns'    )
     if OPTS.out_file:
          with open(OPTS.out_file							,			'w'    ) as f:
               json.dump(_UpperCamelCase							,			_UpperCamelCase    )
     else:
          print(json.dumps(_UpperCamelCase							,			indent=2    )    )


if __name__ == "__main__":
     lowerCamelCase			:				List[str] 			=					parse_args()
     if OPTS.out_image_dir:
          import matplotlib

          matplotlib.use("Agg")
          import matplotlib.pyplot as plt
     main()



'''simple docstring'''
from ..utils import DummyObject, requires_backends







class 							A__   (	metaclass=A__  ):
  A__					=    ['note_seq']





  def __init__(		self				:     List[str]							, *_a				:     Any							, **_a				:     Dict			)     ->   Optional[Any]:





       '''simple docstring'''






       requires_backends(self							, ['note_seq']			)





  @classmethod
  def 						A  (		cls				:     Any							, *_a				:     str							, **_a				:     List[Any]			)     ->   List[str]:





       '''simple docstring'''






       requires_backends(cls							, ['note_seq']			)





  @classmethod
  def 						A  (		cls				:     int							, *_a				:     Optional[Any]							, **_a				:     Optional[int]			)     ->   Tuple:





       '''simple docstring'''






       requires_backends(cls							, ['note_seq']			)
