

import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


_lowerCamelCase    :       List[str]        =							re.compile(R"\b(a|an|the)\b", re.UNICODE)

_lowerCamelCase    :       Optional[int]        =							None

def       a__					(		)  ->						Tuple:
  UpperCAmelCase	:							Optional[int]						=			argparse.ArgumentParser('''Official evaluation script for SQuAD version 2.0.'''      )
  parser.add_argument('''data_file'''    ,			metavar='''data.json'''    ,			help='''Input data JSON file.'''      )
  parser.add_argument('''pred_file'''    ,			metavar='''pred.json'''    ,			help='''Model predictions.'''      )
  parser.add_argument(
      '''--out-file'''    ,			'''-o'''    ,			metavar='''eval.json'''    ,			help='''Write accuracy metrics to file (default is stdout).'''      )
  parser.add_argument(
      '''--na-prob-file'''    ,			'''-n'''    ,			metavar='''na_prob.json'''    ,			help='''Model estimates of probability of no answer.'''      )
  parser.add_argument(
      '''--na-prob-thresh'''    ,			'''-t'''    ,			type=UpperCAmelCase    ,			default=1.0    ,			help='''Predict "" if no-answer probability exceeds this (default = 1.0).'''    ,			)
  parser.add_argument(
      '''--out-image-dir'''    ,			'''-p'''    ,			metavar='''out_images'''    ,			default=UpperCAmelCase    ,			help='''Save precision-recall curves to directory.'''      )
  parser.add_argument('''--verbose'''    ,			'''-v'''    ,			action='''store_true'''      )
  if len(sys.argv      ) == 1:
    parser.print_help()
    sys.exit(1      )
  return parser.parse_args()

def       a__					(		UpperCAmelCase							:      Optional[int]      )  ->						Optional[Any]:
  UpperCAmelCase	:							Dict						=			{}
  for article in dataset:
    for p in article["paragraphs"]:
      for qa in p["qas"]:
        UpperCAmelCase	:							Dict						=			bool(qa['''answers''']['''text''']      )
  return qid_to_has_ans

def       a__					(		UpperCAmelCase							:      Optional[Any]      )  ->						Tuple:

  def remove_articles(UpperCAmelCase							:      Optional[int]      ):
    return ARTICLES_REGEX.sub(''' '''    ,			UpperCAmelCase      )

  def white_space_fix(UpperCAmelCase							:      Dict      ):
    return " ".join(text.split()      )

  def remove_punc(UpperCAmelCase							:      List[Any]      ):
    UpperCAmelCase	:							Optional[int]						=			set(string.punctuation      )
    return "".join(ch for ch in text if ch not in exclude      )

  def lower(UpperCAmelCase							:      str      ):
    return text.lower()

  return white_space_fix(remove_articles(remove_punc(lower(UpperCAmelCase      )      )      )      )

def       a__					(		UpperCAmelCase							:      Dict      )  ->						Optional[int]:
  if not s:
    return []
  return normalize_answer(UpperCAmelCase      ).split()

def       a__					(		UpperCAmelCase							:      Tuple    ,			UpperCAmelCase							:      Optional[Any]      )  ->						Optional[Any]:
  return int(normalize_answer(UpperCAmelCase      ) == normalize_answer(UpperCAmelCase      )      )

def       a__					(		UpperCAmelCase							:      str    ,			UpperCAmelCase							:      Dict      )  ->						Dict:
  UpperCAmelCase	:							Optional[int]						=			get_tokens(UpperCAmelCase      )
  UpperCAmelCase	:							Any						=			get_tokens(UpperCAmelCase      )
  UpperCAmelCase	:							str						=			collections.Counter(UpperCAmelCase      ) & collections.Counter(UpperCAmelCase      )
  UpperCAmelCase	:							Tuple						=			sum(common.values()      )
  if len(UpperCAmelCase      ) == 0 or len(UpperCAmelCase      ) == 0:
    # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
    return int(gold_toks == pred_toks      )
  if num_same == 0:
    return 0
  UpperCAmelCase	:							List[str]						=			1.0 * num_same / len(UpperCAmelCase      )
  UpperCAmelCase	:							int						=			1.0 * num_same / len(UpperCAmelCase      )
  UpperCAmelCase	:							List[Any]						=			(2 * precision * recall) / (precision + recall)
  return fa

def       a__					(		UpperCAmelCase							:      Optional[int]    ,			UpperCAmelCase							:      str      )  ->						int:
  UpperCAmelCase	:							Any						=			{}
  UpperCAmelCase	:							Optional[int]						=			{}
  for article in dataset:
    for p in article["paragraphs"]:
      for qa in p["qas"]:
        UpperCAmelCase	:							Optional[int]						=			qa['''id''']
        UpperCAmelCase	:							Union[str, Any]						=			[t for t in qa['''answers''']['''text'''] if normalize_answer(UpperCAmelCase      )]
        if not gold_answers:
          # For unanswerable questions, only correct answer is empty string
          UpperCAmelCase	:							Union[str, Any]						=			['''''']
        if qid not in preds:
          print(f'''Missing prediction for {qid}'''      )
          continue
        UpperCAmelCase	:							List[str]						=			preds[qid]
        # Take max over all gold answers
        UpperCAmelCase	:							Optional[Any]						=			max(compute_exact(UpperCAmelCase    ,			UpperCAmelCase      ) for a in gold_answers      )
        UpperCAmelCase	:							Any						=			max(compute_fa(UpperCAmelCase    ,			UpperCAmelCase      ) for a in gold_answers      )
  return exact_scores, fa_scores

def       a__					(		UpperCAmelCase							:      Any    ,			UpperCAmelCase							:      List[Any]    ,			UpperCAmelCase							:      Optional[int]    ,			UpperCAmelCase							:      str      )  ->						Any:
  UpperCAmelCase	:							Union[str, Any]						=			{}
  for qid, s in scores.items():
    UpperCAmelCase	:							List[Any]						=			na_probs[qid] > na_prob_thresh
    if pred_na:
      UpperCAmelCase	:							str						=			float(not qid_to_has_ans[qid]      )
    else:
      UpperCAmelCase	:							Tuple						=			s
  return new_scores

def       a__					(		UpperCAmelCase							:      Optional[Any]    ,			UpperCAmelCase							:      Dict    ,			UpperCAmelCase							:      Optional[int]=None      )  ->						str:
  if not qid_list:
    UpperCAmelCase	:							Dict						=			len(UpperCAmelCase      )
    return collections.OrderedDict(
        [
            ('''exact''', 100.0 * sum(exact_scores.values()      ) / total),
            ('''f1''', 100.0 * sum(fa_scores.values()      ) / total),
            ('''total''', total),
        ]      )
  else:
    UpperCAmelCase	:							Tuple						=			len(UpperCAmelCase      )
    return collections.OrderedDict(
        [
            ('''exact''', 100.0 * sum(exact_scores[k] for k in qid_list      ) / total),
            ('''f1''', 100.0 * sum(fa_scores[k] for k in qid_list      ) / total),
            ('''total''', total),
        ]      )

def       a__					(		UpperCAmelCase							:      List[Any]    ,			UpperCAmelCase							:      Any    ,			UpperCAmelCase							:      Dict      )  ->						List[Any]:
  for k in new_eval:
    UpperCAmelCase	:							Dict						=			new_eval[k]

def       a__					(		UpperCAmelCase							:      str    ,			UpperCAmelCase							:      Optional[Any]    ,			UpperCAmelCase							:      List[str]    ,			UpperCAmelCase							:      Any      )  ->						Optional[int]:
  plt.step(UpperCAmelCase    ,			UpperCAmelCase    ,			color='''b'''    ,			alpha=0.2    ,			where='''post'''      )
  plt.fill_between(UpperCAmelCase    ,			UpperCAmelCase    ,			step='''post'''    ,			alpha=0.2    ,			color='''b'''      )
  plt.xlabel('''Recall'''      )
  plt.ylabel('''Precision'''      )
  plt.xlim([0.0, 1.05]      )
  plt.ylim([0.0, 1.05]      )
  plt.title(UpperCAmelCase      )
  plt.savefig(UpperCAmelCase      )
  plt.clf()

def       a__					(		UpperCAmelCase							:      Dict    ,			UpperCAmelCase							:      Any    ,			UpperCAmelCase							:      Tuple    ,			UpperCAmelCase							:      int    ,			UpperCAmelCase							:      List[str]=None    ,			UpperCAmelCase							:      List[str]=None      )  ->						Dict:
  UpperCAmelCase	:							Optional[int]						=			sorted(UpperCAmelCase    ,			key=lambda UpperCAmelCase      : na_probs[k]      )
  UpperCAmelCase	:							Dict						=			0.0
  UpperCAmelCase	:							Optional[Any]						=			1.0
  UpperCAmelCase	:							List[Any]						=			0.0
  UpperCAmelCase	:							Tuple						=			[1.0]
  UpperCAmelCase	:							int						=			[0.0]
  UpperCAmelCase	:							List[Any]						=			0.0
  for i, qid in enumerate(UpperCAmelCase      ):
    if qid_to_has_ans[qid]:
      true_pos += scores[qid]
    UpperCAmelCase	:							str						=			true_pos / float(i + 1      )
    UpperCAmelCase	:							int						=			true_pos / float(UpperCAmelCase      )
    if i == len(UpperCAmelCase      ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
      # i.e., if we can put a threshold after this point
      avg_prec += cur_p * (cur_r - recalls[-1])
      precisions.append(UpperCAmelCase      )
      recalls.append(UpperCAmelCase      )
  if out_image:
    plot_pr_curve(UpperCAmelCase    ,			UpperCAmelCase    ,			UpperCAmelCase    ,			UpperCAmelCase      )
  return {"ap": 100.0 * avg_prec}

def       a__					(		UpperCAmelCase							:      int    ,			UpperCAmelCase							:      List[Any]    ,			UpperCAmelCase							:      int    ,			UpperCAmelCase							:      Union[str, Any]    ,			UpperCAmelCase							:      str    ,			UpperCAmelCase							:      Union[str, Any]      )  ->						List[str]:
  if out_image_dir and not os.path.exists(UpperCAmelCase      ):
    os.makedirs(UpperCAmelCase      )
  UpperCAmelCase	:							str						=			sum(1 for v in qid_to_has_ans.values() if v      )
  if num_true_pos == 0:
    return
  UpperCAmelCase	:							List[str]						=			make_precision_recall_eval(
      UpperCAmelCase    ,			UpperCAmelCase    ,			UpperCAmelCase    ,			UpperCAmelCase    ,			out_image=os.path.join(UpperCAmelCase    ,			'''pr_exact.png'''      )    ,			title='''Precision-Recall curve for Exact Match score'''    ,			)
  UpperCAmelCase	:							Optional[Any]						=			make_precision_recall_eval(
      UpperCAmelCase    ,			UpperCAmelCase    ,			UpperCAmelCase    ,			UpperCAmelCase    ,			out_image=os.path.join(UpperCAmelCase    ,			'''pr_f1.png'''      )    ,			title='''Precision-Recall curve for F1 score'''    ,			)
  UpperCAmelCase	:							Dict						=			{k: float(UpperCAmelCase      ) for k, v in qid_to_has_ans.items()}
  UpperCAmelCase	:							Tuple						=			make_precision_recall_eval(
      UpperCAmelCase    ,			UpperCAmelCase    ,			UpperCAmelCase    ,			UpperCAmelCase    ,			out_image=os.path.join(UpperCAmelCase    ,			'''pr_oracle.png'''      )    ,			title='''Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'''    ,			)
  merge_eval(UpperCAmelCase    ,			UpperCAmelCase    ,			'''pr_exact'''      )
  merge_eval(UpperCAmelCase    ,			UpperCAmelCase    ,			'''pr_f1'''      )
  merge_eval(UpperCAmelCase    ,			UpperCAmelCase    ,			'''pr_oracle'''      )

def       a__					(		UpperCAmelCase							:      Optional[Any]    ,			UpperCAmelCase							:      Union[str, Any]    ,			UpperCAmelCase							:      Optional[int]    ,			UpperCAmelCase							:      Union[str, Any]      )  ->						Optional[Any]:
  if not qid_list:
    return
  UpperCAmelCase	:							str						=			[na_probs[k] for k in qid_list]
  UpperCAmelCase	:							List[str]						=			np.ones_like(UpperCAmelCase      ) / float(len(UpperCAmelCase      )      )
  plt.hist(UpperCAmelCase    ,			weights=UpperCAmelCase    ,			bins=20    ,			range=(0.0, 1.0)      )
  plt.xlabel('''Model probability of no-answer'''      )
  plt.ylabel('''Proportion of dataset'''      )
  plt.title(f'''Histogram of no-answer probability: {name}'''      )
  plt.savefig(os.path.join(UpperCAmelCase    ,			f'''na_prob_hist_{name}.png'''      )      )
  plt.clf()

def       a__					(		UpperCAmelCase							:      str    ,			UpperCAmelCase							:      List[str]    ,			UpperCAmelCase							:      Optional[int]    ,			UpperCAmelCase							:      List[str]      )  ->						str:
  UpperCAmelCase	:							Any						=			sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]      )
  UpperCAmelCase	:							str						=			num_no_ans
  UpperCAmelCase	:							Dict						=			cur_score
  UpperCAmelCase	:							List[str]						=			0.0
  UpperCAmelCase	:							List[str]						=			sorted(UpperCAmelCase    ,			key=lambda UpperCAmelCase      : na_probs[k]      )
  for i, qid in enumerate(UpperCAmelCase      ):
    if qid not in scores:
      continue
    if qid_to_has_ans[qid]:
      UpperCAmelCase	:							Tuple						=			scores[qid]
    else:
      if preds[qid]:
        UpperCAmelCase	:							Dict						=			-1
      else:
        UpperCAmelCase	:							Dict						=			0
    cur_score += diff
    if cur_score > best_score:
      UpperCAmelCase	:							List[Any]						=			cur_score
      UpperCAmelCase	:							int						=			na_probs[qid]
  return 100.0 * best_score / len(UpperCAmelCase      ), best_thresh

def       a__					(		UpperCAmelCase							:      List[str]    ,			UpperCAmelCase							:      List[Any]    ,			UpperCAmelCase							:      List[str]    ,			UpperCAmelCase							:      int    ,			UpperCAmelCase							:      Union[str, Any]    ,			UpperCAmelCase							:      Optional[Any]      )  ->						int:
  UpperCAmelCase			,						UpperCAmelCase	:							str						=			find_best_thresh(UpperCAmelCase    ,			UpperCAmelCase    ,			UpperCAmelCase    ,			UpperCAmelCase      )
  UpperCAmelCase			,						UpperCAmelCase	:							Dict						=			find_best_thresh(UpperCAmelCase    ,			UpperCAmelCase    ,			UpperCAmelCase    ,			UpperCAmelCase      )
  UpperCAmelCase	:							List[str]						=			best_exact
  UpperCAmelCase	:							Any						=			exact_thresh
  UpperCAmelCase	:							Dict						=			best_fa
  UpperCAmelCase	:							Optional[Any]						=			fa_thresh

def       a__					(		)  ->						Any:
  with open(OPTS.data_file      ) as f:
    UpperCAmelCase	:							str						=			json.load(UpperCAmelCase      )
    UpperCAmelCase	:							Dict						=			dataset_json['''data''']
  with open(OPTS.pred_file      ) as f:
    UpperCAmelCase	:							Dict						=			json.load(UpperCAmelCase      )
  if OPTS.na_prob_file:
    with open(OPTS.na_prob_file      ) as f:
      UpperCAmelCase	:							Optional[Any]						=			json.load(UpperCAmelCase      )
  else:
    UpperCAmelCase	:							List[Any]						=			{k: 0.0 for k in preds}
  UpperCAmelCase	:							Union[str, Any]						=			make_qid_to_has_ans(UpperCAmelCase      )  # maps qid to True/False
  UpperCAmelCase	:							Optional[Any]						=			[k for k, v in qid_to_has_ans.items() if v]
  UpperCAmelCase	:							str						=			[k for k, v in qid_to_has_ans.items() if not v]
  UpperCAmelCase			,						UpperCAmelCase	:							Optional[Any]						=			get_raw_scores(UpperCAmelCase    ,			UpperCAmelCase      )
  UpperCAmelCase	:							Dict						=			apply_no_ans_threshold(UpperCAmelCase    ,			UpperCAmelCase    ,			UpperCAmelCase    ,			OPTS.na_prob_thresh      )
  UpperCAmelCase	:							Optional[Any]						=			apply_no_ans_threshold(UpperCAmelCase    ,			UpperCAmelCase    ,			UpperCAmelCase    ,			OPTS.na_prob_thresh      )
  UpperCAmelCase	:							Dict						=			make_eval_dict(UpperCAmelCase    ,			UpperCAmelCase      )
  if has_ans_qids:
    UpperCAmelCase	:							Optional[Any]						=			make_eval_dict(UpperCAmelCase    ,			UpperCAmelCase    ,			qid_list=UpperCAmelCase      )
    merge_eval(UpperCAmelCase    ,			UpperCAmelCase    ,			'''HasAns'''      )
  if no_ans_qids:
    UpperCAmelCase	:							Any						=			make_eval_dict(UpperCAmelCase    ,			UpperCAmelCase    ,			qid_list=UpperCAmelCase      )
    merge_eval(UpperCAmelCase    ,			UpperCAmelCase    ,			'''NoAns'''      )
  if OPTS.na_prob_file:
    find_all_best_thresh(UpperCAmelCase    ,			UpperCAmelCase    ,			UpperCAmelCase    ,			UpperCAmelCase    ,			UpperCAmelCase    ,			UpperCAmelCase      )
  if OPTS.na_prob_file and OPTS.out_image_dir:
    run_precision_recall_analysis(UpperCAmelCase    ,			UpperCAmelCase    ,			UpperCAmelCase    ,			UpperCAmelCase    ,			UpperCAmelCase    ,			OPTS.out_image_dir      )
    histogram_na_prob(UpperCAmelCase    ,			UpperCAmelCase    ,			OPTS.out_image_dir    ,			'''hasAns'''      )
    histogram_na_prob(UpperCAmelCase    ,			UpperCAmelCase    ,			OPTS.out_image_dir    ,			'''noAns'''      )
  if OPTS.out_file:
    with open(OPTS.out_file    ,			'''w'''      ) as f:
      json.dump(UpperCAmelCase    ,			UpperCAmelCase      )
  else:
    print(json.dumps(UpperCAmelCase    ,			indent=2      )      )


if __name__ == "__main__":
     _lowerCamelCase    :       List[str]        =							parse_args()
     if OPTS.out_image_dir:
          import matplotlib

          matplotlib.use("Agg")
          import matplotlib.pyplot as plt
     main()








import logging
import os

from .state import PartialState





class  __UpperCAmelCase					(	logging.LoggerAdapter						):



   @staticmethod
   def 	__magic_name__   (							__A							:					str						):
     UpperCAmelCase	:							Dict						=			PartialState()
     return not main_process_only or (main_process_only and state.is_main_process)





   def 	__magic_name__   (							self							:					Union[str, Any],		__A							:					Union[str, Any],		__A							:					Union[str, Any],		*__A							:					Optional[int],		**__A							:					Tuple						):
     if PartialState._shared_state == {}:
       raise RuntimeError(
           '''You must initialize the accelerate state by calling either `PartialState()` or `Accelerator()` before using the logging utility.'''						)
     UpperCAmelCase	:							List[str]						=			kwargs.pop('''main_process_only''',		__A						)
     UpperCAmelCase	:							int						=			kwargs.pop('''in_order''',		__A						)

     if self.isEnabledFor(__A						):
       if self._should_log(__A						):
         UpperCAmelCase			,						UpperCAmelCase	:							Dict						=			self.process(__A,		__A						)
         self.logger.log(__A,		__A,		*__A,		**__A						)

       elif in_order:
         UpperCAmelCase	:							Dict						=			PartialState()
         for i in range(state.num_processes						):
           if i == state.process_index:
             UpperCAmelCase			,						UpperCAmelCase	:							Optional[int]						=			self.process(__A,		__A						)
             self.logger.log(__A,		__A,		*__A,		**__A						)
           state.wait_for_everyone()






def       a__					(		UpperCAmelCase							:      str    ,			UpperCAmelCase							:      str = None      )  ->						Dict:
  if log_level is None:
    UpperCAmelCase	:							Union[str, Any]						=			os.environ.get('''ACCELERATE_LOG_LEVEL'''    ,			UpperCAmelCase      )
  UpperCAmelCase	:							Tuple						=			logging.getLogger(UpperCAmelCase      )
  if log_level is not None:
    logger.setLevel(log_level.upper()      )
    logger.root.setLevel(log_level.upper()      )
  return MultiProcessAdapter(UpperCAmelCase    ,			{}      )





