


'''simple docstring'''






import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


a_   :      str				    =      re.compile(R"\b(a|an|the)\b", re.UNICODE)

a_   :      List[Any]				    =      None






def 		_A			()				->					Optional[Any]:




     '''simple docstring'''



     _a      =							argparse.ArgumentParser('Official evaluation script for SQuAD version 2.0.'						)
     parser.add_argument('data_file'			,  metavar='data.json'			,  help='Input data JSON file.'						)
     parser.add_argument('pred_file'			,  metavar='pred.json'			,  help='Model predictions.'						)
     parser.add_argument(
         '--out-file'			,  '-o'			,  metavar='eval.json'			,  help='Write accuracy metrics to file (default is stdout).'						)
     parser.add_argument(
         '--na-prob-file'			,  '-n'			,  metavar='na_prob.json'			,  help='Model estimates of probability of no answer.'						)
     parser.add_argument(
         '--na-prob-thresh'			,  '-t'			,  type=lowerCAmelCase__			,  default=1.0			,  help='Predict "" if no-answer probability exceeds this (default = 1.0).'			,  )
     parser.add_argument(
         '--out-image-dir'			,  '-p'			,  metavar='out_images'			,  default=lowerCAmelCase__			,  help='Save precision-recall curves to directory.'						)
     parser.add_argument('--verbose'			,  '-v'			,  action='store_true'						)
     if len(sys.argv						) == 1:
          parser.print_help()
          sys.exit(1						)
     return parser.parse_args()






def 		_A			(lowerCAmelCase__					:str						)				->					str:




     '''simple docstring'''



     _a      =							{}
     for article in dataset:
          for p in article["paragraphs"]:
               for qa in p["qas"]:
                    _a      =							bool(qa['answers']['text']						)
     return qid_to_has_ans






def 		_A			(lowerCAmelCase__					:List[Any]						)				->					Union[str, Any]:




     '''simple docstring'''



     def remove_articles(lowerCAmelCase__					:Dict						):
          return ARTICLES_REGEX.sub(' '			,  lowerCAmelCase__						)

     def white_space_fix(lowerCAmelCase__					:Dict						):
          return " ".join(text.split()						)

     def remove_punc(lowerCAmelCase__					:int						):
          _a      =							set(string.punctuation						)
          return "".join(ch for ch in text if ch not in exclude						)

     def lower(lowerCAmelCase__					:Any						):
          return text.lower()

     return white_space_fix(remove_articles(remove_punc(lower(lowerCAmelCase__						)						)						)						)






def 		_A			(lowerCAmelCase__					:List[Any]						)				->					int:




     '''simple docstring'''



     if not s:
          return []
     return normalize_answer(lowerCAmelCase__						).split()






def 		_A			(lowerCAmelCase__					:Optional[int]			,  lowerCAmelCase__					:Any						)				->					Any:




     '''simple docstring'''



     return int(normalize_answer(lowerCAmelCase__						) == normalize_answer(lowerCAmelCase__						)						)






def 		_A			(lowerCAmelCase__					:Tuple			,  lowerCAmelCase__					:Optional[Any]						)				->					List[Any]:




     '''simple docstring'''



     _a      =							get_tokens(lowerCAmelCase__						)
     _a      =							get_tokens(lowerCAmelCase__						)
     _a      =							collections.Counter(lowerCAmelCase__						) & collections.Counter(lowerCAmelCase__						)
     _a      =							sum(common.values()						)
     if len(lowerCAmelCase__						) == 0 or len(lowerCAmelCase__						) == 0:
          # If either is no-answer, then F1 is 1 if they agree, 0 otherwise
          return int(gold_toks == pred_toks						)
     if num_same == 0:
          return 0
     _a      =							1.0 * num_same / len(lowerCAmelCase__						)
     _a      =							1.0 * num_same / len(lowerCAmelCase__						)
     _a      =							(2 * precision * recall) / (precision + recall)
     return fa






def 		_A			(lowerCAmelCase__					:str			,  lowerCAmelCase__					:int						)				->					Union[str, Any]:




     '''simple docstring'''



     _a      =							{}
     _a      =							{}
     for article in dataset:
          for p in article["paragraphs"]:
               for qa in p["qas"]:
                    _a      =							qa['id']
                    _a      =							[t for t in qa['answers']['text'] if normalize_answer(lowerCAmelCase__						)]
                    if not gold_answers:
                         # For unanswerable questions, only correct answer is empty string
                         _a      =							['']
                    if qid not in preds:
                         print(f'Missing prediction for {qid}'						)
                         continue
                    _a      =							preds[qid]
                    # Take max over all gold answers
                    _a      =							max(compute_exact(lowerCAmelCase__			,  lowerCAmelCase__						) for a in gold_answers						)
                    _a      =							max(compute_fa(lowerCAmelCase__			,  lowerCAmelCase__						) for a in gold_answers						)
     return exact_scores, fa_scores






def 		_A			(lowerCAmelCase__					:Optional[int]			,  lowerCAmelCase__					:Optional[Any]			,  lowerCAmelCase__					:Dict			,  lowerCAmelCase__					:Union[str, Any]						)				->					str:




     '''simple docstring'''



     _a      =							{}
     for qid, s in scores.items():
          _a      =							na_probs[qid] > na_prob_thresh
          if pred_na:
               _a      =							float(not qid_to_has_ans[qid]						)
          else:
               _a      =							s
     return new_scores






def 		_A			(lowerCAmelCase__					:List[str]			,  lowerCAmelCase__					:List[Any]			,  lowerCAmelCase__					:Tuple=None						)				->					Dict:




     '''simple docstring'''



     if not qid_list:
          _a      =							len(lowerCAmelCase__						)
          return collections.OrderedDict(
              [
                  ('exact', 1_00.0 * sum(exact_scores.values()						) / total),
                  ('f1', 1_00.0 * sum(fa_scores.values()						) / total),
                  ('total', total),
              ]						)
     else:
          _a      =							len(lowerCAmelCase__						)
          return collections.OrderedDict(
              [
                  ('exact', 1_00.0 * sum(exact_scores[k] for k in qid_list						) / total),
                  ('f1', 1_00.0 * sum(fa_scores[k] for k in qid_list						) / total),
                  ('total', total),
              ]						)






def 		_A			(lowerCAmelCase__					:str			,  lowerCAmelCase__					:Any			,  lowerCAmelCase__					:int						)				->					Tuple:




     '''simple docstring'''



     for k in new_eval:
          _a      =							new_eval[k]






def 		_A			(lowerCAmelCase__					:int			,  lowerCAmelCase__					:str			,  lowerCAmelCase__					:Any			,  lowerCAmelCase__					:Optional[Any]						)				->					Tuple:




     '''simple docstring'''



     plt.step(lowerCAmelCase__			,  lowerCAmelCase__			,  color='b'			,  alpha=0.2			,  where='post'						)
     plt.fill_between(lowerCAmelCase__			,  lowerCAmelCase__			,  step='post'			,  alpha=0.2			,  color='b'						)
     plt.xlabel('Recall'						)
     plt.ylabel('Precision'						)
     plt.xlim([0.0, 1.0_5]						)
     plt.ylim([0.0, 1.0_5]						)
     plt.title(lowerCAmelCase__						)
     plt.savefig(lowerCAmelCase__						)
     plt.clf()






def 		_A			(lowerCAmelCase__					:str			,  lowerCAmelCase__					:str			,  lowerCAmelCase__					:Dict			,  lowerCAmelCase__					:Tuple			,  lowerCAmelCase__					:Optional[int]=None			,  lowerCAmelCase__					:List[Any]=None						)				->					Optional[int]:




     '''simple docstring'''



     _a      =							sorted(lowerCAmelCase__			,  key=lambda lowerCAmelCase__						: na_probs[k]						)
     _a      =							0.0
     _a      =							1.0
     _a      =							0.0
     _a      =							[1.0]
     _a      =							[0.0]
     _a      =							0.0
     for i, qid in enumerate(lowerCAmelCase__						):
          if qid_to_has_ans[qid]:
               true_pos += scores[qid]
          _a      =							true_pos / float(i + 1						)
          _a      =							true_pos / float(lowerCAmelCase__						)
          if i == len(lowerCAmelCase__						) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
               # i.e., if we can put a threshold after this point
               avg_prec += cur_p * (cur_r - recalls[-1])
               precisions.append(lowerCAmelCase__						)
               recalls.append(lowerCAmelCase__						)
     if out_image:
          plot_pr_curve(lowerCAmelCase__			,  lowerCAmelCase__			,  lowerCAmelCase__			,  lowerCAmelCase__						)
     return {"ap": 1_00.0 * avg_prec}






def 		_A			(lowerCAmelCase__					:Union[str, Any]			,  lowerCAmelCase__					:Union[str, Any]			,  lowerCAmelCase__					:Optional[int]			,  lowerCAmelCase__					:Optional[Any]			,  lowerCAmelCase__					:Dict			,  lowerCAmelCase__					:Optional[Any]						)				->					Tuple:




     '''simple docstring'''



     if out_image_dir and not os.path.exists(lowerCAmelCase__						):
          os.makedirs(lowerCAmelCase__						)
     _a      =							sum(1 for v in qid_to_has_ans.values() if v						)
     if num_true_pos == 0:
          return
     _a      =							make_precision_recall_eval(
         lowerCAmelCase__			,  lowerCAmelCase__			,  lowerCAmelCase__			,  lowerCAmelCase__			,  out_image=os.path.join(lowerCAmelCase__			,  'pr_exact.png'						)			,  title='Precision-Recall curve for Exact Match score'			,  )
     _a      =							make_precision_recall_eval(
         lowerCAmelCase__			,  lowerCAmelCase__			,  lowerCAmelCase__			,  lowerCAmelCase__			,  out_image=os.path.join(lowerCAmelCase__			,  'pr_f1.png'						)			,  title='Precision-Recall curve for F1 score'			,  )
     _a      =							{k: float(lowerCAmelCase__						) for k, v in qid_to_has_ans.items()}
     _a      =							make_precision_recall_eval(
         lowerCAmelCase__			,  lowerCAmelCase__			,  lowerCAmelCase__			,  lowerCAmelCase__			,  out_image=os.path.join(lowerCAmelCase__			,  'pr_oracle.png'						)			,  title='Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'			,  )
     merge_eval(lowerCAmelCase__			,  lowerCAmelCase__			,  'pr_exact'						)
     merge_eval(lowerCAmelCase__			,  lowerCAmelCase__			,  'pr_f1'						)
     merge_eval(lowerCAmelCase__			,  lowerCAmelCase__			,  'pr_oracle'						)






def 		_A			(lowerCAmelCase__					:List[str]			,  lowerCAmelCase__					:str			,  lowerCAmelCase__					:int			,  lowerCAmelCase__					:List[str]						)				->					str:




     '''simple docstring'''



     if not qid_list:
          return
     _a      =							[na_probs[k] for k in qid_list]
     _a      =							np.ones_like(lowerCAmelCase__						) / float(len(lowerCAmelCase__						)						)
     plt.hist(lowerCAmelCase__			,  weights=lowerCAmelCase__			,  bins=20			,  range=(0.0, 1.0)						)
     plt.xlabel('Model probability of no-answer'						)
     plt.ylabel('Proportion of dataset'						)
     plt.title(f'Histogram of no-answer probability: {name}'						)
     plt.savefig(os.path.join(lowerCAmelCase__			,  f'na_prob_hist_{name}.png'						)						)
     plt.clf()






def 		_A			(lowerCAmelCase__					:int			,  lowerCAmelCase__					:int			,  lowerCAmelCase__					:Dict			,  lowerCAmelCase__					:int						)				->					List[Any]:




     '''simple docstring'''



     _a      =							sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]						)
     _a      =							num_no_ans
     _a      =							cur_score
     _a      =							0.0
     _a      =							sorted(lowerCAmelCase__			,  key=lambda lowerCAmelCase__						: na_probs[k]						)
     for i, qid in enumerate(lowerCAmelCase__						):
          if qid not in scores:
               continue
          if qid_to_has_ans[qid]:
               _a      =							scores[qid]
          else:
               if preds[qid]:
                    _a      =							-1
               else:
                    _a      =							0
          cur_score += diff
          if cur_score > best_score:
               _a      =							cur_score
               _a      =							na_probs[qid]
     return 1_00.0 * best_score / len(lowerCAmelCase__						), best_thresh






def 		_A			(lowerCAmelCase__					:List[Any]			,  lowerCAmelCase__					:Tuple			,  lowerCAmelCase__					:Tuple			,  lowerCAmelCase__					:Optional[Any]			,  lowerCAmelCase__					:List[Any]			,  lowerCAmelCase__					:Optional[int]						)				->					List[str]:




     '''simple docstring'''



     _a ,				_a      =							find_best_thresh(lowerCAmelCase__			,  lowerCAmelCase__			,  lowerCAmelCase__			,  lowerCAmelCase__						)
     _a ,				_a      =							find_best_thresh(lowerCAmelCase__			,  lowerCAmelCase__			,  lowerCAmelCase__			,  lowerCAmelCase__						)
     _a      =							best_exact
     _a      =							exact_thresh
     _a      =							best_fa
     _a      =							fa_thresh






def 		_A			()				->					List[str]:




     '''simple docstring'''



     with open(OPTS.data_file						) as f:
          _a      =							json.load(lowerCAmelCase__						)
          _a      =							dataset_json['data']
     with open(OPTS.pred_file						) as f:
          _a      =							json.load(lowerCAmelCase__						)
     if OPTS.na_prob_file:
          with open(OPTS.na_prob_file						) as f:
               _a      =							json.load(lowerCAmelCase__						)
     else:
          _a      =							{k: 0.0 for k in preds}
     _a      =							make_qid_to_has_ans(lowerCAmelCase__						)  # maps qid to True/False
     _a      =							[k for k, v in qid_to_has_ans.items() if v]
     _a      =							[k for k, v in qid_to_has_ans.items() if not v]
     _a ,				_a      =							get_raw_scores(lowerCAmelCase__			,  lowerCAmelCase__						)
     _a      =							apply_no_ans_threshold(lowerCAmelCase__			,  lowerCAmelCase__			,  lowerCAmelCase__			,  OPTS.na_prob_thresh						)
     _a      =							apply_no_ans_threshold(lowerCAmelCase__			,  lowerCAmelCase__			,  lowerCAmelCase__			,  OPTS.na_prob_thresh						)
     _a      =							make_eval_dict(lowerCAmelCase__			,  lowerCAmelCase__						)
     if has_ans_qids:
          _a      =							make_eval_dict(lowerCAmelCase__			,  lowerCAmelCase__			,  qid_list=lowerCAmelCase__						)
          merge_eval(lowerCAmelCase__			,  lowerCAmelCase__			,  'HasAns'						)
     if no_ans_qids:
          _a      =							make_eval_dict(lowerCAmelCase__			,  lowerCAmelCase__			,  qid_list=lowerCAmelCase__						)
          merge_eval(lowerCAmelCase__			,  lowerCAmelCase__			,  'NoAns'						)
     if OPTS.na_prob_file:
          find_all_best_thresh(lowerCAmelCase__			,  lowerCAmelCase__			,  lowerCAmelCase__			,  lowerCAmelCase__			,  lowerCAmelCase__			,  lowerCAmelCase__						)
     if OPTS.na_prob_file and OPTS.out_image_dir:
          run_precision_recall_analysis(lowerCAmelCase__			,  lowerCAmelCase__			,  lowerCAmelCase__			,  lowerCAmelCase__			,  lowerCAmelCase__			,  OPTS.out_image_dir						)
          histogram_na_prob(lowerCAmelCase__			,  lowerCAmelCase__			,  OPTS.out_image_dir			,  'hasAns'						)
          histogram_na_prob(lowerCAmelCase__			,  lowerCAmelCase__			,  OPTS.out_image_dir			,  'noAns'						)
     if OPTS.out_file:
          with open(OPTS.out_file			,  'w'						) as f:
               json.dump(lowerCAmelCase__			,  lowerCAmelCase__						)
     else:
          print(json.dumps(lowerCAmelCase__			,  indent=2						)						)


if __name__ == "__main__":
  a_   :      List[str]				    =      parse_args()
  if OPTS.out_image_dir:
    import matplotlib

    matplotlib.use("Agg")
    import matplotlib.pyplot as plt
  main()





'''simple docstring'''






from __future__ import annotations

from sys import maxsize
from typing import Generic, TypeVar

a_   :      Optional[Any]				    =      TypeVar("T")






def 		_A			(lowerCAmelCase__					:int						)				->					int:




     '''simple docstring'''



     return (position - 1) // 2






def 		_A			(lowerCAmelCase__					:int						)				->					int:




     '''simple docstring'''



     return (2 * position) + 1






def 		_A			(lowerCAmelCase__					:int						)				->					int:




     '''simple docstring'''



     return (2 * position) + 2




class 	a       (       Generic[T]    ):
 def __init__(	self  )   ->							None:
      _a      =							[]
      _a      =							{}
      _a      =							0
 def __len__(	self  )   ->							int:
      return self.elements
 def __repr__(	self  )   ->							str:
      return str(self.heap  )
 def 							__UpperCAmelCase						(	self  )   ->							bool:
      # Check if the priority queue is empty
      return self.elements == 0
 def 							__UpperCAmelCase						(	self ,			__magic_name__ ,			__magic_name__  )   ->							None:
      # Add an element with given priority to the queue
      self.heap.append((elem, weight)  )
      _a      =							self.elements
      self.elements += 1
      self._bubble_up(__magic_name__  )
 def 							__UpperCAmelCase						(	self  )   ->							T:
      # Remove and return the element with lowest weight (highest priority)
      if self.elements > 1:
           self._swap_nodes(0 ,			self.elements - 1  )
      _a ,				_a      =							self.heap.pop()
      del self.position_map[elem]
      self.elements -= 1
      if self.elements > 0:
           _a ,				_a      =							self.heap[0]
           self._bubble_down(__magic_name__  )
      return elem
 def 							__UpperCAmelCase						(	self ,			__magic_name__ ,			__magic_name__  )   ->							None:
      # Update the weight of the given key
      _a      =							self.position_map[elem]
      _a      =							(elem, weight)
      if position > 0:
           _a      =							get_parent_position(__magic_name__  )
           _a ,				_a      =							self.heap[parent_position]
           if parent_weight > weight:
                self._bubble_up(__magic_name__  )
           else:
                self._bubble_down(__magic_name__  )
      else:
           self._bubble_down(__magic_name__  )
 def 							__UpperCAmelCase						(	self ,			__magic_name__  )   ->							None:
      # Place a node at the proper position (upward movement) [to be used internally
      # only]
      _a      =							self.position_map[elem]
      if curr_pos == 0:
           return None
      _a      =							get_parent_position(__magic_name__  )
      _a ,				_a      =							self.heap[curr_pos]
      _a ,				_a      =							self.heap[parent_position]
      if parent_weight > weight:
           self._swap_nodes(__magic_name__ ,			__magic_name__  )
           return self._bubble_up(__magic_name__  )
      return None
 def 							__UpperCAmelCase						(	self ,			__magic_name__  )   ->							None:
      # Place a node at the proper position (downward movement) [to be used
      # internally only]
      _a      =							self.position_map[elem]
      _a ,				_a      =							self.heap[curr_pos]
      _a      =							get_child_left_position(__magic_name__  )
      _a      =							get_child_right_position(__magic_name__  )
      if child_left_position < self.elements and child_right_position < self.elements:
           _a ,				_a      =							self.heap[child_left_position]
           _a ,				_a      =							self.heap[child_right_position]
           if child_right_weight < child_left_weight and child_right_weight < weight:
                self._swap_nodes(__magic_name__ ,			__magic_name__  )
                return self._bubble_down(__magic_name__  )
      if child_left_position < self.elements:
           _a ,				_a      =							self.heap[child_left_position]
           if child_left_weight < weight:
                self._swap_nodes(__magic_name__ ,			__magic_name__  )
                return self._bubble_down(__magic_name__  )
      else:
           return None
      if child_right_position < self.elements:
           _a ,				_a      =							self.heap[child_right_position]
           if child_right_weight < weight:
                self._swap_nodes(__magic_name__ ,			__magic_name__  )
                return self._bubble_down(__magic_name__  )
      return None




 def 							__UpperCAmelCase						(	self ,			__magic_name__ ,			__magic_name__  )   ->							None:
      # Swap the nodes at the given positions
      _a      =							self.heap[nodea_pos][0]
      _a      =							self.heap[nodea_pos][0]
      _a ,				_a      =							(
          self.heap[nodea_pos],
          self.heap[nodea_pos],
      )
      _a      =							nodea_pos
      _a      =							nodea_pos





class 	a       (       Generic[T]    ):
 def __init__(	self  )   ->							None:
      _a      =							{}
      _a      =							0
 def __repr__(	self  )   ->							str:
      return str(self.connections  )
 def __len__(	self  )   ->							int:
      return self.nodes
 def 							__UpperCAmelCase						(	self ,			__magic_name__  )   ->							None:
      # Add a node in the graph if it is not in the graph
      if node not in self.connections:
           _a      =							{}
           self.nodes += 1




 def 							__UpperCAmelCase						(	self ,			__magic_name__ ,			__magic_name__ ,			__magic_name__  )   ->							None:
      # Add an edge between 2 nodes in the graph
      self.add_node(__magic_name__  )
      self.add_node(__magic_name__  )
      _a      =							weight
      _a      =							weight






def 		_A			(lowerCAmelCase__					:GraphUndirectedWeighted[T]			,  )				->					tuple[dict[T, int], dict[T, T | None]]:




     '''simple docstring'''



     _a      =							{node: maxsize for node in graph.connections}
     _a      =							{node: None for node in graph.connections}

     _a      =							MinPriorityQueue()
     for node, weight in dist.items():
          priority_queue.push(lowerCAmelCase__			,  lowerCAmelCase__						)

     if priority_queue.is_empty():
          return dist, parent

     # initialization
     _a      =							priority_queue.extract_min()
     _a      =							0
     for neighbour in graph.connections[node]:
          if dist[neighbour] > dist[node] + graph.connections[node][neighbour]:
               _a      =							dist[node] + graph.connections[node][neighbour]
               priority_queue.update_key(lowerCAmelCase__			,  dist[neighbour]						)
               _a      =							node

    # running prim's algorithm
     while not priority_queue.is_empty():
          _a      =							priority_queue.extract_min()
          for neighbour in graph.connections[node]:
               if dist[neighbour] > dist[node] + graph.connections[node][neighbour]:
                    _a      =							dist[node] + graph.connections[node][neighbour]
                    priority_queue.update_key(lowerCAmelCase__			,  dist[neighbour]						)
                    _a      =							node
     return dist, parent

