



'''simple docstring'''






from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


lowerCAmelCase: Tuple							=						'\\n@article{wang2019superglue,\n  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},\n  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},\n  journal={arXiv preprint arXiv:1905.00537},\n  year={2019}\n}\n'

lowerCAmelCase: str							=						'\\nSuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after\nGLUE with a new set of more difficult language understanding tasks, improved\nresources, and a new public leaderboard.\n'

lowerCAmelCase: Optional[int]							=						'\nCompute SuperGLUE evaluation metric associated to each SuperGLUE dataset.\nArgs:\n    predictions: list of predictions to score. Depending on the SuperGlUE subset:\n        - for \'record\': list of question-answer dictionaries with the following keys:\n            - \'idx\': index of the question as specified by the dataset\n            - \'prediction_text\': the predicted answer text\n        - for \'multirc\': list of question-answer dictionaries with the following keys:\n            - \'idx\': index of the question-answer pair as specified by the dataset\n            - \'prediction\': the predicted answer label\n        - otherwise: list of predicted labels\n    references: list of reference labels. Depending on the SuperGLUE subset:\n        - for \'record\': list of question-answers dictionaries with the following keys:\n            - \'idx\': index of the question as specified by the dataset\n            - \'answers\': list of possible answers\n        - otherwise: list of reference labels\nReturns: depending on the SuperGLUE subset:\n    - for \'record\':\n        - \'exact_match\': Exact match between answer and gold answer\n        - \'f1\': F1 score\n    - for \'multirc\':\n        - \'exact_match\': Exact match between answer and gold answer\n        - \'f1_m\': Per-question macro-F1 score\n        - \'f1_a\': Average F1 score over all answers\n    - for \'axb\':\n        \'matthews_correlation\': Matthew Correlation\n    - for \'cb\':\n        - \'accuracy\': Accuracy\n        - \'f1\': F1 score\n    - for all others:\n        - \'accuracy\': Accuracy\nExamples:\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'copa\')  # any of ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'accuracy\': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'cb\')\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'accuracy\': 1.0, \'f1\': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'record\')\n    >>> predictions = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'prediction_text\': \'answer\'}]\n    >>> references = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'answers\': [\'answer\', \'another_answer\']}]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'exact_match\': 1.0, \'f1\': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'multirc\')\n    >>> predictions = [{\'idx\': {\'answer\': 0, \'paragraph\': 0, \'question\': 0}, \'prediction\': 0}, {\'idx\': {\'answer\': 1, \'paragraph\': 2, \'question\': 3}, \'prediction\': 1}]\n    >>> references = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'exact_match\': 1.0, \'f1_m\': 1.0, \'f1_a\': 1.0}\n\n    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'axb\')\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = super_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'matthews_correlation\': 1.0}\n'
def 				lowerCamelCase__	(						_A			,      _A    ):
  return float((preds == labels).mean()    )
def 				lowerCamelCase__	(						_A			,      _A			,      _A="binary"    ):
  a						: Tuple					       =					simple_accuracy(_A			,      _A    )
  a						: Optional[int]					       =					float(fa_score(y_true=_A			,      y_pred=_A			,      average=_A    )    )
  return {
      "accuracy": acc,
      "f1": fa,
  }
def 				lowerCamelCase__	(						_A			,      _A    ):
  a						: Union[str, Any]					       =					{}
  for id_pred, label in zip(_A			,      _A    ):
    a						: Tuple					       =					f"""{id_pred["idx"]["paragraph"]}-{id_pred["idx"]["question"]}"""
    a						: int					       =					id_pred['prediction']
    if question_id in question_map:
      question_map[question_id].append((pred, label)    )
    else:
      a						: List[Any]					       =					[(pred, label)]
  a       ,     a						: Optional[int]					       =					[], []
  for question, preds_labels in question_map.items():
    a       ,     a						: List[str]					       =					zip(*_A    )
    a						: List[str]					       =					fa_score(y_true=_A			,      y_pred=_A			,      average='macro'    )
    fas.append(_A    )
    a						: str					       =					int(sum(pred == label for pred, label in preds_labels    ) == len(_A    )    )
    ems.append(_A    )
  a						: Dict					       =					float(sum(_A    ) / len(_A    )    )
  a						: Any					       =					sum(_A    ) / len(_A    )
  a						: str					       =					float(fa_score(y_true=_A			,      y_pred=[id_pred['prediction'] for id_pred in ids_preds]    )    )
  return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}

@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION		,			_KWARGS_DESCRIPTION     )
class 							a__( datasets.Metric     ):







    def       lowercase_	(				self			:  List[Any]     ):
      if self.config_name not in [
          "boolq",
          "cb",
          "copa",
          "multirc",
          "record",
          "rte",
          "wic",
          "wsc",
          "wsc.fixed",
          "axb",
          "axg",
      ]:
        raise KeyError(
            'You should supply a configuration name selected in '
            '["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]'     )
      return datasets.MetricInfo(
          description=_DESCRIPTION						,      citation=_CITATION						,      inputs_description=_KWARGS_DESCRIPTION						,      features=datasets.Features(self._get_feature_types()     )						,      codebase_urls=[]						,      reference_urls=[]						,      format='numpy' if not self.config_name == 'record' and not self.config_name == 'multirc' else None						,      )







    def       lowercase_	(				self			:  List[Any]     ):
      if self.config_name == "record":
        return {
            "predictions": {
                "idx": {
                    "passage": datasets.Value('int64'     ),
                    "query": datasets.Value('int64'     ),
                },
                "prediction_text": datasets.Value('string'     ),
            },
            "references": {
                "idx": {
                    "passage": datasets.Value('int64'     ),
                    "query": datasets.Value('int64'     ),
                },
                "answers": datasets.Sequence(datasets.Value('string'     )     ),
            },
        }
      elif self.config_name == "multirc":
        return {
            "predictions": {
                "idx": {
                    "answer": datasets.Value('int64'     ),
                    "paragraph": datasets.Value('int64'     ),
                    "question": datasets.Value('int64'     ),
                },
                "prediction": datasets.Value('int64'     ),
            },
            "references": datasets.Value('int64'     ),
        }
      else:
        return {
            "predictions": datasets.Value('int64'     ),
            "references": datasets.Value('int64'     ),
        }







    def       lowercase_	(				self			:  List[str]						,      __snake_case			:  Dict						,      __snake_case			:  List[str]     ):
      if self.config_name == "axb":
        return {"matthews_correlation": matthews_corrcoef(__snake_case						,      __snake_case     )}
      elif self.config_name == "cb":
        return acc_and_fa(__snake_case						,      __snake_case						,      fa_avg='macro'     )
      elif self.config_name == "record":
        a						: Tuple					       =					[
            {
                'qas': [
                    {'id': ref['idx']['query'], 'answers': [{'text': ans} for ans in ref['answers']]}
                    for ref in references
                ]
            }
        ]
        a						: List[str]					       =					{pred['idx']['query']: pred['prediction_text'] for pred in predictions}
        return evaluate_record(__snake_case						,      __snake_case     )[0]
      elif self.config_name == "multirc":
        return evaluate_multirc(__snake_case						,      __snake_case     )
      elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
        return {"accuracy": simple_accuracy(__snake_case						,      __snake_case     )}
      else:
        raise KeyError(
            'You should supply a configuration name selected in '
            '["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]'     )




'''simple docstring'''






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tokenizers_available,
    is_torch_available,
)


lowerCAmelCase: List[Any]							=						{}

try:
     if not is_sentencepiece_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     lowerCAmelCase: Optional[int]							=						['NllbTokenizer']

try:
     if not is_tokenizers_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     lowerCAmelCase: Dict							=						['NllbTokenizerFast']


if TYPE_CHECKING:
     try:
          if not is_sentencepiece_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .tokenization_nllb import NllbTokenizer

     try:
          if not is_tokenizers_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .tokenization_nllb_fast import NllbTokenizerFast

else:
     import sys

     lowerCAmelCase: Tuple							=						_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)