
import math_equivalence  # From: git+https://github.com/hendrycks/math.git

import datasets


lowercase__          ='\\n@article{hendrycksmath2021,\n  title={Measuring Mathematical Problem Solving With the MATH Dataset},\n  author={Dan Hendrycks\n    and Collin Burns\n    and Saurav Kadavath\n    and Akul Arora\n    and Steven Basart\n    and Eric Tang\n    and Dawn Song\n    and Jacob Steinhardt},\n  journal={arXiv preprint arXiv:2103.03874},\n  year={2021}\n}\n'


lowercase__          ='\\nThis metric is used to assess performance on the Mathematics Aptitude Test of Heuristics (MATH) dataset.\nIt first canonicalizes the inputs (e.g., converting "1/2" to "\\frac{1}{2}") and then computes accuracy.\n'


lowercase__          =R'\nCalculates accuracy after canonicalizing inputs.\n\nArgs:\n    predictions: list of predictions to score. Each prediction\n        is a string that contains natural language and LaTex.\n    references: list of reference for each prediction. Each\n        reference is a string that contains natural language\n        and LaTex.\nReturns:\n    accuracy: accuracy after canonicalizing inputs\n        (e.g., converting "1/2" to "\\frac{1}{2}")\n\nExamples:\n    >>> metric = datasets.load_metric("competition_math")\n    >>> results = metric.compute(references=["\\frac{1}{2}"], predictions=["1/2"])\n    >>> print(results)\n    {\'accuracy\': 1.0}\n'



@datasets.utils.file_utils.add_end_docstrings(_DESCRIPTION   ,_KWARGS_DESCRIPTION   )
class 		UpperCamelCase__  (	datasets.Metric   ):
   def 				lowerCAmelCase      (self			: Any       ):
    return datasets.MetricInfo(
        description=_DESCRIPTION ,    citation=_CITATION ,    inputs_description=_KWARGS_DESCRIPTION ,    features=datasets.Features(
            {
                '''predictions''': datasets.Value('''string'''       ),
                '''references''': datasets.Value('''string'''       ),
            }       ) ,    homepage='''https://github.com/hendrycks/math''' ,    codebase_urls=['''https://github.com/hendrycks/math'''] ,    )







   def 				lowerCAmelCase      (self			: Tuple ,    snake_case_			: Optional[int] ,    snake_case_			: Any       ):
    __a			:  Any		    =  0.0
    for i, j in zip(snake_case_ ,    snake_case_       ):
     n_correct += 1.0 if math_equivalence.is_equiv(snake_case_ ,    snake_case_       ) else 0.0
    __a			:  Union[str, Any]		    =  n_correct / len(snake_case_       )
    return {
        "accuracy": accuracy,
    }


import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device


if is_torch_available():
      from transformers import AutoModelForSeqaSeqLM, AutoTokenizer



@require_torch
@require_sentencepiece
@require_tokenizers
class 		UpperCamelCase__  (	unittest.TestCase   ):
   @slow
   def 				lowerCAmelCase      (self			: Tuple       ):
    __a			:  List[str]		    =  AutoModelForSeqaSeqLM.from_pretrained('''google/mt5-small''' ,    return_dict=snake_case_       ).to(snake_case_       )
    __a			:  List[Any]		    =  AutoTokenizer.from_pretrained('''google/mt5-small'''       )

    __a			:  Optional[int]		    =  tokenizer('''Hello there''' ,    return_tensors='''pt'''       ).input_ids
    __a			:  Dict		    =  tokenizer('''Hi I am''' ,    return_tensors='''pt'''       ).input_ids

    __a			:  Optional[Any]		    =  model(input_ids.to(snake_case_       ) ,    labels=labels.to(snake_case_       )       ).loss
    __a			:  Tuple		    =  -(labels.shape[-1] * loss.item())

    __a			:  Dict		    =  -84.9127
    self.assertTrue(abs(mtf_score - EXPECTED_SCORE       ) < 1E-4       )
