


"""simple docstring"""




import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roberta import RobertaTokenizer


lowerCAmelCase	=							logging.get_logger(__name__)

lowerCAmelCase	=							{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

lowerCAmelCase	=							{
    'vocab_file': {
        'roberta-base': 'https://huggingface.co/roberta-base/resolve/main/vocab.json',
        'roberta-large': 'https://huggingface.co/roberta-large/resolve/main/vocab.json',
        'roberta-large-mnli': 'https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json',
        'distilroberta-base': 'https://huggingface.co/distilroberta-base/resolve/main/vocab.json',
        'roberta-base-openai-detector': 'https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json',
        'roberta-large-openai-detector': (
            'https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json'
        ),
    },
    'merges_file': {
        'roberta-base': 'https://huggingface.co/roberta-base/resolve/main/merges.txt',
        'roberta-large': 'https://huggingface.co/roberta-large/resolve/main/merges.txt',
        'roberta-large-mnli': 'https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt',
        'distilroberta-base': 'https://huggingface.co/distilroberta-base/resolve/main/merges.txt',
        'roberta-base-openai-detector': 'https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt',
        'roberta-large-openai-detector': (
            'https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt'
        ),
    },
    'tokenizer_file': {
        'roberta-base': 'https://huggingface.co/roberta-base/resolve/main/tokenizer.json',
        'roberta-large': 'https://huggingface.co/roberta-large/resolve/main/tokenizer.json',
        'roberta-large-mnli': 'https://huggingface.co/roberta-large-mnli/resolve/main/tokenizer.json',
        'distilroberta-base': 'https://huggingface.co/distilroberta-base/resolve/main/tokenizer.json',
        'roberta-base-openai-detector': (
            'https://huggingface.co/roberta-base-openai-detector/resolve/main/tokenizer.json'
        ),
        'roberta-large-openai-detector': (
            'https://huggingface.co/roberta-large-openai-detector/resolve/main/tokenizer.json'
        ),
    },
}

lowerCAmelCase	=							{
    'roberta-base': 512,
    'roberta-large': 512,
    'roberta-large-mnli': 512,
    'distilroberta-base': 512,
    'roberta-base-openai-detector': 512,
    'roberta-large-openai-detector': 512,
}


class 	_a							(      UpperCamelCase__      ):
							_lowercase     :      Union[str, Any]  				=			VOCAB_FILES_NAMES
							_lowercase     :      Any  				=			PRETRAINED_VOCAB_FILES_MAP
							_lowercase     :      Any  				=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							_lowercase     :      Optional[int]  				=			['''input_ids''', '''attention_mask''']
							_lowercase     :      Dict  				=			RobertaTokenizer

							def __init__(							self:     int   ,						UpperCamelCase_:     Any=None   ,						UpperCamelCase_:     Any=None   ,						UpperCamelCase_:     List[str]=None   ,						UpperCamelCase_:     Optional[int]="replace"   ,						UpperCamelCase_:     Any="<s>"   ,						UpperCamelCase_:     List[Any]="</s>"   ,						UpperCamelCase_:     Optional[Any]="</s>"   ,						UpperCamelCase_:     Dict="<s>"   ,						UpperCamelCase_:     List[Any]="<unk>"   ,						UpperCamelCase_:     Tuple="<pad>"   ,						UpperCamelCase_:     Any="<mask>"   ,						UpperCamelCase_:     Optional[Any]=False   ,						UpperCamelCase_:     int=True   ,						**UpperCamelCase_:     Optional[Any]   ,						)      ->       Optional[Any]:





									"""simple docstring"""







									super().__init__(
									    UpperCamelCase_   ,						UpperCamelCase_   ,						tokenizer_file=UpperCamelCase_   ,						errors=UpperCamelCase_   ,						bos_token=UpperCamelCase_   ,						eos_token=UpperCamelCase_   ,						sep_token=UpperCamelCase_   ,						cls_token=UpperCamelCase_   ,						unk_token=UpperCamelCase_   ,						pad_token=UpperCamelCase_   ,						mask_token=UpperCamelCase_   ,						add_prefix_space=UpperCamelCase_   ,						trim_offsets=UpperCamelCase_   ,						**UpperCamelCase_   ,						)

									lowercase__						=				json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()							)
									if pre_tok_state.get('''add_prefix_space'''   ,						UpperCamelCase_							) != add_prefix_space:
											lowercase__						=				getattr(UpperCamelCase_   ,						pre_tok_state.pop('''type'''							)							)
											lowercase__						=				add_prefix_space
											lowercase__						=				pre_tok_class(**UpperCamelCase_							)

									lowercase__						=				add_prefix_space

									lowercase__						=				'''post_processor'''
									lowercase__						=				getattr(self.backend_tokenizer   ,						UpperCamelCase_   ,						UpperCamelCase_							)
									if tokenizer_component_instance:
											lowercase__						=				json.loads(tokenizer_component_instance.__getstate__()							)

											# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
											if "sep" in state:
													lowercase__						=				tuple(state['''sep''']							)
											if "cls" in state:
													lowercase__						=				tuple(state['''cls''']							)

											lowercase__						=				False

											if state.get('''add_prefix_space'''   ,						UpperCamelCase_							) != add_prefix_space:
													lowercase__						=				add_prefix_space
													lowercase__						=				True

											if state.get('''trim_offsets'''   ,						UpperCamelCase_							) != trim_offsets:
													lowercase__						=				trim_offsets
													lowercase__						=				True

											if changes_to_apply:
													lowercase__						=				getattr(UpperCamelCase_   ,						state.pop('''type'''							)							)
													lowercase__						=				component_class(**UpperCamelCase_							)
													setattr(self.backend_tokenizer   ,						UpperCamelCase_   ,						UpperCamelCase_							)

							@property
							def 				lowerCamelCase_			(							self:     List[str]							)      ->       str:





									"""simple docstring"""







									if self._mask_token is None:
											if self.verbose:
													logger.error('''Using mask_token, but it is not set yet.'''							)
											return None
									return str(self._mask_token							)

							@mask_token.setter
							def 				lowerCamelCase_			(							self:     Dict   ,						UpperCamelCase_:     Tuple							)      ->       Tuple:





									"""simple docstring"""







									lowercase__						=				AddedToken(UpperCamelCase_   ,						lstrip=UpperCamelCase_   ,						rstrip=UpperCamelCase_							) if isinstance(UpperCamelCase_   ,						UpperCamelCase_							) else value
									lowercase__						=				value

							def 				lowerCamelCase_			(							self:     str   ,						*UpperCamelCase_:     Any   ,						**UpperCamelCase_:     List[Any]							)      ->       BatchEncoding:





									"""simple docstring"""







									lowercase__						=				kwargs.get('''is_split_into_words'''   ,						UpperCamelCase_							)
									assert self.add_prefix_space or not is_split_into_words, (
									    f'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
									    "to use it with pretokenized inputs."
									)

									return super()._batch_encode_plus(*UpperCamelCase_   ,						**UpperCamelCase_							)

							def 				lowerCamelCase_			(							self:     Any   ,						*UpperCamelCase_:     List[Any]   ,						**UpperCamelCase_:     int							)      ->       BatchEncoding:





									"""simple docstring"""







									lowercase__						=				kwargs.get('''is_split_into_words'''   ,						UpperCamelCase_							)

									assert self.add_prefix_space or not is_split_into_words, (
									    f'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
									    "to use it with pretokenized inputs."
									)

									return super()._encode_plus(*UpperCamelCase_   ,						**UpperCamelCase_							)

							def 				lowerCamelCase_			(							self:     Optional[int]   ,						UpperCamelCase_:     str   ,						UpperCamelCase_:     Optional[str] = None							)      ->       Tuple[str]:





									"""simple docstring"""







									lowercase__						=				self._tokenizer.model.save(UpperCamelCase_   ,						name=UpperCamelCase_							)
									return tuple(UpperCamelCase_							)

							def 				lowerCamelCase_			(							self:     Union[str, Any]   ,						UpperCamelCase_:     Optional[Any]   ,						UpperCamelCase_:     str=None							)      ->       List[str]:





									"""simple docstring"""







									lowercase__						=				[self.bos_token_id] + token_ids_a + [self.eos_token_id]
									if token_ids_a is None:
											return output

									return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]


							def 				lowerCamelCase_			(							self:     Optional[Any]   ,						UpperCamelCase_:     List[int]   ,						UpperCamelCase_:     Optional[List[int]] = None							)      ->       List[int]:





									"""simple docstring"""







									lowercase__						=				[self.sep_token_id]
									lowercase__						=				[self.cls_token_id]

									if token_ids_a is None:
											return len(cls + token_ids_a + sep							) * [0]
									return len(cls + token_ids_a + sep + sep + token_ids_a + sep							) * [0]






import math_equivalence  # From: git+https://github.com/hendrycks/math.git

import datasets


lowerCAmelCase	=							'\\n@article{hendrycksmath2021,\n  title={Measuring Mathematical Problem Solving With the MATH Dataset},\n  author={Dan Hendrycks\n    and Collin Burns\n    and Saurav Kadavath\n    and Akul Arora\n    and Steven Basart\n    and Eric Tang\n    and Dawn Song\n    and Jacob Steinhardt},\n  journal={arXiv preprint arXiv:2103.03874},\n  year={2021}\n}\n'


lowerCAmelCase	=							'\\nThis metric is used to assess performance on the Mathematics Aptitude Test of Heuristics (MATH) dataset.\nIt first canonicalizes the inputs (e.g., converting "1/2" to "\\frac{1}{2}") and then computes accuracy.\n'


lowerCAmelCase	=							R'\nCalculates accuracy after canonicalizing inputs.\n\nArgs:\n    predictions: list of predictions to score. Each prediction\n        is a string that contains natural language and LaTex.\n    references: list of reference for each prediction. Each\n        reference is a string that contains natural language\n        and LaTex.\nReturns:\n    accuracy: accuracy after canonicalizing inputs\n        (e.g., converting "1/2" to "\\frac{1}{2}")\n\nExamples:\n    >>> metric = datasets.load_metric("competition_math")\n    >>> results = metric.compute(references=["\\frac{1}{2}"], predictions=["1/2"])\n    >>> print(results)\n    {\'accuracy\': 1.0}\n'


@datasets.utils.file_utils.add_end_docstrings(_DESCRIPTION					,     _KWARGS_DESCRIPTION      )
class 	_a							(      datasets.Metric      ):

							def 				lowerCamelCase_			(							self:     List[str]							)      ->       Tuple:





									"""simple docstring"""







									return datasets.MetricInfo(
									    description=_DESCRIPTION   ,						citation=_CITATION   ,						inputs_description=_KWARGS_DESCRIPTION   ,						features=datasets.Features(
									        {
									            '''predictions''': datasets.Value('''string'''							),
									            '''references''': datasets.Value('''string'''							),
									        }							)   ,						homepage='''https://github.com/hendrycks/math'''   ,						codebase_urls=['''https://github.com/hendrycks/math''']   ,						)


							def 				lowerCamelCase_			(							self:     Union[str, Any]   ,						UpperCamelCase_:     Any   ,						UpperCamelCase_:     int							)      ->       List[str]:





									"""simple docstring"""







									lowercase__						=				0.0
									for i, j in zip(UpperCamelCase_   ,						UpperCamelCase_							):
											n_correct += 1.0 if math_equivalence.is_equiv(UpperCamelCase_   ,						UpperCamelCase_							) else 0.0
									lowercase__						=				n_correct / len(UpperCamelCase_							)
									return {
									    "accuracy": accuracy,
									}


