


import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase	=							logging.get_logger(__name__)

lowerCAmelCase	=							'▁'

lowerCAmelCase	=							{
    'vocab_file': 'vocab.json',
    'spm_file': 'sentencepiece.bpe.model',
    'tokenizer_config_file': 'tokenizer_config.json',
}

lowerCAmelCase	=							{
    'vocab_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json',
    },
    'spm_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model',
    },
    'tokenizer_config_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json',
    },
}

lowerCAmelCase	=							{
    'facebook/m2m100_418M': 1024,
}

# fmt: off
lowerCAmelCase	=							{
    'm2m100': ['af', 'am', 'ar', 'ast', 'az', 'ba', 'be', 'bg', 'bn', 'br', 'bs', 'ca', 'ceb', 'cs', 'cy', 'da', 'de', 'el', 'en', 'es', 'et', 'fa', 'ff', 'fi', 'fr', 'fy', 'ga', 'gd', 'gl', 'gu', 'ha', 'he', 'hi', 'hr', 'ht', 'hu', 'hy', 'id', 'ig', 'ilo', 'is', 'it', 'ja', 'jv', 'ka', 'kk', 'km', 'kn', 'ko', 'lb', 'lg', 'ln', 'lo', 'lt', 'lv', 'mg', 'mk', 'ml', 'mn', 'mr', 'ms', 'my', 'ne', 'nl', 'no', 'ns', 'oc', 'or', 'pa', 'pl', 'ps', 'pt', 'ro', 'ru', 'sd', 'si', 'sk', 'sl', 'so', 'sq', 'sr', 'ss', 'su', 'sv', 'sw', 'ta', 'th', 'tl', 'tn', 'tr', 'uk', 'ur', 'uz', 'vi', 'wo', 'xh', 'yi', 'yo', 'zh', 'zu'],
    'wmt21': ['en', 'ha', 'is', 'ja', 'cs', 'ru', 'zh', 'de']
}


class 	_a							(      UpperCamelCase__      ):
							_lowercase     :      Union[str, Any]  				=			VOCAB_FILES_NAMES
							_lowercase     :      str  				=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							_lowercase     :      str  				=			PRETRAINED_VOCAB_FILES_MAP
							_lowercase     :      List[str]  				=			['''input_ids''', '''attention_mask''']

							_lowercase     :      List[int]  				=			[]
							_lowercase     :      List[int]  				=			[]

							def __init__(							self:     Optional[Any]   ,						UpperCamelCase_:     Union[str, Any]   ,						UpperCamelCase_:     Optional[int]   ,						UpperCamelCase_:     str=None   ,						UpperCamelCase_:     int=None   ,						UpperCamelCase_:     Any="<s>"   ,						UpperCamelCase_:     Union[str, Any]="</s>"   ,						UpperCamelCase_:     int="</s>"   ,						UpperCamelCase_:     List[str]="<pad>"   ,						UpperCamelCase_:     Any="<unk>"   ,						UpperCamelCase_:     Optional[int]="m2m100"   ,						UpperCamelCase_:     Optional[Dict[str, Any]] = None   ,						UpperCamelCase_:     str=8   ,						**UpperCamelCase_:     Optional[Any]   ,						)      ->       None:





									"""simple docstring"""







									lowercase__						=				{} if sp_model_kwargs is None else sp_model_kwargs

									lowercase__						=				language_codes
									lowercase__						=				FAIRSEQ_LANGUAGE_CODES[language_codes]
									lowercase__						=				{lang_code: f'__{lang_code}__' for lang_code in fairseq_language_code}

									lowercase__						=				kwargs.get('''additional_special_tokens'''   ,						[]							)
									kwargs["additional_special_tokens"] += [
									    self.get_lang_token(UpperCamelCase_							)
									    for lang_code in fairseq_language_code
									    if self.get_lang_token(UpperCamelCase_							) not in kwargs["additional_special_tokens"]
									]

									super().__init__(
									    src_lang=UpperCamelCase_   ,						tgt_lang=UpperCamelCase_   ,						bos_token=UpperCamelCase_   ,						eos_token=UpperCamelCase_   ,						sep_token=UpperCamelCase_   ,						unk_token=UpperCamelCase_   ,						pad_token=UpperCamelCase_   ,						language_codes=UpperCamelCase_   ,						sp_model_kwargs=self.sp_model_kwargs   ,						num_madeup_words=UpperCamelCase_   ,						**UpperCamelCase_   ,						)

									lowercase__						=				vocab_file
									lowercase__						=				load_json(UpperCamelCase_							)
									lowercase__						=				{v: k for k, v in self.encoder.items()}
									lowercase__						=				spm_file
									lowercase__						=				load_spm(UpperCamelCase_   ,						self.sp_model_kwargs							)

									lowercase__						=				len(self.encoder							)

									lowercase__						=				{
									    self.get_lang_token(UpperCamelCase_							): self.encoder_size + i for i, lang_code in enumerate(UpperCamelCase_							)
									}
									lowercase__						=				{lang_code: self.encoder_size + i for i, lang_code in enumerate(UpperCamelCase_							)}
									lowercase__						=				{v: k for k, v in self.lang_token_to_id.items()}

									lowercase__						=				src_lang if src_lang is not None else '''en'''
									lowercase__						=				tgt_lang
									lowercase__						=				self.get_lang_id(self._src_lang							)
									self.set_src_lang_special_tokens(self._src_lang							)

									lowercase__						=				num_madeup_words

							@property
							def 				lowerCamelCase_			(							self:     Optional[Any]							)      ->       int:





									"""simple docstring"""







									return len(self.encoder							) + len(self.lang_token_to_id							)

							@property
							def 				lowerCamelCase_			(							self:     Optional[Any]							)      ->       str:





									"""simple docstring"""







									return self._src_lang

							@src_lang.setter
							def 				lowerCamelCase_			(							self:     Any   ,						UpperCamelCase_:     str							)      ->       None:





									"""simple docstring"""







									lowercase__						=				new_src_lang
									self.set_src_lang_special_tokens(self._src_lang							)

							def 				lowerCamelCase_			(							self:     int   ,						UpperCamelCase_:     str							)      ->       List[str]:





									"""simple docstring"""







									return self.sp_model.encode(UpperCamelCase_   ,						out_type=UpperCamelCase_							)

							def 				lowerCamelCase_			(							self:     List[str]   ,						UpperCamelCase_:     List[str]							)      ->       int:





									"""simple docstring"""







									if token in self.lang_token_to_id:
											return self.lang_token_to_id[token]
									return self.encoder.get(UpperCamelCase_   ,						self.encoder[self.unk_token]							)

							def 				lowerCamelCase_			(							self:     List[Any]   ,						UpperCamelCase_:     int							)      ->       str:





									"""simple docstring"""







									if index in self.id_to_lang_token:
											return self.id_to_lang_token[index]
									return self.decoder.get(UpperCamelCase_   ,						self.unk_token							)

							def 				lowerCamelCase_			(							self:     Optional[int]   ,						UpperCamelCase_:     Union[str, Any]							)      ->       List[str]:





									"""simple docstring"""







									lowercase__						=				[]
									lowercase__						=				''''''
									for token in tokens:
											# make sure that special tokens are not decoded using sentencepiece model
											if token in self.all_special_tokens:
													out_string += self.sp_model.decode(UpperCamelCase_							) + token
													lowercase__						=				[]
											else:
													current_sub_tokens.append(UpperCamelCase_							)
									out_string += self.sp_model.decode(UpperCamelCase_							)
									return out_string.strip()

							def 				lowerCamelCase_			(							self:     Union[str, Any]   ,						UpperCamelCase_:     List[int]   ,						UpperCamelCase_:     Optional[List[int]] = None   ,						UpperCamelCase_:     bool = False							)      ->       List[int]:





									"""simple docstring"""







									if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=UpperCamelCase_   ,						token_ids_a=UpperCamelCase_   ,						already_has_special_tokens=UpperCamelCase_							)

									lowercase__						=				[1] * len(self.prefix_tokens							)
									lowercase__						=				[1] * len(self.suffix_tokens							)
									if token_ids_a is None:
											return prefix_ones + ([0] * len(UpperCamelCase_							)) + suffix_ones
									return prefix_ones + ([0] * len(UpperCamelCase_							)) + ([0] * len(UpperCamelCase_							)) + suffix_ones

							def 				lowerCamelCase_			(							self:     List[Any]   ,						UpperCamelCase_:     List[int]   ,						UpperCamelCase_:     Optional[List[int]] = None							)      ->       List[int]:





									"""simple docstring"""







									if token_ids_a is None:
											return self.prefix_tokens + token_ids_a + self.suffix_tokens
									# We don't expect to process pairs, but leave the pair logic for API consistency
									return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens

							def 				lowerCamelCase_			(							self:     Optional[Any]							)      ->       Dict:





									"""simple docstring"""







									lowercase__						=				{self.convert_ids_to_tokens(UpperCamelCase_							): i for i in range(self.vocab_size							)}
									vocab.update(self.added_tokens_encoder							)
									return vocab

							def __getstate__(							self:     Dict							)      ->       Dict:





									"""simple docstring"""







									lowercase__						=				self.__dict__.copy()
									lowercase__						=				None
									return state

							def __setstate__(							self:     Union[str, Any]   ,						UpperCamelCase_:     Dict							)      ->       None:





									"""simple docstring"""







									lowercase__						=				d

									# for backward compatibility
									if not hasattr(self   ,						'''sp_model_kwargs'''							):
											lowercase__						=				{}

									lowercase__						=				load_spm(self.spm_file   ,						self.sp_model_kwargs							)

							def 				lowerCamelCase_			(							self:     List[str]   ,						UpperCamelCase_:     str   ,						UpperCamelCase_:     Optional[str] = None							)      ->       Tuple[str]:





									"""simple docstring"""







									lowercase__						=				Path(UpperCamelCase_							)
									if not save_dir.is_dir():
											raise OSError(f'{save_directory} should be a directory'							)
									lowercase__						=				save_dir / (
									    (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''vocab_file''']
									)
									lowercase__						=				save_dir / (
									    (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''spm_file''']
									)

									save_json(self.encoder   ,						UpperCamelCase_							)

									if os.path.abspath(self.spm_file							) != os.path.abspath(UpperCamelCase_							) and os.path.isfile(self.spm_file							):
											copyfile(self.spm_file   ,						UpperCamelCase_							)
									elif not os.path.isfile(self.spm_file							):
											with open(UpperCamelCase_   ,						'''wb'''							) as fi:
													lowercase__						=				self.sp_model.serialized_model_proto()
													fi.write(UpperCamelCase_							)

									return (str(UpperCamelCase_							), str(UpperCamelCase_							))

							def 				lowerCamelCase_			(							self:     Optional[Any]   ,						UpperCamelCase_:     List[str]   ,						UpperCamelCase_:     str = "en"   ,						UpperCamelCase_:     Optional[List[str]] = None   ,						UpperCamelCase_:     str = "ro"   ,						**UpperCamelCase_:     List[str]   ,						)      ->       BatchEncoding:





									"""simple docstring"""







									lowercase__						=				src_lang
									lowercase__						=				tgt_lang
									self.set_src_lang_special_tokens(self.src_lang							)
									return super().prepare_seqaseq_batch(UpperCamelCase_   ,						UpperCamelCase_   ,						**UpperCamelCase_							)

							def 				lowerCamelCase_			(							self:     Optional[int]   ,						UpperCamelCase_:     Tuple   ,						UpperCamelCase_:     Optional[str]   ,						UpperCamelCase_:     Optional[str]   ,						**UpperCamelCase_:     Any							)      ->       Union[str, Any]:





									"""simple docstring"""







									if src_lang is None or tgt_lang is None:
											raise ValueError('''Translation requires a `src_lang` and a `tgt_lang` for this model'''							)
									lowercase__						=				src_lang
									lowercase__						=				self(UpperCamelCase_   ,						add_special_tokens=UpperCamelCase_   ,						**UpperCamelCase_							)
									lowercase__						=				self.get_lang_id(UpperCamelCase_							)
									lowercase__						=				tgt_lang_id
									return inputs

							def 				lowerCamelCase_			(							self:     Dict							)      ->       Optional[Any]:





									"""simple docstring"""







									self.set_src_lang_special_tokens(self.src_lang							)

							def 				lowerCamelCase_			(							self:     int							)      ->       Dict:





									"""simple docstring"""







									self.set_tgt_lang_special_tokens(self.tgt_lang							)

							def 				lowerCamelCase_			(							self:     Dict   ,						UpperCamelCase_:     str							)      ->       None:





									"""simple docstring"""







									lowercase__						=				self.get_lang_token(UpperCamelCase_							)
									lowercase__						=				self.lang_token_to_id[lang_token]
									lowercase__						=				[self.cur_lang_id]
									lowercase__						=				[self.eos_token_id]

							def 				lowerCamelCase_			(							self:     int   ,						UpperCamelCase_:     str							)      ->       None:





									"""simple docstring"""







									lowercase__						=				self.get_lang_token(UpperCamelCase_							)
									lowercase__						=				self.lang_token_to_id[lang_token]
									lowercase__						=				[self.cur_lang_id]
									lowercase__						=				[self.eos_token_id]

							def 				lowerCamelCase_			(							self:     List[str]   ,						UpperCamelCase_:     str							)      ->       str:





									"""simple docstring"""







									return self.lang_code_to_token[lang]


							def 				lowerCamelCase_			(							self:     List[Any]   ,						UpperCamelCase_:     str							)      ->       int:





									"""simple docstring"""







									lowercase__						=				self.get_lang_token(UpperCamelCase_							)
									return self.lang_token_to_id[lang_token]
def       _a			(				SCREAMING_SNAKE_CASE						,    SCREAMING_SNAKE_CASE					):


		"""simple docstring"""




		lowercase__						=				sentencepiece.SentencePieceProcessor(**SCREAMING_SNAKE_CASE					)
		spm.Load(str(SCREAMING_SNAKE_CASE					)					)
		return spm
def       _a			(				SCREAMING_SNAKE_CASE					):


		"""simple docstring"""




		with open(SCREAMING_SNAKE_CASE						,    '''r'''					) as f:
				return json.load(SCREAMING_SNAKE_CASE					)
def       _a			(				SCREAMING_SNAKE_CASE						,    SCREAMING_SNAKE_CASE					):


		"""simple docstring"""




		with open(SCREAMING_SNAKE_CASE						,    '''w'''					) as f:
				json.dump(SCREAMING_SNAKE_CASE						,    SCREAMING_SNAKE_CASE						,    indent=2					)






import math_equivalence  # From: git+https://github.com/hendrycks/math.git

import datasets


lowerCAmelCase	=							'\\n@article{hendrycksmath2021,\n  title={Measuring Mathematical Problem Solving With the MATH Dataset},\n  author={Dan Hendrycks\n    and Collin Burns\n    and Saurav Kadavath\n    and Akul Arora\n    and Steven Basart\n    and Eric Tang\n    and Dawn Song\n    and Jacob Steinhardt},\n  journal={arXiv preprint arXiv:2103.03874},\n  year={2021}\n}\n'


lowerCAmelCase	=							'\\nThis metric is used to assess performance on the Mathematics Aptitude Test of Heuristics (MATH) dataset.\nIt first canonicalizes the inputs (e.g., converting "1/2" to "\\frac{1}{2}") and then computes accuracy.\n'


lowerCAmelCase	=							R'\nCalculates accuracy after canonicalizing inputs.\n\nArgs:\n    predictions: list of predictions to score. Each prediction\n        is a string that contains natural language and LaTex.\n    references: list of reference for each prediction. Each\n        reference is a string that contains natural language\n        and LaTex.\nReturns:\n    accuracy: accuracy after canonicalizing inputs\n        (e.g., converting "1/2" to "\\frac{1}{2}")\n\nExamples:\n    >>> metric = datasets.load_metric("competition_math")\n    >>> results = metric.compute(references=["\\frac{1}{2}"], predictions=["1/2"])\n    >>> print(results)\n    {\'accuracy\': 1.0}\n'


@datasets.utils.file_utils.add_end_docstrings(_DESCRIPTION					,     _KWARGS_DESCRIPTION      )
class 	_a							(      datasets.Metric      ):

							def 				lowerCamelCase_			(							self:     List[str]							)      ->       Tuple:





									"""simple docstring"""







									return datasets.MetricInfo(
									    description=_DESCRIPTION   ,						citation=_CITATION   ,						inputs_description=_KWARGS_DESCRIPTION   ,						features=datasets.Features(
									        {
									            '''predictions''': datasets.Value('''string'''							),
									            '''references''': datasets.Value('''string'''							),
									        }							)   ,						homepage='''https://github.com/hendrycks/math'''   ,						codebase_urls=['''https://github.com/hendrycks/math''']   ,						)


							def 				lowerCamelCase_			(							self:     Union[str, Any]   ,						UpperCamelCase_:     Any   ,						UpperCamelCase_:     int							)      ->       List[str]:





									"""simple docstring"""







									lowercase__						=				0.0
									for i, j in zip(UpperCamelCase_   ,						UpperCamelCase_							):
											n_correct += 1.0 if math_equivalence.is_equiv(UpperCamelCase_   ,						UpperCamelCase_							) else 0.0
									lowercase__						=				n_correct / len(UpperCamelCase_							)
									return {
									    "accuracy": accuracy,
									}


