






from scipy.stats import pearsonr, spearmanr
from sklearn.metrics import fa_score, matthews_corrcoef

import datasets


UpperCamelCase							    =     '''\
@inproceedings{wang2019glue,
  title={{GLUE}: A Multi-Task Benchmark and Analysis Platform for Natural Language Understanding},
  author={Wang, Alex and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R.},
  note={In the Proceedings of ICLR.},
  year={2019}
}
'''

UpperCamelCase							    =     '''\
GLUE, the General Language Understanding Evaluation benchmark
(https://gluebenchmark.com/) is a collection of resources for training,
evaluating, and analyzing natural language understanding systems.
'''

UpperCamelCase							    =     '''
Compute GLUE evaluation metric associated to each GLUE dataset.
Args:
    predictions: list of predictions to score.
        Each translation should be tokenized into a list of tokens.
    references: list of lists of references for each translation.
        Each reference should be tokenized into a list of tokens.
Returns: depending on the GLUE subset, one or several of:
    "accuracy": Accuracy
    "f1": F1 score
    "pearson": Pearson Correlation
    "spearmanr": Spearman Correlation
    "matthews_correlation": Matthew Correlation
Examples:

    >>> glue_metric = datasets.load_metric(\'glue\', \'sst2\')  # \'sst2\' or any of ["mnli", "mnli_mismatched", "mnli_matched", "qnli", "rte", "wnli", "hans"]
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'accuracy\': 1.0}

    >>> glue_metric = datasets.load_metric(\'glue\', \'mrpc\')  # \'mrpc\' or \'qqp\'
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'accuracy\': 1.0, \'f1\': 1.0}

    >>> glue_metric = datasets.load_metric(\'glue\', \'stsb\')
    >>> references = [0., 1., 2., 3., 4., 5.]
    >>> predictions = [0., 1., 2., 3., 4., 5.]
    >>> results = glue_metric.compute(predictions=predictions, references=references)
    >>> print({"pearson": round(results["pearson"], 2), "spearmanr": round(results["spearmanr"], 2)})
    {\'pearson\': 1.0, \'spearmanr\': 1.0}

    >>> glue_metric = datasets.load_metric(\'glue\', \'cola\')
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'matthews_correlation\': 1.0}
'''

def 	__lowerCamelCase      (		snake_case__	,snake_case__		)							->  str:






						"""simple docstring"""





						return float((preds == labels).mean()		)

def 	__lowerCamelCase      (		snake_case__	,snake_case__		)							->  str:






						"""simple docstring"""





						_SCREAMING_SNAKE_CASE     				=  simple_accuracy(snake_case__	,snake_case__		)
						_SCREAMING_SNAKE_CASE     				=  float(fa_score(y_true=snake_case__	,y_pred=snake_case__		)		)
						return {
						    "accuracy": acc,
						    "f1": fa,
						}

def 	__lowerCamelCase      (		snake_case__	,snake_case__		)							->  Dict:






						"""simple docstring"""





						_SCREAMING_SNAKE_CASE     				=  float(pearsonr(snake_case__	,snake_case__		)[0]		)
						_SCREAMING_SNAKE_CASE     				=  float(spearmanr(snake_case__	,snake_case__		)[0]		)
						return {
						    "pearson": pearson_corr,
						    "spearmanr": spearman_corr,
						}




@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION	,_KWARGS_DESCRIPTION   )
class 				__UpperCAmelCase		(datasets.Metric   ):







			def 							UpperCamelCase				(     self:  List[Any]					):







									'''simple docstring'''




									if self.config_name not in [
									    "sst2",
									    "mnli",
									    "mnli_mismatched",
									    "mnli_matched",
									    "cola",
									    "stsb",
									    "mrpc",
									    "qqp",
									    "qnli",
									    "rte",
									    "wnli",
									    "hans",
									]:
															raise KeyError(
															    """You should supply a configuration name selected in """
															    """[\"sst2\", \"mnli\", \"mnli_mismatched\", \"mnli_matched\", """
															    """\"cola\", \"stsb\", \"mrpc\", \"qqp\", \"qnli\", \"rte\", \"wnli\", \"hans\"]"""					)
									return datasets.MetricInfo(
									    description=_DESCRIPTION				,     citation=_CITATION				,     inputs_description=_KWARGS_DESCRIPTION				,     features=datasets.Features(
									        {
									            """predictions""": datasets.Value("""int64""" if self.config_name != """stsb""" else """float32"""					),
									            """references""": datasets.Value("""int64""" if self.config_name != """stsb""" else """float32"""					),
									        }					)				,     codebase_urls=[]				,     reference_urls=[]				,     format="""numpy"""				,     )







			def 							UpperCamelCase				(     self:  List[Any]				,     UpperCAmelCase_:  Any				,     UpperCAmelCase_:  List[str]					):







									'''simple docstring'''




									if self.config_name == "cola":
															return {"matthews_correlation": matthews_corrcoef(UpperCAmelCase_				,     UpperCAmelCase_					)}
									elif self.config_name == "stsb":
															return pearson_and_spearman(UpperCAmelCase_				,     UpperCAmelCase_					)
									elif self.config_name in ["mrpc", "qqp"]:
															return acc_and_fa(UpperCAmelCase_				,     UpperCAmelCase_					)
									elif self.config_name in ["sst2", "mnli", "mnli_mismatched", "mnli_matched", "qnli", "rte", "wnli", "hans"]:
															return {"accuracy": simple_accuracy(UpperCAmelCase_				,     UpperCAmelCase_					)}
									else:
															raise KeyError(
															    """You should supply a configuration name selected in """
															    """[\"sst2\", \"mnli\", \"mnli_mismatched\", \"mnli_matched\", """
															    """\"cola\", \"stsb\", \"mrpc\", \"qqp\", \"qnli\", \"rte\", \"wnli\", \"hans\"]"""					)









import copy
import re



class 				__UpperCAmelCase		:
			__snake_case    :				Any									=							"hp"
			__snake_case    :				str									=							{}
			__snake_case    :				List[Any]									=							None







			@classmethod
			def 							UpperCamelCase				(     cls:  Optional[Any]				,     UpperCAmelCase_:  int				,     UpperCAmelCase_:  Optional[int]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  prefix
									_SCREAMING_SNAKE_CASE     				=  defaults
									cls.build_naming_info()







			@staticmethod
			def 							UpperCamelCase				(     UpperCAmelCase_:  Any				,     UpperCAmelCase_:  str					):







									'''simple docstring'''




									if len(UpperCAmelCase_					) == 0:
															return ""
									_SCREAMING_SNAKE_CASE     				=  None
									if any(char.isdigit() for char in word					):
															raise Exception(F'Parameters should not contain numbers: \'{word}\' contains a number'					)
									if word in info["short_word"]:
															return info["short_word"][word]
									for prefix_len in range(1				,     len(UpperCAmelCase_					) + 1					):
															_SCREAMING_SNAKE_CASE     				=  word[:prefix_len]
															if prefix in info["reverse_short_word"]:
																					continue
															else:
																					_SCREAMING_SNAKE_CASE     				=  prefix
																					break

									if short_word is None:
															# Paranoid fallback
															def int_to_alphabetic(UpperCAmelCase_:  List[Any]					):
																					_SCREAMING_SNAKE_CASE     				=  """"""
																					while integer != 0:
																											_SCREAMING_SNAKE_CASE     				=  chr(ord("""A"""					) + integer % 10					) + s
																											integer //= 10
																					return s

															_SCREAMING_SNAKE_CASE     				=  0
															while True:
																					_SCREAMING_SNAKE_CASE     				=  word + """#""" + int_to_alphabetic(UpperCAmelCase_					)
																					if sword in info["reverse_short_word"]:
																											continue
																					else:
																											_SCREAMING_SNAKE_CASE     				=  sword
																											break

									_SCREAMING_SNAKE_CASE     				=  short_word
									_SCREAMING_SNAKE_CASE     				=  word
									return short_word







			@staticmethod
			def 							UpperCamelCase				(     UpperCAmelCase_:  List[str]				,     UpperCAmelCase_:  List[str]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  param_name.split("""_"""					)

									_SCREAMING_SNAKE_CASE     				=  [TrialShortNamer.shortname_for_word(UpperCAmelCase_				,     UpperCAmelCase_					) for word in words]

									# We try to create a separatorless short name, but if there is a collision we have to fallback
									# to a separated short name
									_SCREAMING_SNAKE_CASE     				=  ["""""", """_"""]

									for separator in separators:
															_SCREAMING_SNAKE_CASE     				=  separator.join(UpperCAmelCase_					)
															if shortname not in info["reverse_short_param"]:
																					_SCREAMING_SNAKE_CASE     				=  shortname
																					_SCREAMING_SNAKE_CASE     				=  param_name
																					return shortname

									return param_name







			@staticmethod
			def 							UpperCamelCase				(     UpperCAmelCase_:  Optional[int]				,     UpperCAmelCase_:  Optional[int]					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  TrialShortNamer.shortname_for_key(UpperCAmelCase_				,     UpperCAmelCase_					)
									_SCREAMING_SNAKE_CASE     				=  short_name
									_SCREAMING_SNAKE_CASE     				=  param_name







			@classmethod
			def 							UpperCamelCase				(     cls:  str					):







									'''simple docstring'''




									if cls.NAMING_INFO is not None:
															return

									_SCREAMING_SNAKE_CASE     				=  {
									    """short_word""": {},
									    """reverse_short_word""": {},
									    """short_param""": {},
									    """reverse_short_param""": {},
									}

									_SCREAMING_SNAKE_CASE     				=  list(cls.DEFAULTS.keys()					)

									for k in field_keys:
															cls.add_new_param_name(UpperCAmelCase_				,     UpperCAmelCase_					)

									_SCREAMING_SNAKE_CASE     				=  info







			@classmethod
			def 							UpperCamelCase				(     cls:  Any				,     UpperCAmelCase_:  Optional[Any]					):







									'''simple docstring'''




									cls.build_naming_info()
									assert cls.PREFIX is not None
									_SCREAMING_SNAKE_CASE     				=  [copy.copy(cls.PREFIX					)]

									for k, v in params.items():
															if k not in cls.DEFAULTS:
																					raise Exception(F'You should provide a default value for the param name {k} with value {v}'					)
															if v == cls.DEFAULTS[k]:
																					# The default value is not added to the name
																					continue

															_SCREAMING_SNAKE_CASE     				=  cls.NAMING_INFO["""short_param"""][k]

															if isinstance(UpperCAmelCase_				,     UpperCAmelCase_					):
																					_SCREAMING_SNAKE_CASE     				=  1 if v else 0

															_SCREAMING_SNAKE_CASE     				=  """""" if isinstance(UpperCAmelCase_				,     (int, float)					) else """-"""
															_SCREAMING_SNAKE_CASE     				=  F'{key}{sep}{v}'
															name.append(UpperCAmelCase_					)

									return "_".join(UpperCAmelCase_					)







			@classmethod
			def 							UpperCamelCase				(     cls:  int				,     UpperCAmelCase_:  int					):







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  repr[len(cls.PREFIX					) + 1 :]
									if repr == "":
															_SCREAMING_SNAKE_CASE     				=  []
									else:
															_SCREAMING_SNAKE_CASE     				=  repr.split("""_"""					)

									_SCREAMING_SNAKE_CASE     				=  {}

									for value in values:
															if "-" in value:
																					_SCREAMING_SNAKE_CASE				,					_SCREAMING_SNAKE_CASE     				=  value.split("""-"""					)
															else:
																					_SCREAMING_SNAKE_CASE     				=  re.sub("""[0-9.]"""				,     """"""				,     UpperCAmelCase_					)
																					_SCREAMING_SNAKE_CASE     				=  float(re.sub("""[^0-9.]"""				,     """"""				,     UpperCAmelCase_					)					)

															_SCREAMING_SNAKE_CASE     				=  cls.NAMING_INFO["""reverse_short_param"""][p_k]

															_SCREAMING_SNAKE_CASE     				=  p_v

									for k in cls.DEFAULTS:
															if k not in parameters:
																					_SCREAMING_SNAKE_CASE     				=  cls.DEFAULTS[k]

									return parameters

