from sklearn.metrics import fa_score, matthews_corrcoef

import datasets

from .record_evaluation import evaluate as evaluate_record


__lowercase										= '''\
@article{wang2019superglue,
  title={SuperGLUE: A Stickier Benchmark for General-Purpose Language Understanding Systems},
  author={Wang, Alex and Pruksachatkun, Yada and Nangia, Nikita and Singh, Amanpreet and Michael, Julian and Hill, Felix and Levy, Omer and Bowman, Samuel R},
  journal={arXiv preprint arXiv:1905.00537},
  year={2019}
}
'''

__lowercase										= '''\
SuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after
GLUE with a new set of more difficult language understanding tasks, improved
resources, and a new public leaderboard.
'''

__lowercase										= '''
Compute SuperGLUE evaluation metric associated to each SuperGLUE dataset.
Args:
    predictions: list of predictions to score. Depending on the SuperGlUE subset:
        - for \'record\': list of question-answer dictionaries with the following keys:
            - \'idx\': index of the question as specified by the dataset
            - \'prediction_text\': the predicted answer text
        - for \'multirc\': list of question-answer dictionaries with the following keys:
            - \'idx\': index of the question-answer pair as specified by the dataset
            - \'prediction\': the predicted answer label
        - otherwise: list of predicted labels
    references: list of reference labels. Depending on the SuperGLUE subset:
        - for \'record\': list of question-answers dictionaries with the following keys:
            - \'idx\': index of the question as specified by the dataset
            - \'answers\': list of possible answers
        - otherwise: list of reference labels
Returns: depending on the SuperGLUE subset:
    - for \'record\':
        - \'exact_match\': Exact match between answer and gold answer
        - \'f1\': F1 score
    - for \'multirc\':
        - \'exact_match\': Exact match between answer and gold answer
        - \'f1_m\': Per-question macro-F1 score
        - \'f1_a\': Average F1 score over all answers
    - for \'axb\':
        \'matthews_correlation\': Matthew Correlation
    - for \'cb\':
        - \'accuracy\': Accuracy
        - \'f1\': F1 score
    - for all others:
        - \'accuracy\': Accuracy
Examples:

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'copa\')  # any of ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'accuracy\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'cb\')
    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'accuracy\': 1.0, \'f1\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'record\')
    >>> predictions = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'prediction_text\': \'answer\'}]
    >>> references = [{\'idx\': {\'passage\': 0, \'query\': 0}, \'answers\': [\'answer\', \'another_answer\']}]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'exact_match\': 1.0, \'f1\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'multirc\')
    >>> predictions = [{\'idx\': {\'answer\': 0, \'paragraph\': 0, \'question\': 0}, \'prediction\': 0}, {\'idx\': {\'answer\': 1, \'paragraph\': 2, \'question\': 3}, \'prediction\': 1}]
    >>> references = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'exact_match\': 1.0, \'f1_m\': 1.0, \'f1_a\': 1.0}

    >>> super_glue_metric = datasets.load_metric(\'super_glue\', \'axb\')
    >>> references = [0, 1]
    >>> predictions = [0, 1]
    >>> results = super_glue_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {\'matthews_correlation\': 1.0}
'''



def lowerCamelCase  (       SCREAMING_SNAKE_CASE  ,      SCREAMING_SNAKE_CASE     ):
    '''simple docstring'''







    return float((preds == labels).mean()     )



def lowerCamelCase  (       SCREAMING_SNAKE_CASE  ,      SCREAMING_SNAKE_CASE  ,      SCREAMING_SNAKE_CASE="binary"     ):
    '''simple docstring'''







    __UpperCamelCase					:List[Any]        =		simple_accuracy(SCREAMING_SNAKE_CASE  ,      SCREAMING_SNAKE_CASE     )
    __UpperCamelCase					:Any        =		float(fa_score(y_true=SCREAMING_SNAKE_CASE  ,      y_pred=SCREAMING_SNAKE_CASE  ,      average=SCREAMING_SNAKE_CASE     )     )
    return {
        "accuracy": acc,
        "f1": fa,
    }



def lowerCamelCase  (       SCREAMING_SNAKE_CASE  ,      SCREAMING_SNAKE_CASE     ):
    '''simple docstring'''







    __UpperCamelCase					:Optional[Any]        =		{}
    for id_pred, label in zip(SCREAMING_SNAKE_CASE  ,      SCREAMING_SNAKE_CASE     ):
        __UpperCamelCase					:Optional[Any]        =		f"""{id_pred['idx']['paragraph']}-{id_pred['idx']['question']}"""
        __UpperCamelCase					:Dict        =		id_pred['''prediction''']
        if question_id in question_map:
            question_map[question_id].append((pred, label)     )
        else:
            __UpperCamelCase					:int        =		[(pred, label)]
    __UpperCamelCase  ,	__UpperCamelCase					:int        =		[], []
    for question, preds_labels in question_map.items():
        __UpperCamelCase  ,	__UpperCamelCase					:List[Any]        =		zip(*SCREAMING_SNAKE_CASE     )
        __UpperCamelCase					:List[str]        =		fa_score(y_true=SCREAMING_SNAKE_CASE  ,      y_pred=SCREAMING_SNAKE_CASE  ,      average='''macro'''     )
        fas.append(SCREAMING_SNAKE_CASE     )
        __UpperCamelCase					:List[str]        =		int(sum(pred == label for pred, label in preds_labels     ) == len(SCREAMING_SNAKE_CASE     )     )
        ems.append(SCREAMING_SNAKE_CASE     )
    __UpperCamelCase					:List[str]        =		float(sum(SCREAMING_SNAKE_CASE     ) / len(SCREAMING_SNAKE_CASE     )     )
    __UpperCamelCase					:List[Any]        =		sum(SCREAMING_SNAKE_CASE     ) / len(SCREAMING_SNAKE_CASE     )
    __UpperCamelCase					:Optional[int]        =		float(fa_score(y_true=SCREAMING_SNAKE_CASE  ,      y_pred=[id_pred['''prediction'''] for id_pred in ids_preds]     )     )
    return {"exact_match": em, "f1_m": fa_m, "f1_a": fa_a}


@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION						,     _KWARGS_DESCRIPTION			)
class      lowerCamelCase_					(       datasets.Metric			):






    '''simple docstring'''






    def       UpperCamelCase__       (     self)	->     str:
        if self.config_name not in [
            "boolq",
            "cb",
            "copa",
            "multirc",
            "record",
            "rte",
            "wic",
            "wsc",
            "wsc.fixed",
            "axb",
            "axg",
        ]:
            raise KeyError(
                '''You should supply a configuration name selected in '''
                '''["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]''')
        return datasets.MetricInfo(
            description=_DESCRIPTION			,			citation=_CITATION			,			inputs_description=_KWARGS_DESCRIPTION			,			features=datasets.Features(self._get_feature_types())			,			codebase_urls=[]			,			reference_urls=[]			,			format='''numpy''' if not self.config_name == '''record''' and not self.config_name == '''multirc''' else None			,			)







    def       UpperCamelCase__       (     self)	->     str:
        if self.config_name == "record":
            return {
                "predictions": {
                    "idx": {
                        "passage": datasets.Value('''int64'''),
                        "query": datasets.Value('''int64'''),
                    },
                    "prediction_text": datasets.Value('''string'''),
                },
                "references": {
                    "idx": {
                        "passage": datasets.Value('''int64'''),
                        "query": datasets.Value('''int64'''),
                    },
                    "answers": datasets.Sequence(datasets.Value('''string''')),
                },
            }
        elif self.config_name == "multirc":
            return {
                "predictions": {
                    "idx": {
                        "answer": datasets.Value('''int64'''),
                        "paragraph": datasets.Value('''int64'''),
                        "question": datasets.Value('''int64'''),
                    },
                    "prediction": datasets.Value('''int64'''),
                },
                "references": datasets.Value('''int64'''),
            }
        else:
            return {
                "predictions": datasets.Value('''int64'''),
                "references": datasets.Value('''int64'''),
            }







    def       UpperCamelCase__       (     self			,			__lowercase			,			__lowercase)	->     int:
        if self.config_name == "axb":
            return {"matthews_correlation": matthews_corrcoef(__lowercase			,			__lowercase)}
        elif self.config_name == "cb":
            return acc_and_fa(__lowercase			,			__lowercase			,			fa_avg='''macro''')
        elif self.config_name == "record":
            __UpperCamelCase					:Optional[Any]        =		[
                {
                    '''qas''': [
                        {'''id''': ref['''idx''']['''query'''], '''answers''': [{'''text''': ans} for ans in ref['''answers''']]}
                        for ref in references
                    ]
                }
            ]
            __UpperCamelCase					:Optional[int]        =		{pred['''idx''']['''query''']: pred['''prediction_text'''] for pred in predictions}
            return evaluate_record(__lowercase			,			__lowercase)[0]
        elif self.config_name == "multirc":
            return evaluate_multirc(__lowercase			,			__lowercase)
        elif self.config_name in ["copa", "rte", "wic", "wsc", "wsc.fixed", "boolq", "axg"]:
            return {"accuracy": simple_accuracy(__lowercase			,			__lowercase)}
        else:
            raise KeyError(
                '''You should supply a configuration name selected in '''
                '''["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]''')






import argparse
import glob
import logging
import os
from argparse import Namespace
from importlib import import_module

import numpy as np
import torch
from lightning_base import BaseTransformer, add_generic_args, generic_train
from seqeval.metrics import accuracy_score, fa_score, precision_score, recall_score
from torch.nn import CrossEntropyLoss
from torch.utils.data import DataLoader, TensorDataset
from utils_ner import TokenClassificationTask


__lowercase										= logging.getLogger(__name__)


class      lowerCamelCase_					(       UpperCAmelCase_			):






    '''simple docstring'''






    a__     :						str      =				"""token-classification"""







    def __init__(     self			,			__lowercase)	->     str:
        if type(__lowercase) == dict:
            __UpperCamelCase					:List[Any]        =		Namespace(**__lowercase)
        __UpperCamelCase					:Dict        =		import_module('''tasks''')
        try:
            __UpperCamelCase					:str        =		getattr(__lowercase			,			hparams.task_type)
            __UpperCamelCase					:TokenClassificationTask        =		token_classification_task_clazz()
        except AttributeError:
            raise ValueError(
                f"""Task {hparams.task_type} needs to be defined as a TokenClassificationTask subclass in {module}. """
                f"""Available tasks classes are: {TokenClassificationTask.__subclasses__()}""")
        __UpperCamelCase					:Tuple        =		self.token_classification_task.get_labels(hparams.labels)
        __UpperCamelCase					:Tuple        =		CrossEntropyLoss().ignore_index
        super().__init__(__lowercase			,			len(self.labels)			,			self.mode)







    def       UpperCamelCase__       (     self			,			**__lowercase)	->     List[Any]:
        return self.model(**__lowercase)







    def       UpperCamelCase__       (     self			,			__lowercase			,			__lowercase)	->     Any:
        __UpperCamelCase					:str        =		{'''input_ids''': batch[0], '''attention_mask''': batch[1], '''labels''': batch[3]}
        if self.config.model_type != "distilbert":
            __UpperCamelCase					:Union[str, Any]        =		(
                batch[2] if self.config.model_type in ['''bert''', '''xlnet'''] else None
            )  # XLM and RoBERTa don"t use token_type_ids

        __UpperCamelCase					:Dict        =		self(**__lowercase)
        __UpperCamelCase					:str        =		outputs[0]
        # tensorboard_logs = {"loss": loss, "rate": self.lr_scheduler.get_last_lr()[-1]}
        return {"loss": loss}







    def       UpperCamelCase__       (     self)	->     List[Any]:
        __UpperCamelCase					:List[Any]        =		self.hparams
        for mode in ["train", "dev", "test"]:
            __UpperCamelCase					:int        =		self._feature_file(__lowercase)
            if os.path.exists(__lowercase) and not args.overwrite_cache:
                logger.info('''Loading features from cached file %s'''			,			__lowercase)
                __UpperCamelCase					:Any        =		torch.load(__lowercase)
            else:
                logger.info('''Creating features from dataset file at %s'''			,			args.data_dir)
                __UpperCamelCase					:Any        =		self.token_classification_task.read_examples_from_file(args.data_dir			,			__lowercase)
                __UpperCamelCase					:Union[str, Any]        =		self.token_classification_task.convert_examples_to_features(
                    __lowercase			,			self.labels			,			args.max_seq_length			,			self.tokenizer			,			cls_token_at_end=bool(self.config.model_type in ['''xlnet'''])			,			cls_token=self.tokenizer.cls_token			,			cls_token_segment_id=2 if self.config.model_type in ['''xlnet'''] else 0			,			sep_token=self.tokenizer.sep_token			,			sep_token_extra=__lowercase			,			pad_on_left=bool(self.config.model_type in ['''xlnet'''])			,			pad_token=self.tokenizer.pad_token_id			,			pad_token_segment_id=self.tokenizer.pad_token_type_id			,			pad_token_label_id=self.pad_token_label_id			,			)
                logger.info('''Saving features into cached file %s'''			,			__lowercase)
                torch.save(__lowercase			,			__lowercase)







    def       UpperCamelCase__       (     self			,			__lowercase			,			__lowercase			,			__lowercase = False)	->     DataLoader:
        __UpperCamelCase					:Tuple        =		self._feature_file(__lowercase)
        logger.info('''Loading features from cached file %s'''			,			__lowercase)
        __UpperCamelCase					:str        =		torch.load(__lowercase)
        __UpperCamelCase					:int        =		torch.tensor([f.input_ids for f in features]			,			dtype=torch.long)
        __UpperCamelCase					:Optional[Any]        =		torch.tensor([f.attention_mask for f in features]			,			dtype=torch.long)
        if features[0].token_type_ids is not None:
            __UpperCamelCase					:str        =		torch.tensor([f.token_type_ids for f in features]			,			dtype=torch.long)
        else:
            __UpperCamelCase					:Union[str, Any]        =		torch.tensor([0 for f in features]			,			dtype=torch.long)
            # HACK(we will not use this anymore soon)
        __UpperCamelCase					:int        =		torch.tensor([f.label_ids for f in features]			,			dtype=torch.long)
        return DataLoader(
            TensorDataset(__lowercase			,			__lowercase			,			__lowercase			,			__lowercase)			,			batch_size=__lowercase)







    def       UpperCamelCase__       (     self			,			__lowercase			,			__lowercase)	->     Dict:
        """Compute validation""" ""
        __UpperCamelCase					:int        =		{'''input_ids''': batch[0], '''attention_mask''': batch[1], '''labels''': batch[3]}
        if self.config.model_type != "distilbert":
            __UpperCamelCase					:Any        =		(
                batch[2] if self.config.model_type in ['''bert''', '''xlnet'''] else None
            )  # XLM and RoBERTa don"t use token_type_ids
        __UpperCamelCase					:Any        =		self(**__lowercase)
        __UpperCamelCase  ,	__UpperCamelCase					:Tuple        =		outputs[:2]
        __UpperCamelCase					:List[str]        =		logits.detach().cpu().numpy()
        __UpperCamelCase					:List[str]        =		inputs['''labels'''].detach().cpu().numpy()
        return {"val_loss": tmp_eval_loss.detach().cpu(), "pred": preds, "target": out_label_ids}







    def       UpperCamelCase__       (     self			,			__lowercase)	->     List[str]:
        __UpperCamelCase					:Tuple        =		torch.stack([x['''val_loss'''] for x in outputs]).mean()
        __UpperCamelCase					:str        =		np.concatenate([x['''pred'''] for x in outputs]			,			axis=0)
        __UpperCamelCase					:Any        =		np.argmax(__lowercase			,			axis=2)
        __UpperCamelCase					:str        =		np.concatenate([x['''target'''] for x in outputs]			,			axis=0)

        __UpperCamelCase					:List[str]        =		dict(enumerate(self.labels))
        __UpperCamelCase					:Tuple        =		[[] for _ in range(out_label_ids.shape[0])]
        __UpperCamelCase					:Any        =		[[] for _ in range(out_label_ids.shape[0])]

        for i in range(out_label_ids.shape[0]):
            for j in range(out_label_ids.shape[1]):
                if out_label_ids[i, j] != self.pad_token_label_id:
                    out_label_list[i].append(label_map[out_label_ids[i][j]])
                    preds_list[i].append(label_map[preds[i][j]])

        __UpperCamelCase					:Any        =		{
            '''val_loss''': val_loss_mean,
            '''accuracy_score''': accuracy_score(__lowercase			,			__lowercase),
            '''precision''': precision_score(__lowercase			,			__lowercase),
            '''recall''': recall_score(__lowercase			,			__lowercase),
            '''f1''': fa_score(__lowercase			,			__lowercase),
        }

        __UpperCamelCase					:Dict        =		dict(results.items())
        __UpperCamelCase					:List[str]        =		results
        return ret, preds_list, out_label_list







    def       UpperCamelCase__       (     self			,			__lowercase)	->     int:
        # when stable
        __UpperCamelCase  ,	__UpperCamelCase  ,	__UpperCamelCase					:List[Any]        =		self._eval_end(__lowercase)
        __UpperCamelCase					:Tuple        =		ret['''log''']
        return {"val_loss": logs["val_loss"], "log": logs, "progress_bar": logs}







    def       UpperCamelCase__       (     self			,			__lowercase)	->     int:
        # updating to test_epoch_end instead of deprecated test_end
        __UpperCamelCase  ,	__UpperCamelCase  ,	__UpperCamelCase					:Optional[int]        =		self._eval_end(__lowercase)

        # Converting to the dict required by pl
        # https://github.com/PyTorchLightning/pytorch-lightning/blob/master/\
        # pytorch_lightning/trainer/logging.py#L139
        __UpperCamelCase					:Optional[Any]        =		ret['''log''']
        # `val_loss` is the key returned by `self._eval_end()` but actually refers to `test_loss`
        return {"avg_test_loss": logs["val_loss"], "log": logs, "progress_bar": logs}







    @staticmethod
    def       UpperCamelCase__       (     __lowercase			,			__lowercase)	->     Union[str, Any]:
        # Add NER specific options
        BaseTransformer.add_model_specific_args(__lowercase			,			__lowercase)
        parser.add_argument(
            '''--task_type'''			,			default='''NER'''			,			type=__lowercase			,			help='''Task type to fine tune in training (e.g. NER, POS, etc)''')
        parser.add_argument(
            '''--max_seq_length'''			,			default=128			,			type=__lowercase			,			help=(
                '''The maximum total input sequence length after tokenization. Sequences longer '''
                '''than this will be truncated, sequences shorter will be padded.'''
            )			,			)

        parser.add_argument(
            '''--labels'''			,			default=''''''			,			type=__lowercase			,			help='''Path to a file containing all labels. If not specified, CoNLL-2003 labels are used.'''			,			)
        parser.add_argument(
            '''--gpus'''			,			default=0			,			type=__lowercase			,			help='''The number of GPUs allocated for this, it is by default 0 meaning none'''			,			)

        parser.add_argument(
            '''--overwrite_cache'''			,			action='''store_true'''			,			help='''Overwrite the cached training and evaluation sets''')

        return parser


if __name__ == "__main__":
    __lowercase										= argparse.ArgumentParser()
    add_generic_args(parser, os.getcwd())
    __lowercase										= NERTransformer.add_model_specific_args(parser, os.getcwd())
    __lowercase										= parser.parse_args()
    __lowercase										= NERTransformer(args)
    __lowercase										= generic_train(model, args)

    if args.do_predict:
        # See https://github.com/huggingface/transformers/issues/3159
        # pl use this default format to create a checkpoint:
        # https://github.com/PyTorchLightning/pytorch-lightning/blob/master\
        # /pytorch_lightning/callbacks/model_checkpoint.py#L322
        __lowercase										= sorted(glob.glob(os.path.join(args.output_dir, '''checkpoint-epoch=*.ckpt'''), recursive=True))
        __lowercase										= model.load_from_checkpoint(checkpoints[-1])
        trainer.test(model)





