





from transformers import BertTokenizer, EncoderDecoderModel, SeqaSeqTrainer, SeqaSeqTrainingArguments
from transformers.testing_utils import TestCasePlus, require_torch, slow
from transformers.utils import is_datasets_available


if is_datasets_available():
				import datasets

class     __SCREAMING_SNAKE_CASE   (       _a    ):






							@slow
							@require_torch
							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     EncoderDecoderModel.from_encoder_decoder_pretrained("""prajjwal1/bert-tiny"""					,	"""prajjwal1/bert-tiny"""					)
														UpperCamelCase__			       =     BertTokenizer.from_pretrained("""bert-base-uncased"""					)

														UpperCamelCase__			       =     bertabert.config.encoder.vocab_size
														UpperCamelCase__			       =     tokenizer.sep_token_id
														UpperCamelCase__			       =     tokenizer.cls_token_id
														UpperCamelCase__			       =     128

														UpperCamelCase__			       =     datasets.load_dataset("""cnn_dailymail"""					,	"""3.0.0"""					,	split="""train[:1%]"""					)
														UpperCamelCase__			       =     datasets.load_dataset("""cnn_dailymail"""					,	"""3.0.0"""					,	split="""validation[:1%]"""					)

														UpperCamelCase__			       =     train_dataset.select(range(32					)					)
														UpperCamelCase__			       =     val_dataset.select(range(16					)					)

														UpperCamelCase__			       =     4

														def _map_to_encoder_decoder_inputs(__lowerCAmelCase					):
																					# Tokenizer will automatically set [BOS] <text> [EOS]
																					UpperCamelCase__			       =     tokenizer(batch["""article"""]					,	padding="""max_length"""					,	truncation=__lowerCAmelCase					,	max_length=512					)
																					UpperCamelCase__			       =     tokenizer(batch["""highlights"""]					,	padding="""max_length"""					,	truncation=__lowerCAmelCase					,	max_length=128					)
																					UpperCamelCase__			       =     inputs.input_ids
																					UpperCamelCase__			       =     inputs.attention_mask

																					UpperCamelCase__			       =     outputs.input_ids
																					UpperCamelCase__			       =     outputs.input_ids.copy()
																					UpperCamelCase__			       =     [
																					    [-100 if token == tokenizer.pad_token_id else token for token in labels] for labels in batch["""labels"""]
																					]
																					UpperCamelCase__			       =     outputs.attention_mask

																					assert all(len(__lowerCAmelCase					) == 512 for x in inputs.input_ids					)
																					assert all(len(__lowerCAmelCase					) == 128 for x in outputs.input_ids					)

																					return batch

														def _compute_metrics(__lowerCAmelCase					):
																					UpperCamelCase__			       =     pred.label_ids
																					UpperCamelCase__			       =     pred.predictions

																					# all unnecessary tokens are removed
																					UpperCamelCase__			       =     tokenizer.batch_decode(__lowerCAmelCase					,	skip_special_tokens=__lowerCAmelCase					)
																					UpperCamelCase__			       =     tokenizer.batch_decode(__lowerCAmelCase					,	skip_special_tokens=__lowerCAmelCase					)

																					UpperCamelCase__			       =     sum([int(pred_str[i] == label_str[i]					) for i in range(len(__lowerCAmelCase					)					)]					) / len(__lowerCAmelCase					)

																					return {"accuracy": accuracy}

														# map train dataset
														UpperCamelCase__			       =     train_dataset.map(
														    _map_to_encoder_decoder_inputs					,	batched=__lowerCAmelCase					,	batch_size=__lowerCAmelCase					,	remove_columns=["""article""", """highlights"""]					,	)
														train_dataset.set_format(
														    type="""torch"""					,	columns=["""input_ids""", """attention_mask""", """decoder_input_ids""", """decoder_attention_mask""", """labels"""]					,	)

														# same for validation dataset
														UpperCamelCase__			       =     val_dataset.map(
														    _map_to_encoder_decoder_inputs					,	batched=__lowerCAmelCase					,	batch_size=__lowerCAmelCase					,	remove_columns=["""article""", """highlights"""]					,	)
														val_dataset.set_format(
														    type="""torch"""					,	columns=["""input_ids""", """attention_mask""", """decoder_input_ids""", """decoder_attention_mask""", """labels"""]					,	)

														UpperCamelCase__			       =     self.get_auto_remove_tmp_dir()

														UpperCamelCase__			       =     SeqaSeqTrainingArguments(
														    output_dir=__lowerCAmelCase					,	per_device_train_batch_size=__lowerCAmelCase					,	per_device_eval_batch_size=__lowerCAmelCase					,	predict_with_generate=__lowerCAmelCase					,	evaluation_strategy="""steps"""					,	do_train=__lowerCAmelCase					,	do_eval=__lowerCAmelCase					,	warmup_steps=0					,	eval_steps=2					,	logging_steps=2					,	)

														# instantiate trainer
														UpperCamelCase__			       =     SeqaSeqTrainer(
														    model=__lowerCAmelCase					,	args=__lowerCAmelCase					,	compute_metrics=_compute_metrics					,	train_dataset=__lowerCAmelCase					,	eval_dataset=__lowerCAmelCase					,	tokenizer=__lowerCAmelCase					,	)

														# start training
														trainer.train()











import argparse
import glob
import logging
import os
from argparse import Namespace
from importlib import import_module

import numpy as np
import torch
from lightning_base import BaseTransformer, add_generic_args, generic_train
from seqeval.metrics import accuracy_score, fa_score, precision_score, recall_score
from torch.nn import CrossEntropyLoss
from torch.utils.data import DataLoader, TensorDataset
from utils_ner import TokenClassificationTask


UpperCamelCase__     =	logging.getLogger(__name__)

class     __SCREAMING_SNAKE_CASE   (       _a    ):
							snake_case       : str         =    """token-classification"""






							def __init__(  self					,	__lowerCAmelCase					):
														if type(__lowerCAmelCase					) == dict:
																					UpperCamelCase__			       =     Namespace(**__lowerCAmelCase					)
														UpperCamelCase__			       =     import_module("""tasks"""					)
														try:
																					UpperCamelCase__			       =     getattr(__lowerCAmelCase					,	hparams.task_type					)
																					UpperCamelCase__			       =     token_classification_task_clazz()
														except AttributeError:
																					raise ValueError(
																					    f"""Task {hparams.task_type} needs to be defined as a TokenClassificationTask subclass in {module}. """
																					    f"""Available tasks classes are: {TokenClassificationTask.__subclasses__()}"""					)
														UpperCamelCase__			       =     self.token_classification_task.get_labels(hparams.labels					)
														UpperCamelCase__			       =     CrossEntropyLoss().ignore_index
														super().__init__(__lowerCAmelCase					,	len(self.labels					)					,	self.mode					)






							def 			_lowerCamelCase						(  self					,	**__lowerCAmelCase					):
														return self.model(**__lowerCAmelCase					)






							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					,	__lowerCAmelCase					):
														UpperCamelCase__			       =     {"""input_ids""": batch[0], """attention_mask""": batch[1], """labels""": batch[3]}
														if self.config.model_type != "distilbert":
																					UpperCamelCase__			       =     (
																					    batch[2] if self.config.model_type in ["""bert""", """xlnet"""] else None
																					)  # XLM and RoBERTa don"t use token_type_ids

														UpperCamelCase__			       =     self(**__lowerCAmelCase					)
														UpperCamelCase__			       =     outputs[0]
														# tensorboard_logs = {"loss": loss, "rate": self.lr_scheduler.get_last_lr()[-1]}
														return {"loss": loss}






							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     self.hparams
														for mode in ["train", "dev", "test"]:
																					UpperCamelCase__			       =     self._feature_file(__lowerCAmelCase					)
																					if os.path.exists(__lowerCAmelCase					) and not args.overwrite_cache:
																												logger.info("""Loading features from cached file %s"""					,	__lowerCAmelCase					)
																												UpperCamelCase__			       =     torch.load(__lowerCAmelCase					)
																					else:
																												logger.info("""Creating features from dataset file at %s"""					,	args.data_dir					)
																												UpperCamelCase__			       =     self.token_classification_task.read_examples_from_file(args.data_dir					,	__lowerCAmelCase					)
																												UpperCamelCase__			       =     self.token_classification_task.convert_examples_to_features(
																												    __lowerCAmelCase					,	self.labels					,	args.max_seq_length					,	self.tokenizer					,	cls_token_at_end=bool(self.config.model_type in ["""xlnet"""]					)					,	cls_token=self.tokenizer.cls_token					,	cls_token_segment_id=2 if self.config.model_type in ["""xlnet"""] else 0					,	sep_token=self.tokenizer.sep_token					,	sep_token_extra=__lowerCAmelCase					,	pad_on_left=bool(self.config.model_type in ["""xlnet"""]					)					,	pad_token=self.tokenizer.pad_token_id					,	pad_token_segment_id=self.tokenizer.pad_token_type_id					,	pad_token_label_id=self.pad_token_label_id					,	)
																												logger.info("""Saving features into cached file %s"""					,	__lowerCAmelCase					)
																												torch.save(__lowerCAmelCase					,	__lowerCAmelCase					)






							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					,	__lowerCAmelCase					,	__lowerCAmelCase = False					):
														UpperCamelCase__			       =     self._feature_file(__lowerCAmelCase					)
														logger.info("""Loading features from cached file %s"""					,	__lowerCAmelCase					)
														UpperCamelCase__			       =     torch.load(__lowerCAmelCase					)
														UpperCamelCase__			       =     torch.tensor([f.input_ids for f in features]					,	dtype=torch.long					)
														UpperCamelCase__			       =     torch.tensor([f.attention_mask for f in features]					,	dtype=torch.long					)
														if features[0].token_type_ids is not None:
																					UpperCamelCase__			       =     torch.tensor([f.token_type_ids for f in features]					,	dtype=torch.long					)
														else:
																					UpperCamelCase__			       =     torch.tensor([0 for f in features]					,	dtype=torch.long					)
																					# HACK(we will not use this anymore soon)
														UpperCamelCase__			       =     torch.tensor([f.label_ids for f in features]					,	dtype=torch.long					)
														return DataLoader(
														    TensorDataset(__lowerCAmelCase					,	__lowerCAmelCase					,	__lowerCAmelCase					,	__lowerCAmelCase					)					,	batch_size=__lowerCAmelCase					)






							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					,	__lowerCAmelCase					):
														"""Compute validation""" ""
														UpperCamelCase__			       =     {"""input_ids""": batch[0], """attention_mask""": batch[1], """labels""": batch[3]}
														if self.config.model_type != "distilbert":
																					UpperCamelCase__			       =     (
																					    batch[2] if self.config.model_type in ["""bert""", """xlnet"""] else None
																					)  # XLM and RoBERTa don"t use token_type_ids
														UpperCamelCase__			       =     self(**__lowerCAmelCase					)
														UpperCamelCase__       ,       UpperCamelCase__			       =     outputs[:2]
														UpperCamelCase__			       =     logits.detach().cpu().numpy()
														UpperCamelCase__			       =     inputs["""labels"""].detach().cpu().numpy()
														return {"val_loss": tmp_eval_loss.detach().cpu(), "pred": preds, "target": out_label_ids}






							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					):
														UpperCamelCase__			       =     torch.stack([x["""val_loss"""] for x in outputs]					).mean()
														UpperCamelCase__			       =     np.concatenate([x["""pred"""] for x in outputs]					,	axis=0					)
														UpperCamelCase__			       =     np.argmax(__lowerCAmelCase					,	axis=2					)
														UpperCamelCase__			       =     np.concatenate([x["""target"""] for x in outputs]					,	axis=0					)

														UpperCamelCase__			       =     dict(enumerate(self.labels					)					)
														UpperCamelCase__			       =     [[] for _ in range(out_label_ids.shape[0]					)]
														UpperCamelCase__			       =     [[] for _ in range(out_label_ids.shape[0]					)]

														for i in range(out_label_ids.shape[0]					):
																					for j in range(out_label_ids.shape[1]					):
																												if out_label_ids[i, j] != self.pad_token_label_id:
																																			out_label_list[i].append(label_map[out_label_ids[i][j]]					)
																																			preds_list[i].append(label_map[preds[i][j]]					)

														UpperCamelCase__			       =     {
														    """val_loss""": val_loss_mean,
														    """accuracy_score""": accuracy_score(__lowerCAmelCase					,	__lowerCAmelCase					),
														    """precision""": precision_score(__lowerCAmelCase					,	__lowerCAmelCase					),
														    """recall""": recall_score(__lowerCAmelCase					,	__lowerCAmelCase					),
														    """f1""": fa_score(__lowerCAmelCase					,	__lowerCAmelCase					),
														}

														UpperCamelCase__			       =     dict(results.items()					)
														UpperCamelCase__			       =     results
														return ret, preds_list, out_label_list






							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					):
														# when stable
														UpperCamelCase__       ,       UpperCamelCase__       ,       UpperCamelCase__			       =     self._eval_end(__lowerCAmelCase					)
														UpperCamelCase__			       =     ret["""log"""]
														return {"val_loss": logs["val_loss"], "log": logs, "progress_bar": logs}






							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					):
														# updating to test_epoch_end instead of deprecated test_end
														UpperCamelCase__       ,       UpperCamelCase__       ,       UpperCamelCase__			       =     self._eval_end(__lowerCAmelCase					)

														# Converting to the dict required by pl
														# https://github.com/PyTorchLightning/pytorch-lightning/blob/master/\
														# pytorch_lightning/trainer/logging.py#L139
														UpperCamelCase__			       =     ret["""log"""]
														# `val_loss` is the key returned by `self._eval_end()` but actually refers to `test_loss`
														return {"avg_test_loss": logs["val_loss"], "log": logs, "progress_bar": logs}







							@staticmethod
							def 			_lowerCamelCase						(  __lowerCAmelCase					,	__lowerCAmelCase					):
														# Add NER specific options
														BaseTransformer.add_model_specific_args(__lowerCAmelCase					,	__lowerCAmelCase					)
														parser.add_argument(
														    """--task_type"""					,	default="""NER"""					,	type=__lowerCAmelCase					,	help="""Task type to fine tune in training (e.g. NER, POS, etc)"""					)
														parser.add_argument(
														    """--max_seq_length"""					,	default=128					,	type=__lowerCAmelCase					,	help=(
														        """The maximum total input sequence length after tokenization. Sequences longer """
														        """than this will be truncated, sequences shorter will be padded."""
														    )					,	)

														parser.add_argument(
														    """--labels"""					,	default=""""""					,	type=__lowerCAmelCase					,	help="""Path to a file containing all labels. If not specified, CoNLL-2003 labels are used."""					,	)
														parser.add_argument(
														    """--gpus"""					,	default=0					,	type=__lowerCAmelCase					,	help="""The number of GPUs allocated for this, it is by default 0 meaning none"""					,	)

														parser.add_argument(
														    """--overwrite_cache"""					,	action="""store_true"""					,	help="""Overwrite the cached training and evaluation sets"""					)

														return parser


if __name__ == "__main__":
					UpperCamelCase__     =	argparse.ArgumentParser()
					add_generic_args(parser, os.getcwd())
					UpperCamelCase__     =	NERTransformer.add_model_specific_args(parser, os.getcwd())
					UpperCamelCase__     =	parser.parse_args()
					UpperCamelCase__     =	NERTransformer(args)
					UpperCamelCase__     =	generic_train(model, args)

					if args.do_predict:
										# See https://github.com/huggingface/transformers/issues/3159
										# pl use this default format to create a checkpoint:
										# https://github.com/PyTorchLightning/pytorch-lightning/blob/master\
										# /pytorch_lightning/callbacks/model_checkpoint.py#L322
										UpperCamelCase__     =	sorted(glob.glob(os.path.join(args.output_dir, "checkpoint-epoch=*.ckpt"), recursive=True))
										UpperCamelCase__     =	model.load_from_checkpoint(checkpoints[-1])
										trainer.test(model)




