





import math
import time
from typing import Dict, List, Optional

from torch.utils.data import Dataset

from transformers import SeqaSeqTrainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput, speed_metrics


if is_torch_tpu_available(check_device=False):
					import torch_xla.core.xla_model as xm
					import torch_xla.debug.metrics as met

class     __SCREAMING_SNAKE_CASE   (       _a    ):






							def __init__(  self					,	*__lowerCAmelCase					,	__lowerCAmelCase=None					,	__lowerCAmelCase=None					,	**__lowerCAmelCase					):
														super().__init__(*__lowerCAmelCase					,	**__lowerCAmelCase					)
														UpperCamelCase__			       =     eval_examples
														UpperCamelCase__			       =     post_process_function






							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase = None					,	__lowerCAmelCase=None					,	__lowerCAmelCase = None					,	__lowerCAmelCase = "eval"					,	**__lowerCAmelCase					,	):
														UpperCamelCase__			       =     gen_kwargs.copy()
														UpperCamelCase__			       =     (
														    gen_kwargs["""max_length"""] if gen_kwargs.get("""max_length"""					) is not None else self.args.generation_max_length
														)
														UpperCamelCase__			       =     (
														    gen_kwargs["""num_beams"""] if gen_kwargs.get("""num_beams"""					) is not None else self.args.generation_num_beams
														)
														UpperCamelCase__			       =     gen_kwargs

														UpperCamelCase__			       =     self.eval_dataset if eval_dataset is None else eval_dataset
														UpperCamelCase__			       =     self.get_eval_dataloader(__lowerCAmelCase					)
														UpperCamelCase__			       =     self.eval_examples if eval_examples is None else eval_examples

														# Temporarily disable metric computation, we will do it in the loop here.
														UpperCamelCase__			       =     self.compute_metrics
														UpperCamelCase__			       =     None
														UpperCamelCase__			       =     time.time()
														UpperCamelCase__			       =     self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
														try:
																					UpperCamelCase__			       =     eval_loop(
																					    __lowerCAmelCase					,	description="""Evaluation"""					,	prediction_loss_only=True if compute_metrics is None else None					,	ignore_keys=__lowerCAmelCase					,	metric_key_prefix=__lowerCAmelCase					,	)
														finally:
																					UpperCamelCase__			       =     compute_metrics
														UpperCamelCase__			       =     self.args.eval_batch_size * self.args.world_size
														if f"""{metric_key_prefix}_jit_compilation_time""" in output.metrics:
																					start_time += output.metrics[f"""{metric_key_prefix}_jit_compilation_time"""]
														output.metrics.update(
														    speed_metrics(
														        __lowerCAmelCase					,	__lowerCAmelCase					,	num_samples=output.num_samples					,	num_steps=math.ceil(output.num_samples / total_batch_size					)					,	)					)

														if self.post_process_function is not None and self.compute_metrics is not None and self.args.should_save:
																					# Only the main node write the results by default
																					UpperCamelCase__			       =     self.post_process_function(__lowerCAmelCase					,	__lowerCAmelCase					,	__lowerCAmelCase					)
																					UpperCamelCase__			       =     self.compute_metrics(__lowerCAmelCase					)

																					# Prefix all keys with metric_key_prefix + '_'
																					for key in list(metrics.keys()					):
																												if not key.startswith(f"""{metric_key_prefix}_"""					):
																																			UpperCamelCase__			       =     metrics.pop(__lowerCAmelCase					)

																					metrics.update(output.metrics					)
														else:
																					UpperCamelCase__			       =     output.metrics

														if self.args.should_log:
																					# Only the main node log the results by default
																					self.log(__lowerCAmelCase					)

														if self.args.tpu_metrics_debug or self.args.debug:
																					# tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
																					xm.master_print(met.metrics_report()					)

														UpperCamelCase__			       =     self.callback_handler.on_evaluate(self.args					,	self.state					,	self.control					,	__lowerCAmelCase					)
														return metrics







							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					,	__lowerCAmelCase					,	__lowerCAmelCase=None					,	__lowerCAmelCase = "test"					,	**__lowerCAmelCase					):
														UpperCamelCase__			       =     gen_kwargs.copy()

														UpperCamelCase__			       =     self.get_test_dataloader(__lowerCAmelCase					)

														# Temporarily disable metric computation, we will do it in the loop here.
														UpperCamelCase__			       =     self.compute_metrics
														UpperCamelCase__			       =     None
														UpperCamelCase__			       =     time.time()
														UpperCamelCase__			       =     self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
														try:
																					UpperCamelCase__			       =     eval_loop(
																					    __lowerCAmelCase					,	description="""Prediction"""					,	prediction_loss_only=True if compute_metrics is None else None					,	ignore_keys=__lowerCAmelCase					,	metric_key_prefix=__lowerCAmelCase					,	)
														finally:
																					UpperCamelCase__			       =     compute_metrics

														UpperCamelCase__			       =     self.args.eval_batch_size * self.args.world_size
														if f"""{metric_key_prefix}_jit_compilation_time""" in output.metrics:
																					start_time += output.metrics[f"""{metric_key_prefix}_jit_compilation_time"""]
														output.metrics.update(
														    speed_metrics(
														        __lowerCAmelCase					,	__lowerCAmelCase					,	num_samples=output.num_samples					,	num_steps=math.ceil(output.num_samples / total_batch_size					)					,	)					)
														if self.post_process_function is None or self.compute_metrics is None:
																					return output

														UpperCamelCase__			       =     self.post_process_function(__lowerCAmelCase					,	__lowerCAmelCase					,	__lowerCAmelCase					,	"""predict"""					)
														UpperCamelCase__			       =     self.compute_metrics(__lowerCAmelCase					)

														# Prefix all keys with metric_key_prefix + '_'
														for key in list(metrics.keys()					):
																					if not key.startswith(f"""{metric_key_prefix}_"""					):
																												UpperCamelCase__			       =     metrics.pop(__lowerCAmelCase					)
														metrics.update(output.metrics					)
														return PredictionOutput(predictions=predictions.predictions					,	label_ids=predictions.label_ids					,	metrics=__lowerCAmelCase					)











import logging

from transformers.configuration_utils import PretrainedConfig


UpperCamelCase__     =	logging.getLogger(__name__)

class     __SCREAMING_SNAKE_CASE   (       _a    ):
							snake_case       : Optional[Any]         =    """masked_bert"""






							def __init__(  self					,	__lowerCAmelCase=30522					,	__lowerCAmelCase=768					,	__lowerCAmelCase=12					,	__lowerCAmelCase=12					,	__lowerCAmelCase=3072					,	__lowerCAmelCase="gelu"					,	__lowerCAmelCase=0.1					,	__lowerCAmelCase=0.1					,	__lowerCAmelCase=512					,	__lowerCAmelCase=2					,	__lowerCAmelCase=0.02					,	__lowerCAmelCase=1E-12					,	__lowerCAmelCase=0					,	__lowerCAmelCase="topK"					,	__lowerCAmelCase="constant"					,	__lowerCAmelCase=0.0					,	**__lowerCAmelCase					,	):
														super().__init__(pad_token_id=__lowerCAmelCase					,	**__lowerCAmelCase					)

														UpperCamelCase__			       =     vocab_size
														UpperCamelCase__			       =     hidden_size
														UpperCamelCase__			       =     num_hidden_layers
														UpperCamelCase__			       =     num_attention_heads
														UpperCamelCase__			       =     hidden_act
														UpperCamelCase__			       =     intermediate_size
														UpperCamelCase__			       =     hidden_dropout_prob
														UpperCamelCase__			       =     attention_probs_dropout_prob
														UpperCamelCase__			       =     max_position_embeddings
														UpperCamelCase__			       =     type_vocab_size
														UpperCamelCase__			       =     initializer_range
														UpperCamelCase__			       =     layer_norm_eps
														UpperCamelCase__			       =     pruning_method
														UpperCamelCase__			       =     mask_init
														UpperCamelCase__			       =     mask_scale




