from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy



@add_start_docstrings(
    """The RoBERTa Model transformer with early exiting (DeeRoBERTa). """       ,      lowerCamelCase__       ,      )
class 					lowerCAmelCase_      (     lowerCamelCase__   ):



				"""simple docstring"""
				_lowerCAmelCase       :      Optional[Any]     					=  RobertaConfig
				_lowerCAmelCase       :      Optional[Any]     					=  'roberta'

				def __init__(			self						,      lowerCAmelCase      ):






										"""simple docstring"""

										super().__init__(lowercase__      )

										snake_case        =     RobertaEmbeddings(lowercase__      )
										self.init_weights()





@add_start_docstrings(
    """RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,\n    also takes care of multi-layer training. """       ,      lowerCamelCase__       ,      )
class 					lowerCAmelCase_      (     lowerCamelCase__   ):



				"""simple docstring"""
				_lowerCAmelCase       :      Any     					=  RobertaConfig
				_lowerCAmelCase       :      Any     					=  'roberta'

				def __init__(			self						,      lowerCAmelCase      ):






										"""simple docstring"""

										super().__init__(lowercase__      )
										snake_case        =     config.num_labels
										snake_case        =     config.num_hidden_layers

										snake_case        =     DeeRobertaModel(lowercase__      )
										snake_case        =     nn.Dropout(config.hidden_dropout_prob      )
										snake_case        =     nn.Linear(config.hidden_size						,      self.config.num_labels      )

				@add_start_docstrings_to_model_forward(lowercase__      )
				def 						snake_case	(			self						,      lowerCAmelCase=None						,      lowerCAmelCase=None						,      lowerCAmelCase=None						,      lowerCAmelCase=None						,      lowerCAmelCase=None						,      lowerCAmelCase=None						,      lowerCAmelCase=None						,      lowerCAmelCase=-1						,      lowerCAmelCase=False						,      ):






										"""simple docstring"""

										snake_case        =     self.num_layers
										try:
																snake_case        =     self.roberta(
																    lowercase__						,      attention_mask=lowercase__						,      token_type_ids=lowercase__						,      position_ids=lowercase__						,      head_mask=lowercase__						,      inputs_embeds=lowercase__						,      )

																snake_case        =     outputs[1]

																snake_case        =     self.dropout(lowercase__      )
																snake_case        =     self.classifier(lowercase__      )
																snake_case        =     (logits,) + outputs[2:]  # add hidden states and attention if they are here
										except HighwayException as e:
																snake_case        =     e.message
																snake_case        =     e.exit_layer
																snake_case        =     outputs[0]

										if not self.training:
																snake_case        =     entropy(lowercase__      )
																snake_case        =     []
																snake_case        =     []
										if labels is not None:
																if self.num_labels == 1:
																						#  We are doing regression
																						snake_case        =     MSELoss()
																						snake_case        =     loss_fct(logits.view(-1      )						,      labels.view(-1      )      )
																else:
																						snake_case        =     CrossEntropyLoss()
																						snake_case        =     loss_fct(logits.view(-1						,      self.num_labels      )						,      labels.view(-1      )      )

																# work with highway exits
																snake_case        =     []
																for highway_exit in outputs[-1]:
																						snake_case        =     highway_exit[0]
																						if not self.training:
																												highway_logits_all.append(lowercase__      )
																												highway_entropy.append(highway_exit[2]      )
																						if self.num_labels == 1:
																												#  We are doing regression
																												snake_case        =     MSELoss()
																												snake_case        =     loss_fct(highway_logits.view(-1      )						,      labels.view(-1      )      )
																						else:
																												snake_case        =     CrossEntropyLoss()
																												snake_case        =     loss_fct(highway_logits.view(-1						,      self.num_labels      )						,      labels.view(-1      )      )
																						highway_losses.append(lowercase__      )

																if train_highway:
																						snake_case        =     (sum(highway_losses[:-1]      ),) + outputs
																						# exclude the final highway, of course
																else:
																						snake_case        =     (loss,) + outputs
										if not self.training:
																snake_case        =     outputs + ((original_entropy, highway_entropy), exit_layer)
																if output_layer >= 0:
																						snake_case        =     (
																						    (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
																						)  # use the highway of the last layer

										return outputs  # (loss), logits, (hidden_states), (attentions), entropy



"""simple docstring"""
SCREAMING_SNAKE_CASE__   =					{str(digit): digit**5 for digit in range(10)}





def 				lowerCAmelCase__      (     _UpperCamelCase    :			int   )   ->			int:





						"""simple docstring"""






						return sum(DIGITS_FIFTH_POWER[digit] for digit in str(_UpperCamelCase   )   )





def 				lowerCAmelCase__      (     )   ->			int:





						"""simple docstring"""






						return sum(
						    number
						    for number in range(1_0_0_0	,     1_0_0_0_0_0_0   )
						    if number == digits_fifth_powers_sum(_UpperCamelCase   )   )


if __name__ == "__main__":
					print(solution())


