'''simple docstring'''





from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy




@add_start_docstrings(
    """The RoBERTa Model transformer with early exiting (DeeRoBERTa). """		,     UpperCamelCase__		,     )
class 	__A				(			UpperCamelCase__     ):
    a__	:						int	       =  RobertaConfig
    a__	:						Optional[Any]	       =  """roberta"""



    def __init__(self   :   Dict       ,      __a   :   str			):
         super().__init__(__a			)

         UpperCAmelCase_	   =       RobertaEmbeddings(__a			)
         self.init_weights()




@add_start_docstrings(
    """RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,
    also takes care of multi-layer training. """		,     UpperCamelCase__		,     )
class 	__A				(			UpperCamelCase__     ):
    a__	:						str	       =  RobertaConfig
    a__	:						Optional[int]	       =  """roberta"""



    def __init__(self   :   List[Any]       ,      __a   :   str			):
         super().__init__(__a			)
         UpperCAmelCase_	   =       config.num_labels
         UpperCAmelCase_	   =       config.num_hidden_layers

         UpperCAmelCase_	   =       DeeRobertaModel(__a			)
         UpperCAmelCase_	   =       nn.Dropout(config.hidden_dropout_prob			)
         UpperCAmelCase_	   =       nn.Linear(config.hidden_size       ,      self.config.num_labels			)






    @add_start_docstrings_to_model_forward(__a			)
    def  _lowercase						(self   :   Optional[int]       ,      __a   :   int=None       ,      __a   :   List[str]=None       ,      __a   :   List[str]=None       ,      __a   :   List[str]=None       ,      __a   :   List[str]=None       ,      __a   :   List[str]=None       ,      __a   :   Union[str, Any]=None       ,      __a   :   Optional[Any]=-1       ,      __a   :   List[str]=False       ,      ):
         UpperCAmelCase_	   =       self.num_layers
         try:
              UpperCAmelCase_	   =       self.roberta(
                  __a       ,      attention_mask=__a       ,      token_type_ids=__a       ,      position_ids=__a       ,      head_mask=__a       ,      inputs_embeds=__a       ,      )

              UpperCAmelCase_	   =       outputs[1]

              UpperCAmelCase_	   =       self.dropout(__a			)
              UpperCAmelCase_	   =       self.classifier(__a			)
              UpperCAmelCase_	   =       (logits,) + outputs[2:]  # add hidden states and attention if they are here
         except HighwayException as e:
              UpperCAmelCase_	   =       e.message
              UpperCAmelCase_	   =       e.exit_layer
              UpperCAmelCase_	   =       outputs[0]

         if not self.training:
              UpperCAmelCase_	   =       entropy(__a			)
              UpperCAmelCase_	   =       []
              UpperCAmelCase_	   =       []
         if labels is not None:
              if self.num_labels == 1:
                   #  We are doing regression
                   UpperCAmelCase_	   =       MSELoss()
                   UpperCAmelCase_	   =       loss_fct(logits.view(-1			)       ,      labels.view(-1			)			)
              else:
                   UpperCAmelCase_	   =       CrossEntropyLoss()
                   UpperCAmelCase_	   =       loss_fct(logits.view(-1       ,      self.num_labels			)       ,      labels.view(-1			)			)

              # work with highway exits
              UpperCAmelCase_	   =       []
              for highway_exit in outputs[-1]:
                   UpperCAmelCase_	   =       highway_exit[0]
                   if not self.training:
                        highway_logits_all.append(__a			)
                        highway_entropy.append(highway_exit[2]			)
                   if self.num_labels == 1:
                        #  We are doing regression
                        UpperCAmelCase_	   =       MSELoss()
                        UpperCAmelCase_	   =       loss_fct(highway_logits.view(-1			)       ,      labels.view(-1			)			)
                   else:
                        UpperCAmelCase_	   =       CrossEntropyLoss()
                        UpperCAmelCase_	   =       loss_fct(highway_logits.view(-1       ,      self.num_labels			)       ,      labels.view(-1			)			)
                   highway_losses.append(__a			)

              if train_highway:
                   UpperCAmelCase_	   =       (sum(highway_losses[:-1]			),) + outputs
                   # exclude the final highway, of course
              else:
                   UpperCAmelCase_	   =       (loss,) + outputs
         if not self.training:
              UpperCAmelCase_	   =       outputs + ((original_entropy, highway_entropy), exit_layer)
              if output_layer >= 0:
                   UpperCAmelCase_	   =       (
                       (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
                   )  # use the highway of the last layer

         return outputs  # (loss), logits, (hidden_states), (attentions), entropy





'''simple docstring'''





import unittest

from transformers import load_tool
from transformers.utils import is_torch_available


if is_torch_available():
    import torch

from transformers.testing_utils import require_torch

from .test_tools_common import ToolTesterMixin




@require_torch
class 	__A				(			unittest.TestCase		,     UpperCamelCase__     ):



    def  _lowercase						(self   :   Tuple			):
         UpperCAmelCase_	   =       load_tool("text-to-speech"			)
         self.tool.setup()



    def  _lowercase						(self   :   Union[str, Any]			):
         # SpeechT5 isn't deterministic
         torch.manual_seed(0			)
         UpperCAmelCase_	   =       self.tool("hey"			)
         UpperCAmelCase_	   =       result.to_raw()
         self.assertTrue(
             torch.allclose(
                 resulting_tensor[:3]       ,      torch.tensor([-0.0_00_59_66_66_88_32_11_58_29, -0.0_00_36_57_64_01_90_79_50_64, -0.00_01_34_39_50_27_99_88_34_85]			)       ,      )			)






    def  _lowercase						(self   :   List[str]			):
         # SpeechT5 isn't deterministic
         torch.manual_seed(0			)
         UpperCAmelCase_	   =       self.tool("hey"			)
         UpperCAmelCase_	   =       result.to_raw()
         self.assertTrue(
             torch.allclose(
                 resulting_tensor[:3]       ,      torch.tensor([-0.0_00_59_66_66_88_32_11_58_29, -0.0_00_36_57_64_01_90_79_50_64, -0.00_01_34_39_50_27_99_88_34_85]			)       ,      )			)




