

"""simple docstring"""


from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy





@add_start_docstrings(
    """The RoBERTa Model transformer with early exiting (DeeRoBERTa). """,							_lowerCAmelCase,							)
class    snake_case     (    _lowerCAmelCase  ):
    SCREAMING_SNAKE_CASE_       :				Any       				=  RobertaConfig
    SCREAMING_SNAKE_CASE_       :				List[Any]       				=  """roberta"""

    def __init__(     self							:    List[str]					,  UpperCamelCase__							:    Union[str, Any])->       str:



      '''simple docstring'''


      super().__init__(SCREAMING_SNAKE_CASE_)

      __lowerCAmelCase:	Optional[int]   				=      RobertaEmbeddings(SCREAMING_SNAKE_CASE_)
      self.init_weights()





@add_start_docstrings(
    """RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,
    also takes care of multi-layer training. """,							_lowerCAmelCase,							)
class    snake_case     (    _lowerCAmelCase  ):
    SCREAMING_SNAKE_CASE_       :				str       				=  RobertaConfig
    SCREAMING_SNAKE_CASE_       :				str       				=  """roberta"""

    def __init__(     self							:    Union[str, Any]					,  UpperCamelCase__							:    int)->       Optional[int]:



      '''simple docstring'''


      super().__init__(SCREAMING_SNAKE_CASE_)
      __lowerCAmelCase:	Optional[int]   				=      config.num_labels
      __lowerCAmelCase:	Any   				=      config.num_hidden_layers

      __lowerCAmelCase:	Dict   				=      DeeRobertaModel(SCREAMING_SNAKE_CASE_)
      __lowerCAmelCase:	Dict   				=      nn.Dropout(config.hidden_dropout_prob)
      __lowerCAmelCase:	List[Any]   				=      nn.Linear(config.hidden_size					,  self.config.num_labels)




    @add_start_docstrings_to_model_forward(SCREAMING_SNAKE_CASE_)
    def 							lowercase_					(     self							:    int					,  UpperCamelCase__							:    Optional[int]=None					,  UpperCamelCase__							:    Optional[int]=None					,  UpperCamelCase__							:    int=None					,  UpperCamelCase__							:    Tuple=None					,  UpperCamelCase__							:    Any=None					,  UpperCamelCase__							:    Union[str, Any]=None					,  UpperCamelCase__							:    int=None					,  UpperCamelCase__							:    Any=-1					,  UpperCamelCase__							:    Union[str, Any]=False					,  )->       Union[str, Any]:



      '''simple docstring'''


      __lowerCAmelCase:	List[Any]   				=      self.num_layers
      try:
        __lowerCAmelCase:	int   				=      self.roberta(
            SCREAMING_SNAKE_CASE_					,  attention_mask=SCREAMING_SNAKE_CASE_					,  token_type_ids=SCREAMING_SNAKE_CASE_					,  position_ids=SCREAMING_SNAKE_CASE_					,  head_mask=SCREAMING_SNAKE_CASE_					,  inputs_embeds=SCREAMING_SNAKE_CASE_					,  )

        __lowerCAmelCase:	Optional[int]   				=      outputs[1]

        __lowerCAmelCase:	List[str]   				=      self.dropout(SCREAMING_SNAKE_CASE_)
        __lowerCAmelCase:	Any   				=      self.classifier(SCREAMING_SNAKE_CASE_)
        __lowerCAmelCase:	List[Any]   				=      (logits,) + outputs[2:]  # add hidden states and attention if they are here
      except HighwayException as e:
        __lowerCAmelCase:	Optional[Any]   				=      e.message
        __lowerCAmelCase:	Optional[Any]   				=      e.exit_layer
        __lowerCAmelCase:	str   				=      outputs[0]

      if not self.training:
        __lowerCAmelCase:	Optional[int]   				=      entropy(SCREAMING_SNAKE_CASE_)
        __lowerCAmelCase:	int   				=      []
        __lowerCAmelCase:	Any   				=      []
      if labels is not None:
        if self.num_labels == 1:
          #  We are doing regression
          __lowerCAmelCase:	Union[str, Any]   				=      MSELoss()
          __lowerCAmelCase:	Dict   				=      loss_fct(logits.view(-1)					,  labels.view(-1))
        else:
          __lowerCAmelCase:	List[str]   				=      CrossEntropyLoss()
          __lowerCAmelCase:	str   				=      loss_fct(logits.view(-1					,  self.num_labels)					,  labels.view(-1))

        # work with highway exits
        __lowerCAmelCase:	List[str]   				=      []
        for highway_exit in outputs[-1]:
          __lowerCAmelCase:	Union[str, Any]   				=      highway_exit[0]
          if not self.training:
            highway_logits_all.append(SCREAMING_SNAKE_CASE_)
            highway_entropy.append(highway_exit[2])
          if self.num_labels == 1:
            #  We are doing regression
            __lowerCAmelCase:	Optional[int]   				=      MSELoss()
            __lowerCAmelCase:	str   				=      loss_fct(highway_logits.view(-1)					,  labels.view(-1))
          else:
            __lowerCAmelCase:	int   				=      CrossEntropyLoss()
            __lowerCAmelCase:	Tuple   				=      loss_fct(highway_logits.view(-1					,  self.num_labels)					,  labels.view(-1))
          highway_losses.append(SCREAMING_SNAKE_CASE_)

        if train_highway:
          __lowerCAmelCase:	Optional[Any]   				=      (sum(highway_losses[:-1]),) + outputs
          # exclude the final highway, of course
        else:
          __lowerCAmelCase:	List[str]   				=      (loss,) + outputs
      if not self.training:
        __lowerCAmelCase:	str   				=      outputs + ((original_entropy, highway_entropy), exit_layer)
        if output_layer >= 0:
          __lowerCAmelCase:	Any   				=      (
              (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
          )  # use the highway of the last layer

      return outputs  # (loss), logits, (hidden_states), (attentions), entropy





"""simple docstring"""


from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


__A						       =      {
    "configuration_swinv2": ["SWINV2_PRETRAINED_CONFIG_ARCHIVE_MAP", "Swinv2Config"],
}


try:
			if not is_torch_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			__A						       =      [
			    "SWINV2_PRETRAINED_MODEL_ARCHIVE_LIST",
			    "Swinv2ForImageClassification",
			    "Swinv2ForMaskedImageModeling",
			    "Swinv2Model",
			    "Swinv2PreTrainedModel",
			]


if TYPE_CHECKING:
			from .configuration_swinva import SWINV2_PRETRAINED_CONFIG_ARCHIVE_MAP, SwinvaConfig

			try:
						if not is_torch_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_swinva import (
						    SWINV2_PRETRAINED_MODEL_ARCHIVE_LIST,
						    SwinvaForImageClassification,
						    SwinvaForMaskedImageModeling,
						    SwinvaModel,
						    SwinvaPreTrainedModel,
						)


else:
			import sys

			__A						       =      _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)


