



from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy






@add_start_docstrings(
    "The RoBERTa Model transformer with early exiting (DeeRoBERTa). "			,						SCREAMING_SNAKE_CASE__			,						)
class SCREAMING_SNAKE_CASE__					(		SCREAMING_SNAKE_CASE__   ):




    '''simple docstring'''






    __lowerCamelCase      :			Any =				RobertaConfig
    __lowerCamelCase      :			Optional[int] =				"roberta"






    def __init__(	self,		lowerCamelCase__		):
        super().__init__(lowerCamelCase__		)

        A			:	Optional[Any]										=       RobertaEmbeddings(lowerCamelCase__		)
        self.init_weights()






@add_start_docstrings(
    "RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,\n    also takes care of multi-layer training. "			,						SCREAMING_SNAKE_CASE__			,						)
class SCREAMING_SNAKE_CASE__					(		SCREAMING_SNAKE_CASE__   ):




    '''simple docstring'''






    __lowerCamelCase      :			str =				RobertaConfig
    __lowerCamelCase      :			str =				"roberta"






    def __init__(	self,		lowerCamelCase__		):
        super().__init__(lowerCamelCase__		)
        A			:	List[Any]										=       config.num_labels
        A			:	str										=       config.num_hidden_layers

        A			:	Union[str, Any]										=       DeeRobertaModel(lowerCamelCase__		)
        A			:	Any										=       nn.Dropout(config.hidden_dropout_prob		)
        A			:	str										=       nn.Linear(config.hidden_size,		self.config.num_labels		)






    @add_start_docstrings_to_model_forward(lowerCamelCase__		)
    def       _lowerCAmelCase (	self,		lowerCamelCase__=None,		lowerCamelCase__=None,		lowerCamelCase__=None,		lowerCamelCase__=None,		lowerCamelCase__=None,		lowerCamelCase__=None,		lowerCamelCase__=None,		lowerCamelCase__=-1,		lowerCamelCase__=False,		):
        A			:	List[str]										=       self.num_layers
        try:
            A			:	List[str]										=       self.roberta(
                lowerCamelCase__,		attention_mask=lowerCamelCase__,		token_type_ids=lowerCamelCase__,		position_ids=lowerCamelCase__,		head_mask=lowerCamelCase__,		inputs_embeds=lowerCamelCase__,		)

            A			:	str										=       outputs[1]

            A			:	Dict										=       self.dropout(lowerCamelCase__		)
            A			:	int										=       self.classifier(lowerCamelCase__		)
            A			:	Optional[Any]										=       (logits,) + outputs[2:]  # add hidden states and attention if they are here
        except HighwayException as e:
            A			:	Optional[Any]										=       e.message
            A			:	Optional[Any]										=       e.exit_layer
            A			:	List[str]										=       outputs[0]

        if not self.training:
            A			:	Dict										=       entropy(lowerCamelCase__		)
            A			:	Dict										=       []
            A			:	Optional[Any]										=       []
        if labels is not None:
            if self.num_labels == 1:
                #  We are doing regression
                A			:	List[Any]										=       MSELoss()
                A			:	Optional[int]										=       loss_fct(logits.view(-1		),		labels.view(-1		)		)
            else:
                A			:	Optional[int]										=       CrossEntropyLoss()
                A			:	Any										=       loss_fct(logits.view(-1,		self.num_labels		),		labels.view(-1		)		)

            # work with highway exits
            A			:	int										=       []
            for highway_exit in outputs[-1]:
                A			:	str										=       highway_exit[0]
                if not self.training:
                    highway_logits_all.append(lowerCamelCase__		)
                    highway_entropy.append(highway_exit[2]		)
                if self.num_labels == 1:
                    #  We are doing regression
                    A			:	Optional[Any]										=       MSELoss()
                    A			:	List[str]										=       loss_fct(highway_logits.view(-1		),		labels.view(-1		)		)
                else:
                    A			:	List[Any]										=       CrossEntropyLoss()
                    A			:	Optional[Any]										=       loss_fct(highway_logits.view(-1,		self.num_labels		),		labels.view(-1		)		)
                highway_losses.append(lowerCamelCase__		)

            if train_highway:
                A			:	Dict										=       (sum(highway_losses[:-1]		),) + outputs
                # exclude the final highway, of course
            else:
                A			:	Optional[Any]										=       (loss,) + outputs
        if not self.training:
            A			:	str										=       outputs + ((original_entropy, highway_entropy), exit_layer)
            if output_layer >= 0:
                A			:	str										=       (
                    (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
                )  # use the highway of the last layer

        return outputs  # (loss), logits, (hidden_states), (attentions), entropy







from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


SCREAMING_SNAKE_CASE_:Any					=						logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_:Any					=						{
    """xlm-mlm-en-2048""": """https://huggingface.co/xlm-mlm-en-2048/resolve/main/config.json""",
    """xlm-mlm-ende-1024""": """https://huggingface.co/xlm-mlm-ende-1024/resolve/main/config.json""",
    """xlm-mlm-enfr-1024""": """https://huggingface.co/xlm-mlm-enfr-1024/resolve/main/config.json""",
    """xlm-mlm-enro-1024""": """https://huggingface.co/xlm-mlm-enro-1024/resolve/main/config.json""",
    """xlm-mlm-tlm-xnli15-1024""": """https://huggingface.co/xlm-mlm-tlm-xnli15-1024/resolve/main/config.json""",
    """xlm-mlm-xnli15-1024""": """https://huggingface.co/xlm-mlm-xnli15-1024/resolve/main/config.json""",
    """xlm-clm-enfr-1024""": """https://huggingface.co/xlm-clm-enfr-1024/resolve/main/config.json""",
    """xlm-clm-ende-1024""": """https://huggingface.co/xlm-clm-ende-1024/resolve/main/config.json""",
    """xlm-mlm-17-1280""": """https://huggingface.co/xlm-mlm-17-1280/resolve/main/config.json""",
    """xlm-mlm-100-1280""": """https://huggingface.co/xlm-mlm-100-1280/resolve/main/config.json""",
}






class SCREAMING_SNAKE_CASE__					(		SCREAMING_SNAKE_CASE__   ):




    '''simple docstring'''






    __lowerCamelCase      :			List[str] =				"xlm"
    __lowerCamelCase      :			Tuple =				{
        "hidden_size": "emb_dim",
        "num_attention_heads": "n_heads",
        "num_hidden_layers": "n_layers",
        "n_words": "vocab_size",  # For backward compatibility
    }






    def __init__(	self,		lowerCamelCase__=3_0145,		lowerCamelCase__=2048,		lowerCamelCase__=12,		lowerCamelCase__=16,		lowerCamelCase__=0.1,		lowerCamelCase__=0.1,		lowerCamelCase__=True,		lowerCamelCase__=False,		lowerCamelCase__=False,		lowerCamelCase__=False,		lowerCamelCase__=1,		lowerCamelCase__=True,		lowerCamelCase__=512,		lowerCamelCase__=2048**-0.5,		lowerCamelCase__=1e-12,		lowerCamelCase__=0.02,		lowerCamelCase__=0,		lowerCamelCase__=1,		lowerCamelCase__=2,		lowerCamelCase__=3,		lowerCamelCase__=5,		lowerCamelCase__=True,		lowerCamelCase__="first",		lowerCamelCase__=True,		lowerCamelCase__=None,		lowerCamelCase__=True,		lowerCamelCase__=0.1,		lowerCamelCase__=5,		lowerCamelCase__=5,		lowerCamelCase__=0,		lowerCamelCase__=0,		lowerCamelCase__=2,		lowerCamelCase__=0,		**lowerCamelCase__,		):
        A			:	Dict										=       vocab_size
        A			:	int										=       emb_dim
        A			:	str										=       n_layers
        A			:	Union[str, Any]										=       n_heads
        A			:	Optional[int]										=       dropout
        A			:	Union[str, Any]										=       attention_dropout
        A			:	Optional[Any]										=       gelu_activation
        A			:	Dict										=       sinusoidal_embeddings
        A			:	int										=       causal
        A			:	Optional[Any]										=       asm
        A			:	Any										=       n_langs
        A			:	List[str]										=       use_lang_emb
        A			:	Union[str, Any]										=       layer_norm_eps
        A			:	str										=       bos_index
        A			:	int										=       eos_index
        A			:	Tuple										=       pad_index
        A			:	str										=       unk_index
        A			:	Optional[Any]										=       mask_index
        A			:	Union[str, Any]										=       is_encoder
        A			:	Tuple										=       max_position_embeddings
        A			:	List[str]										=       embed_init_std
        A			:	Tuple										=       init_std
        A			:	Tuple										=       summary_type
        A			:	int										=       summary_use_proj
        A			:	List[Any]										=       summary_activation
        A			:	Optional[Any]										=       summary_proj_to_labels
        A			:	Optional[Any]										=       summary_first_dropout
        A			:	Optional[int]										=       start_n_top
        A			:	Optional[Any]										=       end_n_top
        A			:	List[str]										=       mask_token_id
        A			:	Tuple										=       lang_id

        if "n_words" in kwargs:
            A			:	List[str]										=       kwargs["""n_words"""]

        super().__init__(pad_token_id=lowerCamelCase__,		bos_token_id=lowerCamelCase__,		**lowerCamelCase__		)






class SCREAMING_SNAKE_CASE__					(		SCREAMING_SNAKE_CASE__   ):




    '''simple docstring'''






    @property
    def       _lowerCAmelCase (	self		):
        if self.task == "multiple-choice":
            A			:	Any										=       {0: """batch""", 1: """choice""", 2: """sequence"""}
        else:
            A			:	Union[str, Any]										=       {0: """batch""", 1: """sequence"""}
        return OrderedDict(
            [
                ("""input_ids""", dynamic_axis),
                ("""attention_mask""", dynamic_axis),
                ("""token_type_ids""", dynamic_axis),
            ]		)


