
import inspect
import tempfile
from collections import OrderedDict, UserDict
from collections.abc import MutableMapping
from contextlib import ExitStack, contextmanager
from dataclasses import fields
from enum import Enum
from typing import Any, ContextManager, List, Tuple

import numpy as np

from .import_utils import is_flax_available, is_tf_available, is_torch_available, is_torch_fx_proxy


if is_flax_available():
							import jax.numpy as jnp







class     lowercase		( snake_case_		):




							def __get__(    self       ,A__       ,A__=None):


												if obj is None:
																	return self
												if self.fget is None:
																	raise AttributeError('''unreadable attribute''')
												lowercase						     =						'__cached_' + self.fget.__name__
												lowercase						     =						getattr(_A       ,_A       ,_A)
												if cached is None:
																	lowercase						     =						self.fget(_A)
																	setattr(_A       ,_A       ,_A)
												return cached


def 	UpperCamelCase   (							lowerCAmelCase__		):
					'''simple docstring'''
					lowercase						     =						val.lower()
					if val in {"y", "yes", "t", "true", "on", "1"}:
										return 1
					if val in {"n", "no", "f", "false", "off", "0"}:
										return 0
					raise ValueError(f'invalid truth value {val!r}'		)


def 	UpperCamelCase   (							lowerCAmelCase__		):
					'''simple docstring'''
					if is_torch_fx_proxy(__a		):
										return True
					if is_torch_available():
										import torch

										if isinstance(__a			,      torch.Tensor		):
															return True
					if is_tf_available():
										import tensorflow as tf

										if isinstance(__a			,      tf.Tensor		):
															return True

					if is_flax_available():
										import jax.numpy as jnp
										from jax.core import Tracer

										if isinstance(__a			,      (jnp.ndarray, Tracer)		):
															return True

					return isinstance(__a			,      np.ndarray		)


def 	UpperCamelCase   (							lowerCAmelCase__		):
					'''simple docstring'''
					return isinstance(__a			,      np.ndarray		)


def 	UpperCamelCase   (							lowerCAmelCase__		):
					'''simple docstring'''
					return _is_numpy(__a		)


def 	UpperCamelCase   (							lowerCAmelCase__		):
					'''simple docstring'''
					import torch

					return isinstance(__a			,      torch.Tensor		)


def 	UpperCamelCase   (							lowerCAmelCase__		):
					'''simple docstring'''
					return False if not is_torch_available() else _is_torch(__a		)


def 	UpperCamelCase   (							lowerCAmelCase__		):
					'''simple docstring'''
					import torch

					return isinstance(__a			,      torch.device		)


def 	UpperCamelCase   (							lowerCAmelCase__		):
					'''simple docstring'''
					return False if not is_torch_available() else _is_torch_device(__a		)


def 	UpperCamelCase   (							lowerCAmelCase__		):
					'''simple docstring'''
					import torch

					if isinstance(__a			,      __a		):
										if hasattr(__a			,      __a		):
															lowercase						     =						getattr(__a			,      __a		)
										else:
															return False
					return isinstance(__a			,      torch.dtype		)


def 	UpperCamelCase   (							lowerCAmelCase__		):
					'''simple docstring'''
					return False if not is_torch_available() else _is_torch_dtype(__a		)


def 	UpperCamelCase   (							lowerCAmelCase__		):
					'''simple docstring'''
					import tensorflow as tf

					return isinstance(__a			,      tf.Tensor		)


def 	UpperCamelCase   (							lowerCAmelCase__		):
					'''simple docstring'''
					return False if not is_tf_available() else _is_tensorflow(__a		)


def 	UpperCamelCase   (							lowerCAmelCase__		):
					'''simple docstring'''
					import tensorflow as tf

					# the `is_symbolic_tensor` predicate is only available starting with TF 2.14
					if hasattr(__a			,      '''is_symbolic_tensor'''		):
										return tf.is_symbolic_tensor(__a		)
					return type(__a		) == tf.Tensor


def 	UpperCamelCase   (							lowerCAmelCase__		):
					'''simple docstring'''
					return False if not is_tf_available() else _is_tf_symbolic_tensor(__a		)


def 	UpperCamelCase   (							lowerCAmelCase__		):
					'''simple docstring'''
					import jax.numpy as jnp  # noqa: F811

					return isinstance(__a			,      jnp.ndarray		)


def 	UpperCamelCase   (							lowerCAmelCase__		):
					'''simple docstring'''
					return False if not is_flax_available() else _is_jax(__a		)


def 	UpperCamelCase   (							lowerCAmelCase__		):
					'''simple docstring'''
					if isinstance(__a			,      (dict, UserDict)		):
										return {k: to_py_obj(__a		) for k, v in obj.items()}
					elif isinstance(__a			,      (list, tuple)		):
										return [to_py_obj(__a		) for o in obj]
					elif is_tf_tensor(__a		):
										return obj.numpy().tolist()
					elif is_torch_tensor(__a		):
										return obj.detach().cpu().tolist()
					elif is_jax_tensor(__a		):
										return np.asarray(__a		).tolist()
					elif isinstance(__a			,      (np.ndarray, np.number)		):  # tolist also works on 0d np arrays
										return obj.tolist()
					else:
										return obj


def 	UpperCamelCase   (							lowerCAmelCase__		):
					'''simple docstring'''
					if isinstance(__a			,      (dict, UserDict)		):
										return {k: to_numpy(__a		) for k, v in obj.items()}
					elif isinstance(__a			,      (list, tuple)		):
										return np.array(__a		)
					elif is_tf_tensor(__a		):
										return obj.numpy()
					elif is_torch_tensor(__a		):
										return obj.detach().cpu().numpy()
					elif is_jax_tensor(__a		):
										return np.asarray(__a		)
					else:
										return obj







class     lowercase		( snake_case_		):




							def 			A__				(    self):
												lowercase						     =						fields(self)

												# Safety and consistency checks
												if not len(_A):
																	raise ValueError(f'{self.__class__.__name__} has no fields.')
												if not all(field.default is None for field in class_fields[1:]):
																	raise ValueError(f'{self.__class__.__name__} should not have more than one required field.')

												lowercase						     =						getattr(self       ,class_fields[0].name)
												lowercase						     =						all(getattr(self       ,field.name) is None for field in class_fields[1:])

												if other_fields_are_none and not is_tensor(_A):
																	if isinstance(_A       ,_A):
																						lowercase						     =						first_field.items()
																						lowercase						     =						True
																	else:
																						try:
																											lowercase						     =						iter(_A)
																											lowercase						     =						True
																						except TypeError:
																											lowercase						     =						False

            # if we provided an iterator as first field and the iterator is a (key, value) iterator
            # set the associated fields
																	if first_field_iterator:
																						for idx, element in enumerate(_A):
																											if (
																											    not isinstance(_A       ,(list, tuple))
																											    or not len(_A) == 2
																											    or not isinstance(element[0]       ,_A)
																											):
																																if idx == 0:
																																					# If we do not have an iterator of key/values, set it as attribute
																																					lowercase						     =						first_field
																																else:
																																					# If we have a mixed iterator, raise an error
																																					raise ValueError(
																																					    f'Cannot set key/value for {element}. It needs to be a tuple (key, value).')
																																break
																											setattr(self       ,element[0]       ,element[1])
																											if element[1] is not None:
																																lowercase						     =						element[1]
																	elif first_field is not None:
																						lowercase						     =						first_field
												else:
																	for field in class_fields:
																						lowercase						     =						getattr(self       ,field.name)
																						if v is not None:
																											lowercase						     =						v




							def __delitem__(    self       ,*A__       ,**A__):
												raise Exception(f'You cannot use ``__delitem__`` on a {self.__class__.__name__} instance.')




							def 			A__				(    self       ,*A__       ,**A__):
												raise Exception(f'You cannot use ``setdefault`` on a {self.__class__.__name__} instance.')




							def 			A__				(    self       ,*A__       ,**A__):
												raise Exception(f'You cannot use ``pop`` on a {self.__class__.__name__} instance.')




							def 			A__				(    self       ,*A__       ,**A__):
												raise Exception(f'You cannot use ``update`` on a {self.__class__.__name__} instance.')




							def __getitem__(    self       ,A__):


												if isinstance(_A       ,_A):
																	lowercase						     =						dict(self.items())
																	return inner_dict[k]
												else:
																	return self.to_tuple()[k]




							def __setattr__(    self       ,A__       ,A__):


												if name in self.keys() and value is not None:
																	# Don't call self.__setitem__ to avoid recursion errors
																	super().__setitem__(_A       ,_A)
												super().__setattr__(_A       ,_A)




							def __setitem__(    self       ,A__       ,A__):
												super().__setitem__(_A       ,_A)
												# Don't call self.__setattr__ to avoid recursion errors
												super().__setattr__(_A       ,_A)





							def 			A__				(    self):
												return tuple(self[k] for k in self.keys())







class     lowercase		( snake_case_	,  snake_case_		):




							@classmethod
							def 			A__				(    cls       ,A__):
												raise ValueError(
												    f'{value} is not a valid {cls.__name__}, please select one of {list(cls._valueamember_map_.keys())}')







class     lowercase		( snake_case_		):
							lowercase_	:	Union[str, Any]	 ="longest"
							lowercase_	:	Union[str, Any]	 ="max_length"
							lowercase_	:	Tuple	 ="do_not_pad"







class     lowercase		( snake_case_		):
							lowercase_	:	Dict	 ="pt"
							lowercase_	:	int	 ="tf"
							lowercase_	:	Any	 ="np"
							lowercase_	:	int	 ="jax"







class     lowercase		:




							def __init__(    self       ,A__):
												lowercase						     =						context_managers
												lowercase						     =						ExitStack()




							def __enter__(    self):


												for context_manager in self.context_managers:
																	self.stack.enter_context(_A)





							def __exit__(    self       ,*A__       ,**A__):
												self.stack.__exit__(*_A       ,**_A)


def 	UpperCamelCase   (							lowerCAmelCase__		):
					'''simple docstring'''
					lowercase						     =						infer_framework(__a		)
					if framework == "tf":
										lowercase						     =						inspect.signature(model_class.call		)  # TensorFlow models
					elif framework == "pt":
										lowercase						     =						inspect.signature(model_class.forward		)  # PyTorch models
					else:
										lowercase						     =						inspect.signature(model_class.__call__		)  # Flax models

					for p in signature.parameters:
										if p == "return_loss" and signature.parameters[p].default is True:
															return True

					return False


def 	UpperCamelCase   (							lowerCAmelCase__		):
					'''simple docstring'''
					lowercase						     =						model_class.__name__
					lowercase						     =						infer_framework(__a		)
					if framework == "tf":
										lowercase						     =						inspect.signature(model_class.call		)  # TensorFlow models
					elif framework == "pt":
										lowercase						     =						inspect.signature(model_class.forward		)  # PyTorch models
					else:
										lowercase						     =						inspect.signature(model_class.__call__		)  # Flax models

					if "QuestionAnswering" in model_name:
										return [p for p in signature.parameters if "label" in p or p in ("start_positions", "end_positions")]
					else:
										return [p for p in signature.parameters if "label" in p]


def 	UpperCamelCase   (							lowerCAmelCase__			,      lowerCAmelCase__ = ""			,      lowerCAmelCase__ = "."		):
					'''simple docstring'''

					def _flatten_dict(lowerCAmelCase__			,      lowerCAmelCase__=""			,      lowerCAmelCase__="."		):
										for k, v in d.items():
															lowercase						     =						str(__a		) + delimiter + str(__a		) if parent_key else k
															if v and isinstance(__a			,      __a		):
																				yield from flatten_dict(__a			,      __a			,      delimiter=__a		).items()
															else:
																				yield key, v

					return dict(_flatten_dict(__a			,      __a			,      __a		)		)


@contextmanager
def 	UpperCamelCase   (							lowerCAmelCase__			,      lowerCAmelCase__ = False		):
					'''simple docstring'''
					if use_temp_dir:
										with tempfile.TemporaryDirectory() as tmp_dir:
															yield tmp_dir
					else:
										yield working_dir


def 	UpperCamelCase   (							lowerCAmelCase__			,      lowerCAmelCase__=None		):
					'''simple docstring'''
					if is_numpy_array(__a		):
										return np.transpose(__a			,      axes=__a		)
					elif is_torch_tensor(__a		):
										return array.T if axes is None else array.permute(*__a		)
					elif is_tf_tensor(__a		):
										import tensorflow as tf

										return tf.transpose(__a			,      perm=__a		)
					elif is_jax_tensor(__a		):
										return jnp.transpose(__a			,      axes=__a		)
					else:
										raise ValueError(f'Type not supported for transpose: {type(__a		)}.'		)


def 	UpperCamelCase   (							lowerCAmelCase__			,      lowerCAmelCase__		):
					'''simple docstring'''
					if is_numpy_array(__a		):
										return np.reshape(__a			,      __a		)
					elif is_torch_tensor(__a		):
										return array.reshape(*__a		)
					elif is_tf_tensor(__a		):
										import tensorflow as tf

										return tf.reshape(__a			,      __a		)
					elif is_jax_tensor(__a		):
										return jnp.reshape(__a			,      __a		)
					else:
										raise ValueError(f'Type not supported for reshape: {type(__a		)}.'		)


def 	UpperCamelCase   (							lowerCAmelCase__			,      lowerCAmelCase__=None		):
					'''simple docstring'''
					if is_numpy_array(__a		):
										return np.squeeze(__a			,      axis=__a		)
					elif is_torch_tensor(__a		):
										return array.squeeze() if axis is None else array.squeeze(dim=__a		)
					elif is_tf_tensor(__a		):
										import tensorflow as tf

										return tf.squeeze(__a			,      axis=__a		)
					elif is_jax_tensor(__a		):
										return jnp.squeeze(__a			,      axis=__a		)
					else:
										raise ValueError(f'Type not supported for squeeze: {type(__a		)}.'		)


def 	UpperCamelCase   (							lowerCAmelCase__			,      lowerCAmelCase__		):
					'''simple docstring'''
					if is_numpy_array(__a		):
										return np.expand_dims(__a			,      __a		)
					elif is_torch_tensor(__a		):
										return array.unsqueeze(dim=__a		)
					elif is_tf_tensor(__a		):
										import tensorflow as tf

										return tf.expand_dims(__a			,      axis=__a		)
					elif is_jax_tensor(__a		):
										return jnp.expand_dims(__a			,      axis=__a		)
					else:
										raise ValueError(f'Type not supported for expand_dims: {type(__a		)}.'		)


def 	UpperCamelCase   (							lowerCAmelCase__		):
					'''simple docstring'''
					if is_numpy_array(__a		):
										return np.size(__a		)
					elif is_torch_tensor(__a		):
										return array.numel()
					elif is_tf_tensor(__a		):
										import tensorflow as tf

										return tf.size(__a		)
					elif is_jax_tensor(__a		):
										return array.size
					else:
										raise ValueError(f'Type not supported for expand_dims: {type(__a		)}.'		)


def 	UpperCamelCase   (							lowerCAmelCase__			,      lowerCAmelCase__		):
					'''simple docstring'''
					for key, value in auto_map.items():
										if isinstance(__a			,      (tuple, list)		):
															lowercase						     =						[f'{repo_id}--{v}' if (v is not None and '--' not in v) else v for v in value]
										elif value is not None and "--" not in value:
															lowercase						     =						f'{repo_id}--{value}'

					return auto_map


def 	UpperCamelCase   (							lowerCAmelCase__		):
					'''simple docstring'''
					for base_class in inspect.getmro(__a		):
										lowercase						     =						base_class.__module__
										lowercase						     =						base_class.__name__
										if module.startswith('''tensorflow'''		) or module.startswith('''keras'''		) or name == "TFPreTrainedModel":
															return "tf"
										elif module.startswith('''torch'''		) or name == "PreTrainedModel":
															return "pt"
										elif module.startswith('''flax'''		) or module.startswith('''jax'''		) or name == "FlaxPreTrainedModel":
															return "flax"
					else:
										raise TypeError(f'Could not infer framework from class {model_class}.'		)






import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.bert.modeling_bert import (
    BERT_INPUTS_DOCSTRING,
    BERT_START_DOCSTRING,
    BertEmbeddings,
    BertLayer,
    BertPooler,
    BertPreTrainedModel,
)



def 	UpperCamelCase   (							lowerCAmelCase__		):
					'''simple docstring'''
					lowercase						     =						torch.exp(lowerCAmelCase__		)
					lowercase						     =						torch.sum(lowerCAmelCase__			,      dim=1		)  # sum of exp(x_i)
					lowercase						     =						torch.sum(x * exp_x			,      dim=1		)  # sum of x_i * exp(x_i)
					return torch.log(lowerCAmelCase__		) - B / A







class     lowercase		( nn.Module		):




							def __init__(    self       ,A__):
												super().__init__()
												lowercase						     =						config.output_attentions
												lowercase						     =						config.output_hidden_states
												lowercase						     =						nn.ModuleList([BertLayer(A__) for _ in range(config.num_hidden_layers)])
												lowercase						     =						nn.ModuleList([BertHighway(A__) for _ in range(config.num_hidden_layers)])

												lowercase						     =						[-1 for _ in range(config.num_hidden_layers)]




							def 			A__				(    self       ,A__):
												if (type(A__) is float) or (type(A__) is int):
																	for i in range(len(self.early_exit_entropy)):
																						lowercase						     =						x
												else:
																	lowercase						     =						x




							def 			A__				(    self       ,A__):
												lowercase						     =						pooler.state_dict()
												for highway in self.highway:
																	for name, param in highway.pooler.state_dict().items():
																						param.copy_(loaded_model[name])





							def 			A__				(    self       ,A__       ,A__=None       ,A__=None       ,A__=None       ,A__=None       ,):
												lowercase						     =						()
												lowercase						     =						()
												lowercase						     =						()
												for i, layer_module in enumerate(self.layer):
																	if self.output_hidden_states:
																						lowercase						     =						all_hidden_states + (hidden_states,)

																	lowercase						     =						layer_module(
																	    A__       ,A__       ,head_mask[i]       ,A__       ,A__)
																	lowercase						     =						layer_outputs[0]

																	if self.output_attentions:
																						lowercase						     =						all_attentions + (layer_outputs[1],)

																	lowercase						     =						(hidden_states,)
																	if self.output_hidden_states:
																						lowercase						     =						current_outputs + (all_hidden_states,)
																	if self.output_attentions:
																						lowercase						     =						current_outputs + (all_attentions,)

																	lowercase						     =						self.highway[i](A__)
																	# logits, pooled_output

																	if not self.training:
																						lowercase						     =						highway_exit[0]
																						lowercase						     =						entropy(A__)
																						lowercase						     =						highway_exit + (highway_entropy,)  # logits, hidden_states(?), entropy
																						lowercase						     =						all_highway_exits + (highway_exit,)

																						if highway_entropy < self.early_exit_entropy[i]:
																											lowercase						     =						(highway_logits,) + current_outputs[1:] + (all_highway_exits,)
																											raise HighwayException(A__       ,i + 1)
																	else:
																						lowercase						     =						all_highway_exits + (highway_exit,)

        # Add last layer
												if self.output_hidden_states:
																	lowercase						     =						all_hidden_states + (hidden_states,)

												lowercase						     =						(hidden_states,)
												if self.output_hidden_states:
																	lowercase						     =						outputs + (all_hidden_states,)
												if self.output_attentions:
																	lowercase						     =						outputs + (all_attentions,)

												lowercase						     =						outputs + (all_highway_exits,)
												return outputs  # last-layer hidden state, (all hidden states), (all attentions), all highway exits







@add_start_docstrings(
    '''The Bert Model transformer with early exiting (DeeBERT). '''	,  SCREAMING_SNAKE_CASE__	,  )
class     lowercase		( SCREAMING_SNAKE_CASE__		):




							def __init__(    self       ,A__):
												super().__init__(A__)
												lowercase						     =						config

												lowercase						     =						BertEmbeddings(A__)
												lowercase						     =						DeeBertEncoder(A__)
												lowercase						     =						BertPooler(A__)

												self.init_weights()




							def 			A__				(    self):
												self.encoder.init_highway_pooler(self.pooler)




							def 			A__				(    self):
												return self.embeddings.word_embeddings




							def 			A__				(    self       ,A__):
												lowercase						     =						value




							def 			A__				(    self       ,A__):
												for layer, heads in heads_to_prune.items():
																	self.encoder.layer[layer].attention.prune_heads(A__)





							@add_start_docstrings_to_model_forward(A__)
							def 			A__				(    self       ,A__=None       ,A__=None       ,A__=None       ,A__=None       ,A__=None       ,A__=None       ,A__=None       ,A__=None       ,):
												if input_ids is not None and inputs_embeds is not None:
																	raise ValueError('''You cannot specify both input_ids and inputs_embeds at the same time''')
												elif input_ids is not None:
																	lowercase						     =						input_ids.size()
												elif inputs_embeds is not None:
																	lowercase						     =						inputs_embeds.size()[:-1]
												else:
																	raise ValueError('''You have to specify either input_ids or inputs_embeds''')

												lowercase						     =						input_ids.device if input_ids is not None else inputs_embeds.device

												if attention_mask is None:
																	lowercase						     =						torch.ones(A__       ,device=A__)
												if encoder_attention_mask is None:
																	lowercase						     =						torch.ones(A__       ,device=A__)
												if token_type_ids is None:
																	lowercase						     =						torch.zeros(A__       ,dtype=torch.long       ,device=A__)

												# We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
												# ourselves in which case we just need to make it broadcastable to all heads.
												lowercase						     =						self.get_extended_attention_mask(A__       ,A__       ,A__)

												# If a 2D ou 3D attention mask is provided for the cross-attention
												# we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
												if encoder_attention_mask.dim() == 3:
																	lowercase						     =						encoder_attention_mask[:, None, :, :]
												if encoder_attention_mask.dim() == 2:
																	lowercase						     =						encoder_attention_mask[:, None, None, :]

												lowercase						     =						encoder_extended_attention_mask.to(
												    dtype=next(self.parameters()).dtype)  # fp16 compatibility
												lowercase						     =						(1.0 - encoder_extended_attention_mask) * -10000.0

												# Prepare head mask if needed
												# 1.0 in head_mask indicate we keep the head
												# attention_probs has shape bsz x n_heads x N x N
												# input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
												# and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
												lowercase						     =						self.get_head_mask(A__       ,self.config.num_hidden_layers)

												lowercase						     =						self.embeddings(
												    input_ids=A__       ,position_ids=A__       ,token_type_ids=A__       ,inputs_embeds=A__)
												lowercase						     =						self.encoder(
												    A__       ,attention_mask=A__       ,head_mask=A__       ,encoder_hidden_states=A__       ,encoder_attention_mask=A__       ,)
												lowercase						     =						encoder_outputs[0]
												lowercase						     =						self.pooler(A__)

												lowercase						     =						(
												    sequence_output,
												    pooled_output,
												) + encoder_outputs[
												    1:
												]  # add hidden_states and attentions if they are here
												return outputs  # sequence_output, pooled_output, (hidden_states), (attentions), highway exits







class     lowercase		( SCREAMING_SNAKE_CASE__		):




							def __init__(    self       ,A__       ,A__):
												lowercase						     =						message
												lowercase						     =						exit_layer  # start from 1!







class     lowercase		( nn.Module		):




							def __init__(    self       ,A__):
												super().__init__()
												lowercase						     =						BertPooler(A__)
												lowercase						     =						nn.Dropout(config.hidden_dropout_prob)
												lowercase						     =						nn.Linear(config.hidden_size       ,config.num_labels)





							def 			A__				(    self       ,A__):
												# Pooler
												lowercase						     =						encoder_outputs[0]
												lowercase						     =						self.pooler(A__)
												# "return" pooler_output

												# BertModel
												lowercase						     =						(pooler_input, pooler_output) + encoder_outputs[1:]
												# "return" bmodel_output

												# Dropout and classification
												lowercase						     =						bmodel_output[1]

												lowercase						     =						self.dropout(A__)
												lowercase						     =						self.classifier(A__)

												return logits, pooled_output







@add_start_docstrings(
    '''Bert Model (with early exiting - DeeBERT) with a classifier on top,
    also takes care of multi-layer training. '''	,  SCREAMING_SNAKE_CASE__	,  )
class     lowercase		( SCREAMING_SNAKE_CASE__		):




							def __init__(    self       ,A__):
												super().__init__(A__)
												lowercase						     =						config.num_labels
												lowercase						     =						config.num_hidden_layers

												lowercase						     =						DeeBertModel(A__)
												lowercase						     =						nn.Dropout(config.hidden_dropout_prob)
												lowercase						     =						nn.Linear(config.hidden_size       ,self.config.num_labels)

												self.init_weights()





							@add_start_docstrings_to_model_forward(A__)
							def 			A__				(    self       ,A__=None       ,A__=None       ,A__=None       ,A__=None       ,A__=None       ,A__=None       ,A__=None       ,A__=-1       ,A__=False       ,):
												lowercase						     =						self.num_layers
												try:
																	lowercase						     =						self.bert(
																	    A__       ,attention_mask=A__       ,token_type_ids=A__       ,position_ids=A__       ,head_mask=A__       ,inputs_embeds=A__       ,)
																	# sequence_output, pooled_output, (hidden_states), (attentions), highway exits

																	lowercase						     =						outputs[1]

																	lowercase						     =						self.dropout(A__)
																	lowercase						     =						self.classifier(A__)
																	lowercase						     =						(logits,) + outputs[2:]  # add hidden states and attention if they are here
												except HighwayException as e:
																	lowercase						     =						e.message
																	lowercase						     =						e.exit_layer
																	lowercase						     =						outputs[0]

												if not self.training:
																	lowercase						     =						entropy(A__)
																	lowercase						     =						[]
																	lowercase						     =						[]
												if labels is not None:
																	if self.num_labels == 1:
																						#  We are doing regression
																						lowercase						     =						MSELoss()
																						lowercase						     =						loss_fct(logits.view(-1)       ,labels.view(-1))
																	else:
																						lowercase						     =						CrossEntropyLoss()
																						lowercase						     =						loss_fct(logits.view(-1       ,self.num_labels)       ,labels.view(-1))

																	# work with highway exits
																	lowercase						     =						[]
																	for highway_exit in outputs[-1]:
																						lowercase						     =						highway_exit[0]
																						if not self.training:
																											highway_logits_all.append(A__)
																											highway_entropy.append(highway_exit[2])
																						if self.num_labels == 1:
																											#  We are doing regression
																											lowercase						     =						MSELoss()
																											lowercase						     =						loss_fct(highway_logits.view(-1)       ,labels.view(-1))
																						else:
																											lowercase						     =						CrossEntropyLoss()
																											lowercase						     =						loss_fct(highway_logits.view(-1       ,self.num_labels)       ,labels.view(-1))
																						highway_losses.append(A__)

																	if train_highway:
																						lowercase						     =						(sum(highway_losses[:-1]),) + outputs
																						# exclude the final highway, of course
																	else:
																						lowercase						     =						(loss,) + outputs
												if not self.training:
																	lowercase						     =						outputs + ((original_entropy, highway_entropy), exit_layer)
																	if output_layer >= 0:
																						lowercase						     =						(
																						    (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
																						)  # use the highway of the last layer

												return outputs  # (loss), logits, (hidden_states), (attentions), (highway_exits)




