

'''simple docstring'''



from collections import OrderedDict
from typing import TYPE_CHECKING, Any, Mapping, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


if TYPE_CHECKING:
		from ... import FeatureExtractionMixin, PreTrainedTokenizerBase, TensorType


__snake_case			:     Dict										=      logging.get_logger(__name__)

__snake_case			:     str										=      {
    'microsoft/deberta-v2-xlarge': 'https://huggingface.co/microsoft/deberta-v2-xlarge/resolve/main/config.json',
    'microsoft/deberta-v2-xxlarge': 'https://huggingface.co/microsoft/deberta-v2-xxlarge/resolve/main/config.json',
    'microsoft/deberta-v2-xlarge-mnli': (
        'https://huggingface.co/microsoft/deberta-v2-xlarge-mnli/resolve/main/config.json'
    ),
    'microsoft/deberta-v2-xxlarge-mnli': (
        'https://huggingface.co/microsoft/deberta-v2-xxlarge-mnli/resolve/main/config.json'
    ),
}



class lowerCamelCase			(  lowercase_						):



			'''simple docstring'''






			__snake_case        =	'deberta-v2'


			def __init__(				self	:						Any		,      lowerCAmelCase_	:						Tuple=12_81_00		,      lowerCAmelCase_	:						List[Any]=15_36		,      lowerCAmelCase_	:						Any=24		,      lowerCAmelCase_	:						List[str]=24		,      lowerCAmelCase_	:						List[Any]=61_44		,      lowerCAmelCase_	:						Tuple="gelu"		,      lowerCAmelCase_	:						Optional[Any]=0.1		,      lowerCAmelCase_	:						int=0.1		,      lowerCAmelCase_	:						Dict=5_12		,      lowerCAmelCase_	:						Dict=0		,      lowerCAmelCase_	:						str=0.02		,      lowerCAmelCase_	:						Union[str, Any]=1e-7		,      lowerCAmelCase_	:						List[Any]=False		,      lowerCAmelCase_	:						List[Any]=-1		,      lowerCAmelCase_	:						int=0		,      lowerCAmelCase_	:						List[Any]=True		,      lowerCAmelCase_	:						Dict=None		,      lowerCAmelCase_	:						Union[str, Any]=0		,      lowerCAmelCase_	:						Dict="gelu"		,      **lowerCAmelCase_	:						List[str]		,      )      ->		List[str]:





									'''simple docstring'''
									super().__init__(**lowerCAmelCase_			)

									A__					:       str     =hidden_size
									A__					:       int     =num_hidden_layers
									A__					:       List[Any]     =num_attention_heads
									A__					:       List[Any]     =intermediate_size
									A__					:       Union[str, Any]     =hidden_act
									A__					:       List[str]     =hidden_dropout_prob
									A__					:       Dict     =attention_probs_dropout_prob
									A__					:       List[Any]     =max_position_embeddings
									A__					:       Any     =type_vocab_size
									A__					:       Dict     =initializer_range
									A__					:       int     =relative_attention
									A__					:       Any     =max_relative_positions
									A__					:       Dict     =pad_token_id
									A__					:       int     =position_biased_input

									# Backwards compatibility
									if type(lowerCAmelCase_			) == str:
															A__					:       Optional[int]     =[x.strip() for x in pos_att_type.lower().split("""|"""			)]

									A__					:       str     =pos_att_type
									A__					:       Optional[Any]     =vocab_size
									A__					:       Optional[Any]     =layer_norm_eps

									A__					:       int     =kwargs.get("""pooler_hidden_size"""		,      lowerCAmelCase_			)
									A__					:       Tuple     =pooler_dropout
									A__					:       Any     =pooler_hidden_act



class lowerCamelCase			(  lowercase_						):



			'''simple docstring'''






			@property
			def lowercase__						(				self	:						Optional[int]			)      ->		Mapping[str, Mapping[int, str]]:





									'''simple docstring'''
									if self.task == "multiple-choice":
															A__					:       Any     ={0: """batch""", 1: """choice""", 2: """sequence"""}
									else:
															A__					:       List[Any]     ={0: """batch""", 1: """sequence"""}
									if self._config.type_vocab_size > 0:
															return OrderedDict(
															    [("""input_ids""", dynamic_axis), ("""attention_mask""", dynamic_axis), ("""token_type_ids""", dynamic_axis)]			)
									else:
															return OrderedDict([("""input_ids""", dynamic_axis), ("""attention_mask""", dynamic_axis)]			)


			@property
			def lowercase__						(				self	:						Tuple			)      ->		int:





									'''simple docstring'''
									return 12



			def lowercase__						(				self	:						Optional[Any]		,      lowerCAmelCase_	:						Union["PreTrainedTokenizerBase", "FeatureExtractionMixin"]		,      lowerCAmelCase_	:						int = -1		,      lowerCAmelCase_	:						int = -1		,      lowerCAmelCase_	:						int = -1		,      lowerCAmelCase_	:						bool = False		,      lowerCAmelCase_	:						Optional["TensorType"] = None		,      lowerCAmelCase_	:						int = 3		,      lowerCAmelCase_	:						int = 40		,      lowerCAmelCase_	:						int = 40		,      lowerCAmelCase_	:						"PreTrainedTokenizerBase" = None		,      )      ->		Mapping[str, Any]:





									'''simple docstring'''
									A__					:       List[Any]     =super().generate_dummy_inputs(preprocessor=lowerCAmelCase_		,      framework=lowerCAmelCase_			)
									if self._config.type_vocab_size == 0 and "token_type_ids" in dummy_inputs:
															del dummy_inputs["token_type_ids"]
									return dummy_inputs




'''simple docstring'''



import torch
from torch import nn



class lowerCamelCase			(  nn.Module						):



			'''simple docstring'''






			def __init__(				self	:						Any		,      lowerCAmelCase_	:						List[str]		,      lowerCAmelCase_	:						List[Any]		,      lowerCAmelCase_	:						Dict		,      lowerCAmelCase_	:						str		,      lowerCAmelCase_	:						Optional[int]=1		,      lowerCAmelCase_	:						str=False			)      ->		List[str]:





									'''simple docstring'''
									super().__init__()

									A__					:       Any     =n_token
									A__					:       int     =d_embed
									A__					:       Any     =d_proj

									A__					:       Tuple     =cutoffs + [n_token]
									A__					:       Optional[Any]     =[0] + self.cutoffs
									A__					:       Dict     =div_val

									A__					:       str     =self.cutoffs[0]
									A__					:       Optional[Any]     =len(self.cutoffs			) - 1
									A__					:       List[Any]     =self.shortlist_size + self.n_clusters

									if self.n_clusters > 0:
															A__					:       Any     =nn.Parameter(torch.zeros(self.n_clusters		,      self.d_embed			)			)
															A__					:       str     =nn.Parameter(torch.zeros(self.n_clusters			)			)

									A__					:       Union[str, Any]     =nn.ModuleList()
									A__					:       Optional[int]     =nn.ParameterList()

									if div_val == 1:
															for i in range(len(self.cutoffs			)			):
																					if d_proj != d_embed:
																											self.out_projs.append(nn.Parameter(torch.FloatTensor(lowerCAmelCase_		,      lowerCAmelCase_			)			)			)
																					else:
																											self.out_projs.append(lowerCAmelCase_			)

															self.out_layers.append(nn.Linear(lowerCAmelCase_		,      lowerCAmelCase_			)			)
									else:
															for i in range(len(self.cutoffs			)			):
																					A__   ,						A__					:       Optional[int]     =self.cutoff_ends[i], self.cutoff_ends[i + 1]
																					A__					:       Tuple     =d_embed // (div_val**i)

																					self.out_projs.append(nn.Parameter(torch.FloatTensor(lowerCAmelCase_		,      lowerCAmelCase_			)			)			)

																					self.out_layers.append(nn.Linear(lowerCAmelCase_		,      r_idx - l_idx			)			)

									A__					:       Optional[int]     =keep_order


			def lowercase__						(				self	:						Dict		,      lowerCAmelCase_	:						int		,      lowerCAmelCase_	:						Tuple		,      lowerCAmelCase_	:						Union[str, Any]		,      lowerCAmelCase_	:						Any			)      ->		Union[str, Any]:





									'''simple docstring'''
									if proj is None:
															A__					:       Optional[int]     =nn.functional.linear(lowerCAmelCase_		,      lowerCAmelCase_		,      bias=lowerCAmelCase_			)
									else:
															# if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
															A__					:       Optional[int]     =nn.functional.linear(lowerCAmelCase_		,      proj.t().contiguous()			)
															A__					:       Union[str, Any]     =nn.functional.linear(lowerCAmelCase_		,      lowerCAmelCase_		,      bias=lowerCAmelCase_			)
															# else:
															#     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
															#     if bias is not None:
															#         logit = logit + bias

									return logit


			def lowercase__						(				self	:						Tuple		,      lowerCAmelCase_	:						List[str]		,      lowerCAmelCase_	:						List[Any]=None		,      lowerCAmelCase_	:						Dict=False			)      ->		Optional[int]:





									'''simple docstring'''

									if labels is not None:
															# Shift so that tokens < n predict n
															A__					:       Optional[Any]     =hidden[..., :-1, :].contiguous()
															A__					:       List[Any]     =labels[..., 1:].contiguous()
															A__					:       Optional[int]     =hidden.view(-1		,      hidden.size(-1			)			)
															A__					:       str     =labels.view(-1			)
															if hidden.size(0			) != labels.size(0			):
																					raise RuntimeError("""Input and labels should have the same size in the batch dimension."""			)
									else:
															A__					:       Optional[int]     =hidden.view(-1		,      hidden.size(-1			)			)

									if self.n_clusters == 0:
															A__					:       Optional[Any]     =self._compute_logit(lowerCAmelCase_		,      self.out_layers[0].weight		,      self.out_layers[0].bias		,      self.out_projs[0]			)
															if labels is not None:
																					A__					:       Tuple     =labels != -1_00
																					A__					:       int     =torch.zeros_like(lowerCAmelCase_		,      dtype=hidden.dtype		,      device=hidden.device			)
																					A__					:       Union[str, Any]     =(
																					    -nn.functional.log_softmax(lowerCAmelCase_		,      dim=-1			)[mask].gather(1		,      labels[mask].unsqueeze(1			)			).squeeze(1			)
																					)
															else:
																					A__					:       List[Any]     =nn.functional.log_softmax(lowerCAmelCase_		,      dim=-1			)
									else:
															# construct weights and biases
															A__   ,						A__					:       Any     =[], []
															for i in range(len(self.cutoffs			)			):
																					if self.div_val == 1:
																											A__   ,						A__					:       Optional[int]     =self.cutoff_ends[i], self.cutoff_ends[i + 1]
																											A__					:       int     =self.out_layers[0].weight[l_idx:r_idx]
																											A__					:       List[str]     =self.out_layers[0].bias[l_idx:r_idx]
																					else:
																											A__					:       List[str]     =self.out_layers[i].weight
																											A__					:       Union[str, Any]     =self.out_layers[i].bias

																					if i == 0:
																											A__					:       Tuple     =torch.cat([weight_i, self.cluster_weight]		,      dim=0			)
																											A__					:       List[str]     =torch.cat([bias_i, self.cluster_bias]		,      dim=0			)

																					weights.append(lowerCAmelCase_			)
																					biases.append(lowerCAmelCase_			)

															A__   ,						A__   ,						A__					:       Tuple     =weights[0], biases[0], self.out_projs[0]

															A__					:       List[Any]     =self._compute_logit(lowerCAmelCase_		,      lowerCAmelCase_		,      lowerCAmelCase_		,      lowerCAmelCase_			)
															A__					:       int     =nn.functional.log_softmax(lowerCAmelCase_		,      dim=1			)

															if labels is None:
																					A__					:       Union[str, Any]     =hidden.new_empty((head_logit.size(0			), self.n_token)			)
															else:
																					A__					:       Union[str, Any]     =torch.zeros_like(lowerCAmelCase_		,      dtype=hidden.dtype		,      device=hidden.device			)

															A__					:       Any     =0
															A__					:       Tuple     =[0] + self.cutoffs
															for i in range(len(lowerCAmelCase_			) - 1			):
																					A__   ,						A__					:       Tuple     =cutoff_values[i], cutoff_values[i + 1]

																					if labels is not None:
																											A__					:       Tuple     =(labels >= l_idx) & (labels < r_idx)
																											A__					:       Any     =mask_i.nonzero().squeeze()

																											if indices_i.numel() == 0:
																																	continue

																											A__					:       int     =labels.index_select(0		,      lowerCAmelCase_			) - l_idx
																											A__					:       List[str]     =head_logprob.index_select(0		,      lowerCAmelCase_			)
																											A__					:       str     =hidden.index_select(0		,      lowerCAmelCase_			)
																					else:
																											A__					:       Optional[Any]     =hidden

																					if i == 0:
																											if labels is not None:
																																	A__					:       Optional[Any]     =head_logprob_i.gather(1		,      target_i[:, None]			).squeeze(1			)
																											else:
																																	A__					:       Union[str, Any]     =head_logprob[:, : self.cutoffs[0]]
																					else:
																											A__   ,						A__   ,						A__					:       Dict     =weights[i], biases[i], self.out_projs[i]

																											A__					:       List[Any]     =self._compute_logit(lowerCAmelCase_		,      lowerCAmelCase_		,      lowerCAmelCase_		,      lowerCAmelCase_			)
																											A__					:       List[Any]     =nn.functional.log_softmax(lowerCAmelCase_		,      dim=1			)
																											A__					:       Optional[Any]     =self.cutoffs[0] + i - 1  # No probability for the head cluster
																											if labels is not None:
																																	A__					:       Union[str, Any]     =head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
																																	    1		,      target_i[:, None]			).squeeze(1			)
																											else:
																																	A__					:       List[str]     =head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
																																	A__					:       Tuple     =logprob_i

																					if labels is not None:
																											if (hasattr(self		,      """keep_order"""			) and self.keep_order) or keep_order:
																																	out.index_copy_(0		,      lowerCAmelCase_		,      -logprob_i			)
																											else:
																																	out[offset : offset + logprob_i.size(0			)].copy_(-logprob_i			)
																											offset += logprob_i.size(0			)

									return out



			def lowercase__						(				self	:						List[str]		,      lowerCAmelCase_	:						Optional[Any]			)      ->		Any:





									'''simple docstring'''
									if self.n_clusters == 0:
															A__					:       List[str]     =self._compute_logit(lowerCAmelCase_		,      self.out_layers[0].weight		,      self.out_layers[0].bias		,      self.out_projs[0]			)
															return nn.functional.log_softmax(lowerCAmelCase_		,      dim=-1			)
									else:
															# construct weights and biases
															A__   ,						A__					:       List[str]     =[], []
															for i in range(len(self.cutoffs			)			):
																					if self.div_val == 1:
																											A__   ,						A__					:       int     =self.cutoff_ends[i], self.cutoff_ends[i + 1]
																											A__					:       List[str]     =self.out_layers[0].weight[l_idx:r_idx]
																											A__					:       List[Any]     =self.out_layers[0].bias[l_idx:r_idx]
																					else:
																											A__					:       Dict     =self.out_layers[i].weight
																											A__					:       Any     =self.out_layers[i].bias

																					if i == 0:
																											A__					:       List[str]     =torch.cat([weight_i, self.cluster_weight]		,      dim=0			)
																											A__					:       Tuple     =torch.cat([bias_i, self.cluster_bias]		,      dim=0			)

																					weights.append(lowerCAmelCase_			)
																					biases.append(lowerCAmelCase_			)

															A__   ,						A__   ,						A__					:       Optional[int]     =weights[0], biases[0], self.out_projs[0]
															A__					:       Any     =self._compute_logit(lowerCAmelCase_		,      lowerCAmelCase_		,      lowerCAmelCase_		,      lowerCAmelCase_			)

															A__					:       Dict     =hidden.new_empty((head_logit.size(0			), self.n_token)			)
															A__					:       Dict     =nn.functional.log_softmax(lowerCAmelCase_		,      dim=1			)

															A__					:       Tuple     =[0] + self.cutoffs
															for i in range(len(lowerCAmelCase_			) - 1			):
																					A__   ,						A__					:       List[Any]     =cutoff_values[i], cutoff_values[i + 1]

																					if i == 0:
																											A__					:       Tuple     =head_logprob[:, : self.cutoffs[0]]
																					else:
																											A__   ,						A__   ,						A__					:       Any     =weights[i], biases[i], self.out_projs[i]

																											A__					:       Dict     =self._compute_logit(lowerCAmelCase_		,      lowerCAmelCase_		,      lowerCAmelCase_		,      lowerCAmelCase_			)
																											A__					:       str     =nn.functional.log_softmax(lowerCAmelCase_		,      dim=1			)

																											A__					:       str     =head_logprob[:, -i] + tail_logprob_i
																											A__					:       List[Any]     =logprob_i

															return out

