





"""simple docstring"""




import torch
from torch import nn




class 			_a						(				nn.Module    ):






				def __init__(			self		:   str,			lowerCAmelCase__		:   Tuple,			lowerCAmelCase__		:   Optional[int],			lowerCAmelCase__		:   Optional[Any],			lowerCAmelCase__		:   str,			lowerCAmelCase__		:   Dict=1,			lowerCAmelCase__		:   Any=False							)     ->				Any:




										'''simple docstring'''



										super().__init__()

										_UpperCamelCase      : str         =   n_token
										_UpperCamelCase      : str         =   d_embed
										_UpperCamelCase      : Tuple         =   d_proj

										_UpperCamelCase      : str         =   cutoffs + [n_token]
										_UpperCamelCase      : int         =   [0] + self.cutoffs
										_UpperCamelCase      : int         =   div_val

										_UpperCamelCase      : Any         =   self.cutoffs[0]
										_UpperCamelCase      : List[str]         =   len(self.cutoffs							) - 1
										_UpperCamelCase      : List[str]         =   self.shortlist_size + self.n_clusters

										if self.n_clusters > 0:
																_UpperCamelCase      : Optional[Any]         =   nn.Parameter(torch.zeros(self.n_clusters,			self.d_embed							)							)
																_UpperCamelCase      : str         =   nn.Parameter(torch.zeros(self.n_clusters							)							)

										_UpperCamelCase      : Tuple         =   nn.ModuleList()
										_UpperCamelCase      : Union[str, Any]         =   nn.ParameterList()

										if div_val == 1:
																for i in range(len(self.cutoffs							)							):
																						if d_proj != d_embed:
																												self.out_projs.append(nn.Parameter(torch.FloatTensor(lowerCAmelCase__,			lowerCAmelCase__							)							)							)
																						else:
																												self.out_projs.append(lowerCAmelCase__							)

																self.out_layers.append(nn.Linear(lowerCAmelCase__,			lowerCAmelCase__							)							)
										else:
																for i in range(len(self.cutoffs							)							):
																						_UpperCamelCase       ,							_UpperCamelCase      : Optional[Any]         =   self.cutoff_ends[i], self.cutoff_ends[i + 1]
																						_UpperCamelCase      : int         =   d_embed // (div_val**i)

																						self.out_projs.append(nn.Parameter(torch.FloatTensor(lowerCAmelCase__,			lowerCAmelCase__							)							)							)

																						self.out_layers.append(nn.Linear(lowerCAmelCase__,			r_idx - l_idx							)							)

										_UpperCamelCase      : str         =   keep_order






				def     snake_case     (			self		:   Union[str, Any],			lowerCAmelCase__		:   List[str],			lowerCAmelCase__		:   Optional[int],			lowerCAmelCase__		:   Tuple,			lowerCAmelCase__		:   Optional[int]							)     ->				List[Any]:




										'''simple docstring'''



										if proj is None:
																_UpperCamelCase      : Dict         =   nn.functional.linear(lowerCAmelCase__,			lowerCAmelCase__,			bias=lowerCAmelCase__							)
										else:
																# if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
																_UpperCamelCase      : List[str]         =   nn.functional.linear(lowerCAmelCase__,			proj.t().contiguous()							)
																_UpperCamelCase      : Union[str, Any]         =   nn.functional.linear(lowerCAmelCase__,			lowerCAmelCase__,			bias=lowerCAmelCase__							)
																# else:
																#     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
																#     if bias is not None:
																#         logit = logit + bias

										return logit






				def     snake_case     (			self		:   List[str],			lowerCAmelCase__		:   int,			lowerCAmelCase__		:   Optional[Any]=None,			lowerCAmelCase__		:   Optional[int]=False							)     ->				Optional[int]:




										'''simple docstring'''



										if labels is not None:
																# Shift so that tokens < n predict n
																_UpperCamelCase      : List[Any]         =   hidden[..., :-1, :].contiguous()
																_UpperCamelCase      : int         =   labels[..., 1:].contiguous()
																_UpperCamelCase      : Optional[Any]         =   hidden.view(-1,			hidden.size(-1							)							)
																_UpperCamelCase      : List[str]         =   labels.view(-1							)
																if hidden.size(0							) != labels.size(0							):
																						raise RuntimeError('''Input and labels should have the same size in the batch dimension.'''							)
										else:
																_UpperCamelCase      : Dict         =   hidden.view(-1,			hidden.size(-1							)							)

										if self.n_clusters == 0:
																_UpperCamelCase      : Any         =   self._compute_logit(lowerCAmelCase__,			self.out_layers[0].weight,			self.out_layers[0].bias,			self.out_projs[0]							)
																if labels is not None:
																						_UpperCamelCase      : Tuple         =   labels != -1_0_0
																						_UpperCamelCase      : List[str]         =   torch.zeros_like(lowerCAmelCase__,			dtype=hidden.dtype,			device=hidden.device							)
																						_UpperCamelCase      : Any         =   (
																						    -nn.functional.log_softmax(lowerCAmelCase__,			dim=-1							)[mask].gather(1,			labels[mask].unsqueeze(1							)							).squeeze(1							)
																						)
																else:
																						_UpperCamelCase      : Dict         =   nn.functional.log_softmax(lowerCAmelCase__,			dim=-1							)
										else:
																# construct weights and biases
																_UpperCamelCase       ,							_UpperCamelCase      : Tuple         =   [], []
																for i in range(len(self.cutoffs							)							):
																						if self.div_val == 1:
																												_UpperCamelCase       ,							_UpperCamelCase      : List[str]         =   self.cutoff_ends[i], self.cutoff_ends[i + 1]
																												_UpperCamelCase      : Union[str, Any]         =   self.out_layers[0].weight[l_idx:r_idx]
																												_UpperCamelCase      : Optional[int]         =   self.out_layers[0].bias[l_idx:r_idx]
																						else:
																												_UpperCamelCase      : Optional[Any]         =   self.out_layers[i].weight
																												_UpperCamelCase      : List[str]         =   self.out_layers[i].bias

																						if i == 0:
																												_UpperCamelCase      : Any         =   torch.cat([weight_i, self.cluster_weight],			dim=0							)
																												_UpperCamelCase      : Tuple         =   torch.cat([bias_i, self.cluster_bias],			dim=0							)

																						weights.append(lowerCAmelCase__							)
																						biases.append(lowerCAmelCase__							)

																_UpperCamelCase       ,							_UpperCamelCase       ,							_UpperCamelCase      : List[Any]         =   weights[0], biases[0], self.out_projs[0]

																_UpperCamelCase      : Dict         =   self._compute_logit(lowerCAmelCase__,			lowerCAmelCase__,			lowerCAmelCase__,			lowerCAmelCase__							)
																_UpperCamelCase      : Optional[Any]         =   nn.functional.log_softmax(lowerCAmelCase__,			dim=1							)

																if labels is None:
																						_UpperCamelCase      : Optional[int]         =   hidden.new_empty((head_logit.size(0							), self.n_token)							)
																else:
																						_UpperCamelCase      : Dict         =   torch.zeros_like(lowerCAmelCase__,			dtype=hidden.dtype,			device=hidden.device							)

																_UpperCamelCase      : List[Any]         =   0
																_UpperCamelCase      : Union[str, Any]         =   [0] + self.cutoffs
																for i in range(len(lowerCAmelCase__							) - 1							):
																						_UpperCamelCase       ,							_UpperCamelCase      : Any         =   cutoff_values[i], cutoff_values[i + 1]

																						if labels is not None:
																												_UpperCamelCase      : Any         =   (labels >= l_idx) & (labels < r_idx)
																												_UpperCamelCase      : Optional[int]         =   mask_i.nonzero().squeeze()

																												if indices_i.numel() == 0:
																																		continue

																												_UpperCamelCase      : Union[str, Any]         =   labels.index_select(0,			lowerCAmelCase__							) - l_idx
																												_UpperCamelCase      : List[Any]         =   head_logprob.index_select(0,			lowerCAmelCase__							)
																												_UpperCamelCase      : Optional[Any]         =   hidden.index_select(0,			lowerCAmelCase__							)
																						else:
																												_UpperCamelCase      : Optional[int]         =   hidden

																						if i == 0:
																												if labels is not None:
																																		_UpperCamelCase      : Union[str, Any]         =   head_logprob_i.gather(1,			target_i[:, None]							).squeeze(1							)
																												else:
																																		_UpperCamelCase      : Dict         =   head_logprob[:, : self.cutoffs[0]]
																						else:
																												_UpperCamelCase       ,							_UpperCamelCase       ,							_UpperCamelCase      : int         =   weights[i], biases[i], self.out_projs[i]

																												_UpperCamelCase      : Any         =   self._compute_logit(lowerCAmelCase__,			lowerCAmelCase__,			lowerCAmelCase__,			lowerCAmelCase__							)
																												_UpperCamelCase      : Optional[Any]         =   nn.functional.log_softmax(lowerCAmelCase__,			dim=1							)
																												_UpperCamelCase      : Tuple         =   self.cutoffs[0] + i - 1  # No probability for the head cluster
																												if labels is not None:
																																		_UpperCamelCase      : Any         =   head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
																																		    1,			target_i[:, None]							).squeeze(1							)
																												else:
																																		_UpperCamelCase      : str         =   head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
																																		_UpperCamelCase      : List[str]         =   logprob_i

																						if labels is not None:
																												if (hasattr(self,			'''keep_order'''							) and self.keep_order) or keep_order:
																																		out.index_copy_(0,			lowerCAmelCase__,			-logprob_i							)
																												else:
																																		out[offset : offset + logprob_i.size(0							)].copy_(-logprob_i							)
																												offset += logprob_i.size(0							)

										return out






				def     snake_case     (			self		:   Optional[int],			lowerCAmelCase__		:   Tuple							)     ->				str:




										'''simple docstring'''



										if self.n_clusters == 0:
																_UpperCamelCase      : Any         =   self._compute_logit(lowerCAmelCase__,			self.out_layers[0].weight,			self.out_layers[0].bias,			self.out_projs[0]							)
																return nn.functional.log_softmax(lowerCAmelCase__,			dim=-1							)
										else:
																# construct weights and biases
																_UpperCamelCase       ,							_UpperCamelCase      : Tuple         =   [], []
																for i in range(len(self.cutoffs							)							):
																						if self.div_val == 1:
																												_UpperCamelCase       ,							_UpperCamelCase      : List[str]         =   self.cutoff_ends[i], self.cutoff_ends[i + 1]
																												_UpperCamelCase      : Tuple         =   self.out_layers[0].weight[l_idx:r_idx]
																												_UpperCamelCase      : Union[str, Any]         =   self.out_layers[0].bias[l_idx:r_idx]
																						else:
																												_UpperCamelCase      : Optional[Any]         =   self.out_layers[i].weight
																												_UpperCamelCase      : str         =   self.out_layers[i].bias

																						if i == 0:
																												_UpperCamelCase      : List[Any]         =   torch.cat([weight_i, self.cluster_weight],			dim=0							)
																												_UpperCamelCase      : Optional[int]         =   torch.cat([bias_i, self.cluster_bias],			dim=0							)

																						weights.append(lowerCAmelCase__							)
																						biases.append(lowerCAmelCase__							)

																_UpperCamelCase       ,							_UpperCamelCase       ,							_UpperCamelCase      : Union[str, Any]         =   weights[0], biases[0], self.out_projs[0]
																_UpperCamelCase      : Any         =   self._compute_logit(lowerCAmelCase__,			lowerCAmelCase__,			lowerCAmelCase__,			lowerCAmelCase__							)

																_UpperCamelCase      : List[Any]         =   hidden.new_empty((head_logit.size(0							), self.n_token)							)
																_UpperCamelCase      : Tuple         =   nn.functional.log_softmax(lowerCAmelCase__,			dim=1							)

																_UpperCamelCase      : List[str]         =   [0] + self.cutoffs
																for i in range(len(lowerCAmelCase__							) - 1							):
																						_UpperCamelCase       ,							_UpperCamelCase      : int         =   cutoff_values[i], cutoff_values[i + 1]

																						if i == 0:
																												_UpperCamelCase      : str         =   head_logprob[:, : self.cutoffs[0]]
																						else:
																												_UpperCamelCase       ,							_UpperCamelCase       ,							_UpperCamelCase      : Union[str, Any]         =   weights[i], biases[i], self.out_projs[i]

																												_UpperCamelCase      : List[Any]         =   self._compute_logit(lowerCAmelCase__,			lowerCAmelCase__,			lowerCAmelCase__,			lowerCAmelCase__							)
																												_UpperCamelCase      : Optional[Any]         =   nn.functional.log_softmax(lowerCAmelCase__,			dim=1							)

																												_UpperCamelCase      : Any         =   head_logprob[:, -i] + tail_logprob_i
																												_UpperCamelCase      : str         =   logprob_i

																return out










"""simple docstring"""




from functools import lru_cache





@lru_cache
def 	a_    (							_lowercase       ):
						if num < 0:
												raise ValueError('''Number should not be negative.'''       )

						return 1 if num in (0, 1) else num * factorial(num - 1       )


if __name__ == "__main__":
						import doctest

						doctest.testmod()



