
import torch
from torch import nn







class     lowercase		( nn.Module		):




							def __init__(    self       ,A__       ,A__       ,A__       ,A__       ,A__=1       ,A__=False):
												super().__init__()

												lowercase						     =						n_token
												lowercase						     =						d_embed
												lowercase						     =						d_proj

												lowercase						     =						cutoffs + [n_token]
												lowercase						     =						[0] + self.cutoffs
												lowercase						     =						div_val

												lowercase						     =						self.cutoffs[0]
												lowercase						     =						len(self.cutoffs) - 1
												lowercase						     =						self.shortlist_size + self.n_clusters

												if self.n_clusters > 0:
																	lowercase						     =						nn.Parameter(torch.zeros(self.n_clusters       ,self.d_embed))
																	lowercase						     =						nn.Parameter(torch.zeros(self.n_clusters))

												lowercase						     =						nn.ModuleList()
												lowercase						     =						nn.ParameterList()

												if div_val == 1:
																	for i in range(len(self.cutoffs)):
																						if d_proj != d_embed:
																											self.out_projs.append(nn.Parameter(torch.FloatTensor(A__       ,A__)))
																						else:
																											self.out_projs.append(A__)

																	self.out_layers.append(nn.Linear(A__       ,A__))
												else:
																	for i in range(len(self.cutoffs)):
																						lowercase     ,						lowercase						     =						self.cutoff_ends[i], self.cutoff_ends[i + 1]
																						lowercase						     =						d_embed // (div_val**i)

																						self.out_projs.append(nn.Parameter(torch.FloatTensor(A__       ,A__)))

																						self.out_layers.append(nn.Linear(A__       ,r_idx - l_idx))

												lowercase						     =						keep_order




							def 			A__				(    self       ,A__       ,A__       ,A__       ,A__):
												if proj is None:
																	lowercase						     =						nn.functional.linear(A__       ,A__       ,bias=A__)
												else:
																	# if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
																	lowercase						     =						nn.functional.linear(A__       ,proj.t().contiguous())
																	lowercase						     =						nn.functional.linear(A__       ,A__       ,bias=A__)
																	# else:
																	#     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
																	#     if bias is not None:
																	#         logit = logit + bias

												return logit




							def 			A__				(    self       ,A__       ,A__=None       ,A__=False):

												if labels is not None:
																	# Shift so that tokens < n predict n
																	lowercase						     =						hidden[..., :-1, :].contiguous()
																	lowercase						     =						labels[..., 1:].contiguous()
																	lowercase						     =						hidden.view(-1       ,hidden.size(-1))
																	lowercase						     =						labels.view(-1)
																	if hidden.size(0) != labels.size(0):
																						raise RuntimeError('''Input and labels should have the same size in the batch dimension.''')
												else:
																	lowercase						     =						hidden.view(-1       ,hidden.size(-1))

												if self.n_clusters == 0:
																	lowercase						     =						self._compute_logit(A__       ,self.out_layers[0].weight       ,self.out_layers[0].bias       ,self.out_projs[0])
																	if labels is not None:
																						lowercase						     =						labels != -1_0_0
																						lowercase						     =						torch.zeros_like(A__       ,dtype=hidden.dtype       ,device=hidden.device)
																						lowercase						     =						(
																						    -nn.functional.log_softmax(A__       ,dim=-1)[mask].gather(1       ,labels[mask].unsqueeze(1)).squeeze(1)
																						)
																	else:
																						lowercase						     =						nn.functional.log_softmax(A__       ,dim=-1)
												else:
																	# construct weights and biases
																	lowercase     ,						lowercase						     =						[], []
																	for i in range(len(self.cutoffs)):
																						if self.div_val == 1:
																											lowercase     ,						lowercase						     =						self.cutoff_ends[i], self.cutoff_ends[i + 1]
																											lowercase						     =						self.out_layers[0].weight[l_idx:r_idx]
																											lowercase						     =						self.out_layers[0].bias[l_idx:r_idx]
																						else:
																											lowercase						     =						self.out_layers[i].weight
																											lowercase						     =						self.out_layers[i].bias

																						if i == 0:
																											lowercase						     =						torch.cat([weight_i, self.cluster_weight]       ,dim=0)
																											lowercase						     =						torch.cat([bias_i, self.cluster_bias]       ,dim=0)

																						weights.append(A__)
																						biases.append(A__)

																	lowercase     ,						lowercase     ,						lowercase						     =						weights[0], biases[0], self.out_projs[0]

																	lowercase						     =						self._compute_logit(A__       ,A__       ,A__       ,A__)
																	lowercase						     =						nn.functional.log_softmax(A__       ,dim=1)

																	if labels is None:
																						lowercase						     =						hidden.new_empty((head_logit.size(0), self.n_token))
																	else:
																						lowercase						     =						torch.zeros_like(A__       ,dtype=hidden.dtype       ,device=hidden.device)

																	lowercase						     =						0
																	lowercase						     =						[0] + self.cutoffs
																	for i in range(len(A__) - 1):
																						lowercase     ,						lowercase						     =						cutoff_values[i], cutoff_values[i + 1]

																						if labels is not None:
																											lowercase						     =						(labels >= l_idx) & (labels < r_idx)
																											lowercase						     =						mask_i.nonzero().squeeze()

																											if indices_i.numel() == 0:
																																continue

																											lowercase						     =						labels.index_select(0       ,A__) - l_idx
																											lowercase						     =						head_logprob.index_select(0       ,A__)
																											lowercase						     =						hidden.index_select(0       ,A__)
																						else:
																											lowercase						     =						hidden

																						if i == 0:
																											if labels is not None:
																																lowercase						     =						head_logprob_i.gather(1       ,target_i[:, None]).squeeze(1)
																											else:
																																lowercase						     =						head_logprob[:, : self.cutoffs[0]]
																						else:
																											lowercase     ,						lowercase     ,						lowercase						     =						weights[i], biases[i], self.out_projs[i]

																											lowercase						     =						self._compute_logit(A__       ,A__       ,A__       ,A__)
																											lowercase						     =						nn.functional.log_softmax(A__       ,dim=1)
																											lowercase						     =						self.cutoffs[0] + i - 1  # No probability for the head cluster
																											if labels is not None:
																																lowercase						     =						head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
																																    1       ,target_i[:, None]).squeeze(1)
																											else:
																																lowercase						     =						head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
																																lowercase						     =						logprob_i

																						if labels is not None:
																											if (hasattr(self       ,'''keep_order''') and self.keep_order) or keep_order:
																																out.index_copy_(0       ,A__       ,-logprob_i)
																											else:
																																out[offset : offset + logprob_i.size(0)].copy_(-logprob_i)
																											offset += logprob_i.size(0)

												return out





							def 			A__				(    self       ,A__):
												if self.n_clusters == 0:
																	lowercase						     =						self._compute_logit(A__       ,self.out_layers[0].weight       ,self.out_layers[0].bias       ,self.out_projs[0])
																	return nn.functional.log_softmax(A__       ,dim=-1)
												else:
																	# construct weights and biases
																	lowercase     ,						lowercase						     =						[], []
																	for i in range(len(self.cutoffs)):
																						if self.div_val == 1:
																											lowercase     ,						lowercase						     =						self.cutoff_ends[i], self.cutoff_ends[i + 1]
																											lowercase						     =						self.out_layers[0].weight[l_idx:r_idx]
																											lowercase						     =						self.out_layers[0].bias[l_idx:r_idx]
																						else:
																											lowercase						     =						self.out_layers[i].weight
																											lowercase						     =						self.out_layers[i].bias

																						if i == 0:
																											lowercase						     =						torch.cat([weight_i, self.cluster_weight]       ,dim=0)
																											lowercase						     =						torch.cat([bias_i, self.cluster_bias]       ,dim=0)

																						weights.append(A__)
																						biases.append(A__)

																	lowercase     ,						lowercase     ,						lowercase						     =						weights[0], biases[0], self.out_projs[0]
																	lowercase						     =						self._compute_logit(A__       ,A__       ,A__       ,A__)

																	lowercase						     =						hidden.new_empty((head_logit.size(0), self.n_token))
																	lowercase						     =						nn.functional.log_softmax(A__       ,dim=1)

																	lowercase						     =						[0] + self.cutoffs
																	for i in range(len(A__) - 1):
																						lowercase     ,						lowercase						     =						cutoff_values[i], cutoff_values[i + 1]

																						if i == 0:
																											lowercase						     =						head_logprob[:, : self.cutoffs[0]]
																						else:
																											lowercase     ,						lowercase     ,						lowercase						     =						weights[i], biases[i], self.out_projs[i]

																											lowercase						     =						self._compute_logit(A__       ,A__       ,A__       ,A__)
																											lowercase						     =						nn.functional.log_softmax(A__       ,dim=1)

																											lowercase						     =						head_logprob[:, -i] + tail_logprob_i
																											lowercase						     =						logprob_i

																	return out






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_torch_available,
    is_vision_available,
)


lowercase__		:str			  =   {"configuration_vit": ["VIT_PRETRAINED_CONFIG_ARCHIVE_MAP", "ViTConfig", "ViTOnnxConfig"]}

try:
							if not is_vision_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							lowercase__		:str			  =   ["ViTFeatureExtractor"]
							lowercase__		:int			  =   ["ViTImageProcessor"]

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							lowercase__		:int			  =   [
							    "VIT_PRETRAINED_MODEL_ARCHIVE_LIST",
							    "ViTForImageClassification",
							    "ViTForMaskedImageModeling",
							    "ViTModel",
							    "ViTPreTrainedModel",
							]

try:
							if not is_tf_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							lowercase__		:Any			  =   [
							    "TFViTForImageClassification",
							    "TFViTModel",
							    "TFViTPreTrainedModel",
							]

try:
							if not is_flax_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							lowercase__		:Optional[int]			  =   [
							    "FlaxViTForImageClassification",
							    "FlaxViTModel",
							    "FlaxViTPreTrainedModel",
							]

if TYPE_CHECKING:
							from .configuration_vit import VIT_PRETRAINED_CONFIG_ARCHIVE_MAP, ViTConfig, ViTOnnxConfig

							try:
														if not is_vision_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .feature_extraction_vit import ViTFeatureExtractor
														from .image_processing_vit import ViTImageProcessor

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_vit import (
														    VIT_PRETRAINED_MODEL_ARCHIVE_LIST,
														    ViTForImageClassification,
														    ViTForMaskedImageModeling,
														    ViTModel,
														    ViTPreTrainedModel,
														)

							try:
														if not is_tf_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_tf_vit import TFViTForImageClassification, TFViTModel, TFViTPreTrainedModel

							try:
														if not is_flax_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_flax_vit import FlaxViTForImageClassification, FlaxViTModel, FlaxViTPreTrainedModel


else:
							import sys

							lowercase__		:Dict			  =   _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)




