

'''simple docstring'''



import inspect
import unittest

from transformers import ConvNextConfig
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_backbone_common import BackboneTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import ConvNextBackbone, ConvNextForImageClassification, ConvNextModel
		from transformers.models.convnext.modeling_convnext import CONVNEXT_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
		from PIL import Image

		from transformers import AutoImageProcessor



class lowerCamelCase			:



			'''simple docstring'''






			def __init__(				self	:						List[Any]		,      lowerCAmelCase_	:						str		,      lowerCAmelCase_	:						str=13		,      lowerCAmelCase_	:						Optional[Any]=32		,      lowerCAmelCase_	:						Dict=3		,      lowerCAmelCase_	:						List[Any]=4		,      lowerCAmelCase_	:						List[str]=[10, 20, 30, 40]		,      lowerCAmelCase_	:						Any=[2, 2, 3, 2]		,      lowerCAmelCase_	:						Union[str, Any]=True		,      lowerCAmelCase_	:						int=True		,      lowerCAmelCase_	:						List[str]=37		,      lowerCAmelCase_	:						str="gelu"		,      lowerCAmelCase_	:						int=10		,      lowerCAmelCase_	:						Dict=0.02		,      lowerCAmelCase_	:						Optional[int]=["stage2", "stage3", "stage4"]		,      lowerCAmelCase_	:						Union[str, Any]=[2, 3, 4]		,      lowerCAmelCase_	:						List[str]=None		,      )      ->		Optional[int]:





									'''simple docstring'''
									A__					:       List[str]     =parent
									A__					:       str     =batch_size
									A__					:       List[str]     =image_size
									A__					:       Union[str, Any]     =num_channels
									A__					:       str     =num_stages
									A__					:       int     =hidden_sizes
									A__					:       Optional[int]     =depths
									A__					:       Any     =is_training
									A__					:       Any     =use_labels
									A__					:       List[str]     =intermediate_size
									A__					:       str     =hidden_act
									A__					:       Tuple     =num_labels
									A__					:       Optional[int]     =initializer_range
									A__					:       Dict     =out_features
									A__					:       List[Any]     =out_indices
									A__					:       List[Any]     =scope


			def lowercase__						(				self	:						Optional[Any]			)      ->		Any:





									'''simple docstring'''
									A__					:       Dict     =floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]			)

									A__					:       str     =None
									if self.use_labels:
															A__					:       Tuple     =ids_tensor([self.batch_size]		,      self.num_labels			)

									A__					:       Dict     =self.get_config()
									return config, pixel_values, labels


			def lowercase__						(				self	:						str			)      ->		List[str]:





									'''simple docstring'''
									return ConvNextConfig(
									    num_channels=self.num_channels		,      hidden_sizes=self.hidden_sizes		,      depths=self.depths		,      num_stages=self.num_stages		,      hidden_act=self.hidden_act		,      is_decoder=lowerCAmelCase_		,      initializer_range=self.initializer_range		,      out_features=self.out_features		,      out_indices=self.out_indices		,      num_labels=self.num_labels		,      )


			def lowercase__						(				self	:						Tuple		,      lowerCAmelCase_	:						str		,      lowerCAmelCase_	:						Dict		,      lowerCAmelCase_	:						Union[str, Any]			)      ->		List[Any]:





									'''simple docstring'''
									A__					:       Union[str, Any]     =ConvNextModel(config=lowerCAmelCase_			)
									model.to(lowerCAmelCase_			)
									model.eval()
									A__					:       Dict     =model(lowerCAmelCase_			)
									# expected last hidden states: B, C, H // 32, W // 32
									self.parent.assertEqual(
									    result.last_hidden_state.shape		,      (self.batch_size, self.hidden_sizes[-1], self.image_size // 32, self.image_size // 32)		,      )


			def lowercase__						(				self	:						Any		,      lowerCAmelCase_	:						List[str]		,      lowerCAmelCase_	:						str		,      lowerCAmelCase_	:						Dict			)      ->		str:





									'''simple docstring'''
									A__					:       Dict     =ConvNextForImageClassification(lowerCAmelCase_			)
									model.to(lowerCAmelCase_			)
									model.eval()
									A__					:       Optional[int]     =model(lowerCAmelCase_		,      labels=lowerCAmelCase_			)
									self.parent.assertEqual(result.logits.shape		,      (self.batch_size, self.num_labels)			)


			def lowercase__						(				self	:						Optional[int]		,      lowerCAmelCase_	:						Any		,      lowerCAmelCase_	:						List[Any]		,      lowerCAmelCase_	:						Optional[Any]			)      ->		Any:





									'''simple docstring'''
									A__					:       int     =ConvNextBackbone(config=lowerCAmelCase_			)
									model.to(lowerCAmelCase_			)
									model.eval()
									A__					:       Tuple     =model(lowerCAmelCase_			)

									# verify hidden states
									self.parent.assertEqual(len(result.feature_maps			)		,      len(config.out_features			)			)
									self.parent.assertListEqual(list(result.feature_maps[0].shape			)		,      [self.batch_size, self.hidden_sizes[1], 4, 4]			)

									# verify channels
									self.parent.assertEqual(len(model.channels			)		,      len(config.out_features			)			)
									self.parent.assertListEqual(model.channels		,      config.hidden_sizes[1:]			)

									# verify backbone works with out_features=None
									A__					:       Tuple     =None
									A__					:       List[str]     =ConvNextBackbone(config=lowerCAmelCase_			)
									model.to(lowerCAmelCase_			)
									model.eval()
									A__					:       Optional[int]     =model(lowerCAmelCase_			)

									# verify feature maps
									self.parent.assertEqual(len(result.feature_maps			)		,      1			)
									self.parent.assertListEqual(list(result.feature_maps[0].shape			)		,      [self.batch_size, self.hidden_sizes[-1], 1, 1]			)

									# verify channels
									self.parent.assertEqual(len(model.channels			)		,      1			)
									self.parent.assertListEqual(model.channels		,      [config.hidden_sizes[-1]]			)



			def lowercase__						(				self	:						List[Any]			)      ->		Optional[Any]:





									'''simple docstring'''
									A__					:       List[str]     =self.prepare_config_and_inputs()
									A__   ,						A__   ,						A__					:       Any     =config_and_inputs
									A__					:       Union[str, Any]     ={"""pixel_values""": pixel_values}
									return config, inputs_dict



@require_torch
class lowerCamelCase			(  lowercase_	,						lowercase_	,						unittest.TestCase						):



			'''simple docstring'''






			__snake_case        =	(
			    (
			        ConvNextModel,
			        ConvNextForImageClassification,
			        ConvNextBackbone,
			    )
			    if is_torch_available()
			    else ()
			)
			__snake_case        =	(
			    {'feature-extraction': ConvNextModel, 'image-classification': ConvNextForImageClassification}
			    if is_torch_available()
			    else {}
			)

			__snake_case        =	True
			__snake_case        =	False
			__snake_case        =	False
			__snake_case        =	False
			__snake_case        =	False


			def lowercase__						(				self	:						Tuple			)      ->		str:





									'''simple docstring'''
									A__					:       Dict     =ConvNextModelTester(self			)
									A__					:       Tuple     =ConfigTester(self		,      config_class=lowerCAmelCase_		,      has_text_modality=lowerCAmelCase_		,      hidden_size=37			)


			def lowercase__						(				self	:						str			)      ->		List[str]:





									'''simple docstring'''
									self.create_and_test_config_common_properties()
									self.config_tester.create_and_test_config_to_json_string()
									self.config_tester.create_and_test_config_to_json_file()
									self.config_tester.create_and_test_config_from_and_save_pretrained()
									self.config_tester.create_and_test_config_with_num_labels()
									self.config_tester.check_config_can_be_init_without_params()
									self.config_tester.check_config_arguments_init()


			def lowercase__						(				self	:						List[str]			)      ->		str:





									'''simple docstring'''
									return


			@unittest.skip(reason="""ConvNext does not use inputs_embeds"""			)
			def lowercase__						(				self	:						int			)      ->		int:





									'''simple docstring'''
									pass


			@unittest.skip(reason="""ConvNext does not support input and output embeddings"""			)
			def lowercase__						(				self	:						Union[str, Any]			)      ->		Optional[int]:





									'''simple docstring'''
									pass


			@unittest.skip(reason="""ConvNext does not use feedforward chunking"""			)
			def lowercase__						(				self	:						Dict			)      ->		Optional[Any]:





									'''simple docstring'''
									pass


			def lowercase__						(				self	:						int			)      ->		Optional[int]:





									'''simple docstring'''
									A__   ,						A__					:       List[Any]     =self.model_tester.prepare_config_and_inputs_for_common()

									for model_class in self.all_model_classes:
															A__					:       Optional[Any]     =model_class(lowerCAmelCase_			)
															A__					:       Union[str, Any]     =inspect.signature(model.forward			)
															# signature.parameters is an OrderedDict => so arg_names order is deterministic
															A__					:       List[str]     =[*signature.parameters.keys()]

															A__					:       List[str]     =["""pixel_values"""]
															self.assertListEqual(arg_names[:1]		,      lowerCAmelCase_			)


			def lowercase__						(				self	:						Tuple			)      ->		Dict:





									'''simple docstring'''
									A__					:       Optional[Any]     =self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_model(*lowerCAmelCase_			)


			def lowercase__						(				self	:						Tuple			)      ->		Dict:





									'''simple docstring'''
									A__					:       Dict     =self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_backbone(*lowerCAmelCase_			)


			def lowercase__						(				self	:						Optional[Any]			)      ->		str:





									'''simple docstring'''
									def check_hidden_states_output(lowerCAmelCase_	:						str		,      lowerCAmelCase_	:						Tuple		,      lowerCAmelCase_	:						str			):
															A__					:       int     =model_class(lowerCAmelCase_			)
															model.to(lowerCAmelCase_			)
															model.eval()

															with torch.no_grad():
																					A__					:       Optional[int]     =model(**self._prepare_for_class(lowerCAmelCase_		,      lowerCAmelCase_			)			)

															A__					:       str     =outputs.encoder_hidden_states if config.is_encoder_decoder else outputs.hidden_states

															A__					:       Union[str, Any]     =self.model_tester.num_stages
															self.assertEqual(len(lowerCAmelCase_			)		,      expected_num_stages + 1			)

															# ConvNext's feature maps are of shape (batch_size, num_channels, height, width)
															self.assertListEqual(
															    list(hidden_states[0].shape[-2:]			)		,      [self.model_tester.image_size // 4, self.model_tester.image_size // 4]		,      )

									A__   ,						A__					:       int     =self.model_tester.prepare_config_and_inputs_for_common()

									for model_class in self.all_model_classes:
															A__					:       Optional[int]     =True
															check_hidden_states_output(lowerCAmelCase_		,      lowerCAmelCase_		,      lowerCAmelCase_			)

															# check that output_hidden_states also work using config
															del inputs_dict["output_hidden_states"]
															A__					:       Optional[Any]     =True

															check_hidden_states_output(lowerCAmelCase_		,      lowerCAmelCase_		,      lowerCAmelCase_			)


			def lowercase__						(				self	:						Union[str, Any]			)      ->		List[Any]:





									'''simple docstring'''
									A__					:       int     =self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_image_classification(*lowerCAmelCase_			)



			@slow
			def lowercase__						(				self	:						Tuple			)      ->		str:





									'''simple docstring'''
									for model_name in CONVNEXT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
															A__					:       List[str]     =ConvNextModel.from_pretrained(lowerCAmelCase_			)
															self.assertIsNotNone(lowerCAmelCase_			)


def 						__lowerCamelCase	(       )	->							Optional[Any]:






						"""simple docstring"""





						A__					:       List[str]     =Image.open("""./tests/fixtures/tests_samples/COCO/000000039769.png"""	)
						return image



@require_torch
@require_vision
class lowerCamelCase			(  unittest.TestCase						):



			'''simple docstring'''






			@cached_property
			def lowercase__						(				self	:						Any			)      ->		Optional[Any]:





									'''simple docstring'''
									return AutoImageProcessor.from_pretrained("""facebook/convnext-tiny-224"""			) if is_vision_available() else None



			@slow
			def lowercase__						(				self	:						Optional[Any]			)      ->		Optional[Any]:





									'''simple docstring'''
									A__					:       Optional[Any]     =ConvNextForImageClassification.from_pretrained("""facebook/convnext-tiny-224"""			).to(lowerCAmelCase_			)

									A__					:       int     =self.default_image_processor
									A__					:       int     =prepare_img()
									A__					:       Optional[Any]     =image_processor(images=lowerCAmelCase_		,      return_tensors="""pt"""			).to(lowerCAmelCase_			)

									# forward pass
									with torch.no_grad():
															A__					:       int     =model(**lowerCAmelCase_			)

									# verify the logits
									A__					:       str     =torch.Size((1, 10_00)			)
									self.assertEqual(outputs.logits.shape		,      lowerCAmelCase_			)

									A__					:       Optional[int]     =torch.tensor([-0.0260, -0.4739, 0.1911]			).to(lowerCAmelCase_			)

									self.assertTrue(torch.allclose(outputs.logits[0, :3]		,      lowerCAmelCase_		,      atol=1e-4			)			)



@require_torch
class lowerCamelCase			(  unittest.TestCase	,						lowercase_						):



			'''simple docstring'''






			__snake_case        =	(ConvNextBackbone,) if is_torch_available() else ()
			__snake_case        =	ConvNextConfig

			__snake_case        =	False


			def lowercase__						(				self	:						int			)      ->		str:





									'''simple docstring'''
									A__					:       Union[str, Any]     =ConvNextModelTester(self			)




'''simple docstring'''



import torch
from torch import nn



class lowerCamelCase			(  nn.Module						):



			'''simple docstring'''






			def __init__(				self	:						Any		,      lowerCAmelCase_	:						List[str]		,      lowerCAmelCase_	:						List[Any]		,      lowerCAmelCase_	:						Dict		,      lowerCAmelCase_	:						str		,      lowerCAmelCase_	:						Optional[int]=1		,      lowerCAmelCase_	:						str=False			)      ->		List[str]:





									'''simple docstring'''
									super().__init__()

									A__					:       Any     =n_token
									A__					:       int     =d_embed
									A__					:       Any     =d_proj

									A__					:       Tuple     =cutoffs + [n_token]
									A__					:       Optional[Any]     =[0] + self.cutoffs
									A__					:       Dict     =div_val

									A__					:       str     =self.cutoffs[0]
									A__					:       Optional[Any]     =len(self.cutoffs			) - 1
									A__					:       List[Any]     =self.shortlist_size + self.n_clusters

									if self.n_clusters > 0:
															A__					:       Any     =nn.Parameter(torch.zeros(self.n_clusters		,      self.d_embed			)			)
															A__					:       str     =nn.Parameter(torch.zeros(self.n_clusters			)			)

									A__					:       Union[str, Any]     =nn.ModuleList()
									A__					:       Optional[int]     =nn.ParameterList()

									if div_val == 1:
															for i in range(len(self.cutoffs			)			):
																					if d_proj != d_embed:
																											self.out_projs.append(nn.Parameter(torch.FloatTensor(lowerCAmelCase_		,      lowerCAmelCase_			)			)			)
																					else:
																											self.out_projs.append(lowerCAmelCase_			)

															self.out_layers.append(nn.Linear(lowerCAmelCase_		,      lowerCAmelCase_			)			)
									else:
															for i in range(len(self.cutoffs			)			):
																					A__   ,						A__					:       Optional[int]     =self.cutoff_ends[i], self.cutoff_ends[i + 1]
																					A__					:       Tuple     =d_embed // (div_val**i)

																					self.out_projs.append(nn.Parameter(torch.FloatTensor(lowerCAmelCase_		,      lowerCAmelCase_			)			)			)

																					self.out_layers.append(nn.Linear(lowerCAmelCase_		,      r_idx - l_idx			)			)

									A__					:       Optional[int]     =keep_order


			def lowercase__						(				self	:						Dict		,      lowerCAmelCase_	:						int		,      lowerCAmelCase_	:						Tuple		,      lowerCAmelCase_	:						Union[str, Any]		,      lowerCAmelCase_	:						Any			)      ->		Union[str, Any]:





									'''simple docstring'''
									if proj is None:
															A__					:       Optional[int]     =nn.functional.linear(lowerCAmelCase_		,      lowerCAmelCase_		,      bias=lowerCAmelCase_			)
									else:
															# if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
															A__					:       Optional[int]     =nn.functional.linear(lowerCAmelCase_		,      proj.t().contiguous()			)
															A__					:       Union[str, Any]     =nn.functional.linear(lowerCAmelCase_		,      lowerCAmelCase_		,      bias=lowerCAmelCase_			)
															# else:
															#     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
															#     if bias is not None:
															#         logit = logit + bias

									return logit


			def lowercase__						(				self	:						Tuple		,      lowerCAmelCase_	:						List[str]		,      lowerCAmelCase_	:						List[Any]=None		,      lowerCAmelCase_	:						Dict=False			)      ->		Optional[int]:





									'''simple docstring'''

									if labels is not None:
															# Shift so that tokens < n predict n
															A__					:       Optional[Any]     =hidden[..., :-1, :].contiguous()
															A__					:       List[Any]     =labels[..., 1:].contiguous()
															A__					:       Optional[int]     =hidden.view(-1		,      hidden.size(-1			)			)
															A__					:       str     =labels.view(-1			)
															if hidden.size(0			) != labels.size(0			):
																					raise RuntimeError("""Input and labels should have the same size in the batch dimension."""			)
									else:
															A__					:       Optional[int]     =hidden.view(-1		,      hidden.size(-1			)			)

									if self.n_clusters == 0:
															A__					:       Optional[Any]     =self._compute_logit(lowerCAmelCase_		,      self.out_layers[0].weight		,      self.out_layers[0].bias		,      self.out_projs[0]			)
															if labels is not None:
																					A__					:       Tuple     =labels != -1_00
																					A__					:       int     =torch.zeros_like(lowerCAmelCase_		,      dtype=hidden.dtype		,      device=hidden.device			)
																					A__					:       Union[str, Any]     =(
																					    -nn.functional.log_softmax(lowerCAmelCase_		,      dim=-1			)[mask].gather(1		,      labels[mask].unsqueeze(1			)			).squeeze(1			)
																					)
															else:
																					A__					:       List[Any]     =nn.functional.log_softmax(lowerCAmelCase_		,      dim=-1			)
									else:
															# construct weights and biases
															A__   ,						A__					:       Any     =[], []
															for i in range(len(self.cutoffs			)			):
																					if self.div_val == 1:
																											A__   ,						A__					:       Optional[int]     =self.cutoff_ends[i], self.cutoff_ends[i + 1]
																											A__					:       int     =self.out_layers[0].weight[l_idx:r_idx]
																											A__					:       List[str]     =self.out_layers[0].bias[l_idx:r_idx]
																					else:
																											A__					:       List[str]     =self.out_layers[i].weight
																											A__					:       Union[str, Any]     =self.out_layers[i].bias

																					if i == 0:
																											A__					:       Tuple     =torch.cat([weight_i, self.cluster_weight]		,      dim=0			)
																											A__					:       List[str]     =torch.cat([bias_i, self.cluster_bias]		,      dim=0			)

																					weights.append(lowerCAmelCase_			)
																					biases.append(lowerCAmelCase_			)

															A__   ,						A__   ,						A__					:       Tuple     =weights[0], biases[0], self.out_projs[0]

															A__					:       List[Any]     =self._compute_logit(lowerCAmelCase_		,      lowerCAmelCase_		,      lowerCAmelCase_		,      lowerCAmelCase_			)
															A__					:       int     =nn.functional.log_softmax(lowerCAmelCase_		,      dim=1			)

															if labels is None:
																					A__					:       Union[str, Any]     =hidden.new_empty((head_logit.size(0			), self.n_token)			)
															else:
																					A__					:       Union[str, Any]     =torch.zeros_like(lowerCAmelCase_		,      dtype=hidden.dtype		,      device=hidden.device			)

															A__					:       Any     =0
															A__					:       Tuple     =[0] + self.cutoffs
															for i in range(len(lowerCAmelCase_			) - 1			):
																					A__   ,						A__					:       Tuple     =cutoff_values[i], cutoff_values[i + 1]

																					if labels is not None:
																											A__					:       Tuple     =(labels >= l_idx) & (labels < r_idx)
																											A__					:       Any     =mask_i.nonzero().squeeze()

																											if indices_i.numel() == 0:
																																	continue

																											A__					:       int     =labels.index_select(0		,      lowerCAmelCase_			) - l_idx
																											A__					:       List[str]     =head_logprob.index_select(0		,      lowerCAmelCase_			)
																											A__					:       str     =hidden.index_select(0		,      lowerCAmelCase_			)
																					else:
																											A__					:       Optional[Any]     =hidden

																					if i == 0:
																											if labels is not None:
																																	A__					:       Optional[Any]     =head_logprob_i.gather(1		,      target_i[:, None]			).squeeze(1			)
																											else:
																																	A__					:       Union[str, Any]     =head_logprob[:, : self.cutoffs[0]]
																					else:
																											A__   ,						A__   ,						A__					:       Dict     =weights[i], biases[i], self.out_projs[i]

																											A__					:       List[Any]     =self._compute_logit(lowerCAmelCase_		,      lowerCAmelCase_		,      lowerCAmelCase_		,      lowerCAmelCase_			)
																											A__					:       List[Any]     =nn.functional.log_softmax(lowerCAmelCase_		,      dim=1			)
																											A__					:       Optional[Any]     =self.cutoffs[0] + i - 1  # No probability for the head cluster
																											if labels is not None:
																																	A__					:       Union[str, Any]     =head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
																																	    1		,      target_i[:, None]			).squeeze(1			)
																											else:
																																	A__					:       List[str]     =head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
																																	A__					:       Tuple     =logprob_i

																					if labels is not None:
																											if (hasattr(self		,      """keep_order"""			) and self.keep_order) or keep_order:
																																	out.index_copy_(0		,      lowerCAmelCase_		,      -logprob_i			)
																											else:
																																	out[offset : offset + logprob_i.size(0			)].copy_(-logprob_i			)
																											offset += logprob_i.size(0			)

									return out



			def lowercase__						(				self	:						List[str]		,      lowerCAmelCase_	:						Optional[Any]			)      ->		Any:





									'''simple docstring'''
									if self.n_clusters == 0:
															A__					:       List[str]     =self._compute_logit(lowerCAmelCase_		,      self.out_layers[0].weight		,      self.out_layers[0].bias		,      self.out_projs[0]			)
															return nn.functional.log_softmax(lowerCAmelCase_		,      dim=-1			)
									else:
															# construct weights and biases
															A__   ,						A__					:       List[str]     =[], []
															for i in range(len(self.cutoffs			)			):
																					if self.div_val == 1:
																											A__   ,						A__					:       int     =self.cutoff_ends[i], self.cutoff_ends[i + 1]
																											A__					:       List[str]     =self.out_layers[0].weight[l_idx:r_idx]
																											A__					:       List[Any]     =self.out_layers[0].bias[l_idx:r_idx]
																					else:
																											A__					:       Dict     =self.out_layers[i].weight
																											A__					:       Any     =self.out_layers[i].bias

																					if i == 0:
																											A__					:       List[str]     =torch.cat([weight_i, self.cluster_weight]		,      dim=0			)
																											A__					:       Tuple     =torch.cat([bias_i, self.cluster_bias]		,      dim=0			)

																					weights.append(lowerCAmelCase_			)
																					biases.append(lowerCAmelCase_			)

															A__   ,						A__   ,						A__					:       Optional[int]     =weights[0], biases[0], self.out_projs[0]
															A__					:       Any     =self._compute_logit(lowerCAmelCase_		,      lowerCAmelCase_		,      lowerCAmelCase_		,      lowerCAmelCase_			)

															A__					:       Dict     =hidden.new_empty((head_logit.size(0			), self.n_token)			)
															A__					:       Dict     =nn.functional.log_softmax(lowerCAmelCase_		,      dim=1			)

															A__					:       Tuple     =[0] + self.cutoffs
															for i in range(len(lowerCAmelCase_			) - 1			):
																					A__   ,						A__					:       List[Any]     =cutoff_values[i], cutoff_values[i + 1]

																					if i == 0:
																											A__					:       Tuple     =head_logprob[:, : self.cutoffs[0]]
																					else:
																											A__   ,						A__   ,						A__					:       Any     =weights[i], biases[i], self.out_projs[i]

																											A__					:       Dict     =self._compute_logit(lowerCAmelCase_		,      lowerCAmelCase_		,      lowerCAmelCase_		,      lowerCAmelCase_			)
																											A__					:       str     =nn.functional.log_softmax(lowerCAmelCase_		,      dim=1			)

																											A__					:       str     =head_logprob[:, -i] + tail_logprob_i
																											A__					:       List[Any]     =logprob_i

															return out

