




"""simple docstring"""


import gc
import random
import tempfile
import unittest

import numpy as np
import torch
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import AutoencoderKL, DDIMScheduler, LMSDiscreteScheduler, PNDMScheduler, UNetaDConditionModel
from diffusers.pipelines.stable_diffusion_safe import StableDiffusionPipelineSafe as StableDiffusionPipeline
from diffusers.utils import floats_tensor, nightly, torch_device
from diffusers.utils.testing_utils import require_torch_gpu






class  lowercase(					unittest.TestCase			):







	'''simple docstring'''





	def 			UpperCamelCase_       (    self:		Optional[int]		):




			'''simple docstring'''



			super().tearDown()
			gc.collect()
			torch.cuda.empty_cache()


	@property
	def 			UpperCamelCase_       (    self:		Tuple		):




			'''simple docstring'''



			_snake_case							:		Dict								=  1
			_snake_case							:		Optional[int]								=  3
			_snake_case							:		str								=  (32, 32)

			_snake_case							:		Optional[int]								=  floats_tensor((batch_size, num_channels) + sizes,				rng=random.Random(0		)		).to(a_		)
			return image


	@property
	def 			UpperCamelCase_       (    self:		List[str]		):




			'''simple docstring'''



			torch.manual_seed(0		)
			_snake_case							:		str								=  UNetaDConditionModel(
			    block_out_channels=(32, 64),				layers_per_block=2,				sample_size=32,				in_channels=4,				out_channels=4,				down_block_types=("""DownBlock2D""", """CrossAttnDownBlock2D"""),				up_block_types=("""CrossAttnUpBlock2D""", """UpBlock2D"""),				cross_attention_dim=32,				)
			return model


	@property
	def 			UpperCamelCase_       (    self:		Tuple		):




			'''simple docstring'''



			torch.manual_seed(0		)
			_snake_case							:		Union[str, Any]								=  AutoencoderKL(
			    block_out_channels=[32, 64],				in_channels=3,				out_channels=3,				down_block_types=["""DownEncoderBlock2D""", """DownEncoderBlock2D"""],				up_block_types=["""UpDecoderBlock2D""", """UpDecoderBlock2D"""],				latent_channels=4,				)
			return model


	@property
	def 			UpperCamelCase_       (    self:		List[Any]		):




			'''simple docstring'''



			torch.manual_seed(0		)
			_snake_case							:		List[Any]								=  CLIPTextConfig(
			    bos_token_id=0,				eos_token_id=2,				hidden_size=32,				intermediate_size=37,				layer_norm_eps=1E-05,				num_attention_heads=4,				num_hidden_layers=5,				pad_token_id=1,				vocab_size=1_000,				)
			return CLIPTextModel(a_		)


	@property
	def 			UpperCamelCase_       (    self:		List[Any]		):




			'''simple docstring'''



			def extract(*a_:		List[Any],				**a_:		Optional[int]		):






					class  lowercase:







						'''simple docstring'''





						def __init__(    self:		Dict		):




								'''simple docstring'''



								_snake_case							:		Any								=  torch.ones([0]		)


						def 			UpperCamelCase_       (    self:		Any,				a_:		List[Any]		):




								'''simple docstring'''



								self.pixel_values.to(a_		)
								return self







					return Out()

			return extract


	def 			UpperCamelCase_       (    self:		List[Any]		):




			'''simple docstring'''



			_snake_case							:		int								=  'cpu'  # ensure determinism for the device-dependent torch.Generator
			_snake_case							:		int								=  self.dummy_cond_unet
			_snake_case							:		Union[str, Any]								=  DDIMScheduler(
			    beta_start=0.00_085,				beta_end=0.012,				beta_schedule="""scaled_linear""",				clip_sample=a_,				set_alpha_to_one=a_,				)

			_snake_case							:		List[Any]								=  self.dummy_vae
			_snake_case							:		int								=  self.dummy_text_encoder
			_snake_case							:		Tuple								=  CLIPTokenizer.from_pretrained("""hf-internal-testing/tiny-random-clip"""		)

			# make sure here that pndm scheduler skips prk
			_snake_case							:		str								=  StableDiffusionPipeline(
			    unet=a_,				scheduler=a_,				vae=a_,				text_encoder=a_,				tokenizer=a_,				safety_checker=a_,				feature_extractor=self.dummy_extractor,				)
			_snake_case							:		Dict								=  sd_pipe.to(a_		)
			sd_pipe.set_progress_bar_config(disable=a_		)

			_snake_case							:		List[str]								=  'A painting of a squirrel eating a burger'

			_snake_case							:		str								=  torch.Generator(device=a_		).manual_seed(0		)
			_snake_case							:		Dict								=  sd_pipe([prompt],				generator=a_,				guidance_scale=6.0,				num_inference_steps=2,				output_type="""np"""		)
			_snake_case							:		List[str]								=  output.images

			_snake_case							:		Optional[Any]								=  torch.Generator(device=a_		).manual_seed(0		)
			_snake_case							:		Union[str, Any]								=  sd_pipe(
			    [prompt],				generator=a_,				guidance_scale=6.0,				num_inference_steps=2,				output_type="""np""",				return_dict=a_,				)[0]

			_snake_case							:		str								=  image[0, -3:, -3:, -1]
			_snake_case							:		Optional[int]								=  image_from_tuple[0, -3:, -3:, -1]

			assert image.shape == (1, 64, 64, 3)
			_snake_case							:		str								=  np.array([0.5_756, 0.6_118, 0.5_005, 0.5_041, 0.5_471, 0.4_726, 0.4_976, 0.4_865, 0.4_864]		)

			assert np.abs(image_slice.flatten() - expected_slice		).max() < 1E-2
			assert np.abs(image_from_tuple_slice.flatten() - expected_slice		).max() < 1E-2


	def 			UpperCamelCase_       (    self:		List[Any]		):




			'''simple docstring'''



			_snake_case							:		List[Any]								=  'cpu'  # ensure determinism for the device-dependent torch.Generator
			_snake_case							:		List[Any]								=  self.dummy_cond_unet
			_snake_case							:		Tuple								=  PNDMScheduler(skip_prk_steps=a_		)
			_snake_case							:		List[str]								=  self.dummy_vae
			_snake_case							:		List[str]								=  self.dummy_text_encoder
			_snake_case							:		Union[str, Any]								=  CLIPTokenizer.from_pretrained("""hf-internal-testing/tiny-random-clip"""		)

			# make sure here that pndm scheduler skips prk
			_snake_case							:		str								=  StableDiffusionPipeline(
			    unet=a_,				scheduler=a_,				vae=a_,				text_encoder=a_,				tokenizer=a_,				safety_checker=a_,				feature_extractor=self.dummy_extractor,				)
			_snake_case							:		Optional[int]								=  sd_pipe.to(a_		)
			sd_pipe.set_progress_bar_config(disable=a_		)

			_snake_case							:		Tuple								=  'A painting of a squirrel eating a burger'
			_snake_case							:		int								=  torch.Generator(device=a_		).manual_seed(0		)
			_snake_case							:		List[Any]								=  sd_pipe([prompt],				generator=a_,				guidance_scale=6.0,				num_inference_steps=2,				output_type="""np"""		)

			_snake_case							:		Union[str, Any]								=  output.images

			_snake_case							:		str								=  torch.Generator(device=a_		).manual_seed(0		)
			_snake_case							:		Any								=  sd_pipe(
			    [prompt],				generator=a_,				guidance_scale=6.0,				num_inference_steps=2,				output_type="""np""",				return_dict=a_,				)[0]

			_snake_case							:		List[Any]								=  image[0, -3:, -3:, -1]
			_snake_case							:		List[str]								=  image_from_tuple[0, -3:, -3:, -1]

			assert image.shape == (1, 64, 64, 3)
			_snake_case							:		str								=  np.array([0.5_125, 0.5_716, 0.4_828, 0.5_060, 0.5_650, 0.4_768, 0.5_185, 0.4_895, 0.4_993]		)

			assert np.abs(image_slice.flatten() - expected_slice		).max() < 1E-2
			assert np.abs(image_from_tuple_slice.flatten() - expected_slice		).max() < 1E-2


	def 			UpperCamelCase_       (    self:		Tuple		):




			'''simple docstring'''



			_snake_case							:		Union[str, Any]								=  StableDiffusionPipeline.from_pretrained(
			    """hf-internal-testing/tiny-stable-diffusion-lms-pipe""",				safety_checker=a_		)
			assert isinstance(a_,				a_		)
			assert isinstance(pipe.scheduler,				a_		)
			assert pipe.safety_checker is None

			_snake_case							:		Any								=  pipe("""example prompt""",				num_inference_steps=2		).images[0]
			assert image is not None

			# check that there's no error when saving a pipeline with one of the models being None
			with tempfile.TemporaryDirectory() as tmpdirname:
					pipe.save_pretrained(a_		)
					_snake_case							:		Dict								=  StableDiffusionPipeline.from_pretrained(a_		)

			# sanity check that the pipeline still works
			assert pipe.safety_checker is None
			_snake_case							:		Tuple								=  pipe("""example prompt""",				num_inference_steps=2		).images[0]
			assert image is not None


	@unittest.skipIf(torch_device != """cuda""",				"""This test requires a GPU"""		)
	def 			UpperCamelCase_       (    self:		List[Any]		):




			'''simple docstring'''



			_snake_case							:		int								=  self.dummy_cond_unet
			_snake_case							:		Optional[int]								=  PNDMScheduler(skip_prk_steps=a_		)
			_snake_case							:		Any								=  self.dummy_vae
			_snake_case							:		Union[str, Any]								=  self.dummy_text_encoder
			_snake_case							:		Tuple								=  CLIPTokenizer.from_pretrained("""hf-internal-testing/tiny-random-clip"""		)

			# put models in fp16
			_snake_case							:		Optional[Any]								=  unet.half()
			_snake_case							:		Any								=  vae.half()
			_snake_case							:		Tuple								=  bert.half()

			# make sure here that pndm scheduler skips prk
			_snake_case							:		Tuple								=  StableDiffusionPipeline(
			    unet=a_,				scheduler=a_,				vae=a_,				text_encoder=a_,				tokenizer=a_,				safety_checker=a_,				feature_extractor=self.dummy_extractor,				)
			_snake_case							:		Optional[Any]								=  sd_pipe.to(a_		)
			sd_pipe.set_progress_bar_config(disable=a_		)

			_snake_case							:		Any								=  'A painting of a squirrel eating a burger'
			_snake_case							:		List[Any]								=  sd_pipe([prompt],				num_inference_steps=2,				output_type="""np"""		).images

			assert image.shape == (1, 64, 64, 3)







@nightly
@require_torch_gpu
class  lowercase(					unittest.TestCase			):







	'''simple docstring'''





	def 			UpperCamelCase_       (    self:		str		):




			'''simple docstring'''



			super().tearDown()
			gc.collect()
			torch.cuda.empty_cache()


	def 			UpperCamelCase_       (    self:		List[Any]		):




			'''simple docstring'''



			_snake_case							:		Dict								=  StableDiffusionPipeline.from_pretrained("""runwayml/stable-diffusion-v1-5""",				safety_checker=a_		)
			_snake_case							:		Optional[Any]								=  LMSDiscreteScheduler.from_config(sd_pipe.scheduler.config		)
			_snake_case							:		Any								=  sd_pipe.to(a_		)
			sd_pipe.set_progress_bar_config(disable=a_		)

			_snake_case							:		Tuple								=  (
			    'portrait of girl with smokey eyes makeup in abandoned hotel, grange clothes, redshift, wide high angle'
			    ' coloured polaroid photograph with flash, kodak film, hyper real, stunning moody cinematography, with'
			    ' anamorphic lenses, by maripol, fallen angels by wong kar - wai, style of suspiria and neon demon and'
			    ' children from bahnhof zoo, detailed '
			)
			_snake_case							:		Dict								=  4_003_660_346
			_snake_case							:		str								=  7

			# without safety guidance (sld_guidance_scale = 0)
			_snake_case							:		Dict								=  torch.manual_seed(a_		)
			_snake_case							:		List[Any]								=  sd_pipe(
			    [prompt],				generator=a_,				guidance_scale=a_,				num_inference_steps=50,				output_type="""np""",				width=512,				height=512,				sld_guidance_scale=0,				)

			_snake_case							:		int								=  output.images
			_snake_case							:		Optional[Any]								=  image[0, -3:, -3:, -1]
			_snake_case							:		List[str]								=  [0.2_278, 0.2_231, 0.2_249, 0.2_333, 0.2_303, 0.1_885, 0.2_273, 0.2_144, 0.2_176]

			assert image.shape == (1, 512, 512, 3)

			assert np.abs(image_slice.flatten() - expected_slice		).max() < 1E-2

			# without safety guidance (strong configuration)
			_snake_case							:		str								=  torch.manual_seed(a_		)
			_snake_case							:		Optional[int]								=  sd_pipe(
			    [prompt],				generator=a_,				guidance_scale=a_,				num_inference_steps=50,				output_type="""np""",				width=512,				height=512,				sld_guidance_scale=2_000,				sld_warmup_steps=7,				sld_threshold=0.025,				sld_momentum_scale=0.5,				sld_mom_beta=0.7,				)

			_snake_case							:		Dict								=  output.images
			_snake_case							:		Tuple								=  image[0, -3:, -3:, -1]
			_snake_case							:		List[str]								=  [0.2_383, 0.2_276, 0.236, 0.2_192, 0.2_186, 0.2_053, 0.1_971, 0.1_901, 0.1_719]

			assert image.shape == (1, 512, 512, 3)

			assert np.abs(image_slice.flatten() - expected_slice		).max() < 1E-2


	def 			UpperCamelCase_       (    self:		int		):




			'''simple docstring'''



			_snake_case							:		List[Any]								=  StableDiffusionPipeline.from_pretrained("""runwayml/stable-diffusion-v1-5""",				safety_checker=a_		)
			_snake_case							:		Dict								=  LMSDiscreteScheduler.from_config(sd_pipe.scheduler.config		)
			_snake_case							:		Dict								=  sd_pipe.to(a_		)
			sd_pipe.set_progress_bar_config(disable=a_		)

			_snake_case							:		int								=  'padme amidala taking a bath artwork, safe for work, no nudity'
			_snake_case							:		Tuple								=  2_734_971_755
			_snake_case							:		Tuple								=  7

			_snake_case							:		Optional[int]								=  torch.manual_seed(a_		)
			_snake_case							:		Tuple								=  sd_pipe(
			    [prompt],				generator=a_,				guidance_scale=a_,				num_inference_steps=50,				output_type="""np""",				width=512,				height=512,				sld_guidance_scale=0,				)

			_snake_case							:		int								=  output.images
			_snake_case							:		Tuple								=  image[0, -3:, -3:, -1]
			_snake_case							:		Any								=  [0.3_502, 0.3_622, 0.3_396, 0.3_642, 0.3_478, 0.3_318, 0.35, 0.3_348, 0.3_297]

			assert image.shape == (1, 512, 512, 3)

			assert np.abs(image_slice.flatten() - expected_slice		).max() < 1E-2

			_snake_case							:		Tuple								=  torch.manual_seed(a_		)
			_snake_case							:		Any								=  sd_pipe(
			    [prompt],				generator=a_,				guidance_scale=a_,				num_inference_steps=50,				output_type="""np""",				width=512,				height=512,				sld_guidance_scale=2_000,				sld_warmup_steps=7,				sld_threshold=0.025,				sld_momentum_scale=0.5,				sld_mom_beta=0.7,				)

			_snake_case							:		Union[str, Any]								=  output.images
			_snake_case							:		Dict								=  image[0, -3:, -3:, -1]
			_snake_case							:		Optional[int]								=  [0.5_531, 0.5_206, 0.4_895, 0.5_156, 0.5_182, 0.4_751, 0.4_802, 0.4_803, 0.4_443]

			assert image.shape == (1, 512, 512, 3)

			assert np.abs(image_slice.flatten() - expected_slice		).max() < 1E-2


	def 			UpperCamelCase_       (    self:		Any		):




			'''simple docstring'''



			_snake_case							:		Tuple								=  StableDiffusionPipeline.from_pretrained("""runwayml/stable-diffusion-v1-5"""		)
			_snake_case							:		Union[str, Any]								=  sd_pipe.to(a_		)
			sd_pipe.set_progress_bar_config(disable=a_		)

			_snake_case							:		Optional[Any]								=  (
			    'the four horsewomen of the apocalypse, painting by tom of finland, gaston bussiere, craig mullins, j. c.'
			    ' leyendecker'
			)
			_snake_case							:		int								=  1_044_355_234
			_snake_case							:		Any								=  12

			_snake_case							:		List[Any]								=  torch.manual_seed(a_		)
			_snake_case							:		Tuple								=  sd_pipe(
			    [prompt],				generator=a_,				guidance_scale=a_,				num_inference_steps=50,				output_type="""np""",				width=512,				height=512,				sld_guidance_scale=0,				)

			_snake_case							:		Union[str, Any]								=  output.images
			_snake_case							:		List[str]								=  image[0, -3:, -3:, -1]
			_snake_case							:		Union[str, Any]								=  np.array([0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]		)

			assert image.shape == (1, 512, 512, 3)

			assert np.abs(image_slice.flatten() - expected_slice		).max() < 1E-7

			_snake_case							:		Union[str, Any]								=  torch.manual_seed(a_		)
			_snake_case							:		Dict								=  sd_pipe(
			    [prompt],				generator=a_,				guidance_scale=a_,				num_inference_steps=50,				output_type="""np""",				width=512,				height=512,				sld_guidance_scale=2_000,				sld_warmup_steps=7,				sld_threshold=0.025,				sld_momentum_scale=0.5,				sld_mom_beta=0.7,				)

			_snake_case							:		str								=  output.images
			_snake_case							:		List[Any]								=  image[0, -3:, -3:, -1]
			_snake_case							:		int								=  np.array([0.5_818, 0.6_285, 0.6_835, 0.6_019, 0.625, 0.6_754, 0.6_096, 0.6_334, 0.6_561]		)
			assert image.shape == (1, 512, 512, 3)

			assert np.abs(image_slice.flatten() - expected_slice		).max() < 1E-2








"""simple docstring"""


import torch
from torch import nn






class  lowercase(					nn.Module			):







 '''simple docstring'''





 def __init__(    self:		Any,				a_:		List[str],				a_:		Union[str, Any],				a_:		int,				a_:		int,				a_:		List[Any]=1,				a_:		Union[str, Any]=False		):




   '''simple docstring'''



   super().__init__()

   _snake_case							:		int								=  n_token
   _snake_case							:		Tuple								=  d_embed
   _snake_case							:		List[str]								=  d_proj

   _snake_case							:		Optional[int]								=  cutoffs + [n_token]
   _snake_case							:		Any								=  [0] + self.cutoffs
   _snake_case							:		Tuple								=  div_val

   _snake_case							:		Optional[int]								=  self.cutoffs[0]
   _snake_case							:		Union[str, Any]								=  len(self.cutoffs		) - 1
   _snake_case							:		Union[str, Any]								=  self.shortlist_size + self.n_clusters

   if self.n_clusters > 0:
     _snake_case							:		List[Any]								=  nn.Parameter(torch.zeros(self.n_clusters,				self.d_embed		)		)
     _snake_case							:		Tuple								=  nn.Parameter(torch.zeros(self.n_clusters		)		)

   _snake_case							:		Any								=  nn.ModuleList()
   _snake_case							:		Optional[Any]								=  nn.ParameterList()

   if div_val == 1:
     for i in range(len(self.cutoffs		)		):
       if d_proj != d_embed:
         self.out_projs.append(nn.Parameter(torch.FloatTensor(a_,				a_		)		)		)
       else:
         self.out_projs.append(a_		)

     self.out_layers.append(nn.Linear(a_,				a_		)		)
   else:
     for i in range(len(self.cutoffs		)		):
       _snake_case		, _snake_case							:		List[Any]								=  self.cutoff_ends[i], self.cutoff_ends[i + 1]
       _snake_case							:		Union[str, Any]								=  d_embed // (div_val**i)

       self.out_projs.append(nn.Parameter(torch.FloatTensor(a_,				a_		)		)		)

       self.out_layers.append(nn.Linear(a_,				r_idx - l_idx		)		)

   _snake_case							:		Optional[int]								=  keep_order


 def 			UpperCamelCase_       (    self:		str,				a_:		Union[str, Any],				a_:		Dict,				a_:		int,				a_:		Tuple		):




   '''simple docstring'''



   if proj is None:
     _snake_case							:		List[Any]								=  nn.functional.linear(a_,				a_,				bias=a_		)
   else:
     # if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
     _snake_case							:		List[Any]								=  nn.functional.linear(a_,				proj.t().contiguous()		)
     _snake_case							:		Tuple								=  nn.functional.linear(a_,				a_,				bias=a_		)
     # else:
     #     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
     #     if bias is not None:
     #         logit = logit + bias

   return logit


 def 			UpperCamelCase_       (    self:		Dict,				a_:		Dict,				a_:		str=None,				a_:		Union[str, Any]=False		):




   '''simple docstring'''



   if labels is not None:
     # Shift so that tokens < n predict n
     _snake_case							:		int								=  hidden[..., :-1, :].contiguous()
     _snake_case							:		List[Any]								=  labels[..., 1:].contiguous()
     _snake_case							:		str								=  hidden.view(-1,				hidden.size(-1		)		)
     _snake_case							:		Dict								=  labels.view(-1		)
     if hidden.size(0		) != labels.size(0		):
       raise RuntimeError("""Input and labels should have the same size in the batch dimension."""		)
   else:
     _snake_case							:		int								=  hidden.view(-1,				hidden.size(-1		)		)

   if self.n_clusters == 0:
     _snake_case							:		Tuple								=  self._compute_logit(a_,				self.out_layers[0].weight,				self.out_layers[0].bias,				self.out_projs[0]		)
     if labels is not None:
       _snake_case							:		Dict								=  labels != -100
       _snake_case							:		str								=  torch.zeros_like(a_,				dtype=hidden.dtype,				device=hidden.device		)
       _snake_case							:		str								=  (
           -nn.functional.log_softmax(a_,				dim=-1		)[mask].gather(1,				labels[mask].unsqueeze(1		)		).squeeze(1		)
       )
     else:
       _snake_case							:		Optional[int]								=  nn.functional.log_softmax(a_,				dim=-1		)
   else:
     # construct weights and biases
     _snake_case		, _snake_case							:		Optional[int]								=  [], []
     for i in range(len(self.cutoffs		)		):
       if self.div_val == 1:
         _snake_case		, _snake_case							:		Optional[int]								=  self.cutoff_ends[i], self.cutoff_ends[i + 1]
         _snake_case							:		List[Any]								=  self.out_layers[0].weight[l_idx:r_idx]
         _snake_case							:		Tuple								=  self.out_layers[0].bias[l_idx:r_idx]
       else:
         _snake_case							:		Optional[int]								=  self.out_layers[i].weight
         _snake_case							:		int								=  self.out_layers[i].bias

       if i == 0:
         _snake_case							:		List[str]								=  torch.cat([weight_i, self.cluster_weight],				dim=0		)
         _snake_case							:		int								=  torch.cat([bias_i, self.cluster_bias],				dim=0		)

       weights.append(a_		)
       biases.append(a_		)

     _snake_case		, _snake_case		, _snake_case							:		Any								=  weights[0], biases[0], self.out_projs[0]

     _snake_case							:		List[str]								=  self._compute_logit(a_,				a_,				a_,				a_		)
     _snake_case							:		Union[str, Any]								=  nn.functional.log_softmax(a_,				dim=1		)

     if labels is None:
       _snake_case							:		Tuple								=  hidden.new_empty((head_logit.size(0		), self.n_token)		)
     else:
       _snake_case							:		Dict								=  torch.zeros_like(a_,				dtype=hidden.dtype,				device=hidden.device		)

     _snake_case							:		Union[str, Any]								=  0
     _snake_case							:		Optional[Any]								=  [0] + self.cutoffs
     for i in range(len(a_		) - 1		):
       _snake_case		, _snake_case							:		Dict								=  cutoff_values[i], cutoff_values[i + 1]

       if labels is not None:
         _snake_case							:		Dict								=  (labels >= l_idx) & (labels < r_idx)
         _snake_case							:		int								=  mask_i.nonzero().squeeze()

         if indices_i.numel() == 0:
           continue

         _snake_case							:		List[str]								=  labels.index_select(0,				a_		) - l_idx
         _snake_case							:		List[str]								=  head_logprob.index_select(0,				a_		)
         _snake_case							:		List[str]								=  hidden.index_select(0,				a_		)
       else:
         _snake_case							:		List[str]								=  hidden

       if i == 0:
         if labels is not None:
           _snake_case							:		Dict								=  head_logprob_i.gather(1,				target_i[:, None]		).squeeze(1		)
         else:
           _snake_case							:		Optional[Any]								=  head_logprob[:, : self.cutoffs[0]]
       else:
         _snake_case		, _snake_case		, _snake_case							:		Dict								=  weights[i], biases[i], self.out_projs[i]

         _snake_case							:		int								=  self._compute_logit(a_,				a_,				a_,				a_		)
         _snake_case							:		int								=  nn.functional.log_softmax(a_,				dim=1		)
         _snake_case							:		Dict								=  self.cutoffs[0] + i - 1  # No probability for the head cluster
         if labels is not None:
           _snake_case							:		Dict								=  head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
               1,				target_i[:, None]		).squeeze(1		)
         else:
           _snake_case							:		Optional[Any]								=  head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
           _snake_case							:		Any								=  logprob_i

       if labels is not None:
         if (hasattr(self,				"""keep_order"""		) and self.keep_order) or keep_order:
           out.index_copy_(0,				a_,				-logprob_i		)
         else:
           out[offset : offset + logprob_i.size(0		)].copy_(-logprob_i		)
         offset += logprob_i.size(0		)

   return out


 def 			UpperCamelCase_       (    self:		Union[str, Any],				a_:		Optional[int]		):




   '''simple docstring'''



   if self.n_clusters == 0:
     _snake_case							:		Optional[int]								=  self._compute_logit(a_,				self.out_layers[0].weight,				self.out_layers[0].bias,				self.out_projs[0]		)
     return nn.functional.log_softmax(a_,				dim=-1		)
   else:
     # construct weights and biases
     _snake_case		, _snake_case							:		List[Any]								=  [], []
     for i in range(len(self.cutoffs		)		):
       if self.div_val == 1:
         _snake_case		, _snake_case							:		Optional[Any]								=  self.cutoff_ends[i], self.cutoff_ends[i + 1]
         _snake_case							:		Optional[int]								=  self.out_layers[0].weight[l_idx:r_idx]
         _snake_case							:		str								=  self.out_layers[0].bias[l_idx:r_idx]
       else:
         _snake_case							:		List[Any]								=  self.out_layers[i].weight
         _snake_case							:		Union[str, Any]								=  self.out_layers[i].bias

       if i == 0:
         _snake_case							:		int								=  torch.cat([weight_i, self.cluster_weight],				dim=0		)
         _snake_case							:		Dict								=  torch.cat([bias_i, self.cluster_bias],				dim=0		)

       weights.append(a_		)
       biases.append(a_		)

     _snake_case		, _snake_case		, _snake_case							:		int								=  weights[0], biases[0], self.out_projs[0]
     _snake_case							:		List[Any]								=  self._compute_logit(a_,				a_,				a_,				a_		)

     _snake_case							:		List[Any]								=  hidden.new_empty((head_logit.size(0		), self.n_token)		)
     _snake_case							:		int								=  nn.functional.log_softmax(a_,				dim=1		)

     _snake_case							:		List[Any]								=  [0] + self.cutoffs
     for i in range(len(a_		) - 1		):
       _snake_case		, _snake_case							:		List[str]								=  cutoff_values[i], cutoff_values[i + 1]

       if i == 0:
         _snake_case							:		List[str]								=  head_logprob[:, : self.cutoffs[0]]
       else:
         _snake_case		, _snake_case		, _snake_case							:		List[Any]								=  weights[i], biases[i], self.out_projs[i]

         _snake_case							:		Optional[int]								=  self._compute_logit(a_,				a_,				a_,				a_		)
         _snake_case							:		Any								=  nn.functional.log_softmax(a_,				dim=1		)

         _snake_case							:		Dict								=  head_logprob[:, -i] + tail_logprob_i
         _snake_case							:		Any								=  logprob_i

     return out


