




import tempfile
import unittest

import numpy as np

from diffusers import (
    DDIMScheduler,
    DPMSolverMultistepScheduler,
    EulerAncestralDiscreteScheduler,
    EulerDiscreteScheduler,
    LMSDiscreteScheduler,
    OnnxStableDiffusionPipeline,
    PNDMScheduler,
)
from diffusers.utils.testing_utils import is_onnx_available, nightly, require_onnxruntime, require_torch_gpu

from ..test_pipelines_onnx_common import OnnxPipelineTesterMixin


if is_onnx_available():
					import onnxruntime as ort
class       a__   ( snake_case__			,    unittest.TestCase				):
						_a							:   List[str] =   """hf-internal-testing/tiny-random-OnnxStableDiffusionPipeline"""






						def     __SCREAMING_SNAKE_CASE(       self		,      _A=0      ):




											"""simple docstring"""



											__lowerCAmelCase           =	np.random.RandomState(_A      )
											__lowerCAmelCase           =	{
											    "prompt": "A painting of a squirrel eating a burger",
											    "generator": generator,
											    "num_inference_steps": 2,
											    "guidance_scale": 7.5,
											    "output_type": "numpy",
											}
											return inputs






						def     __SCREAMING_SNAKE_CASE(       self      ):




											"""simple docstring"""



											__lowerCAmelCase           =	OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint		,      provider="CPUExecutionProvider"      )
											pipe.set_progress_bar_config(disable=_A      )

											__lowerCAmelCase           =	self.get_dummy_inputs()
											__lowerCAmelCase           =	pipe(**_A      ).images
											__lowerCAmelCase           =	image[0, -3:, -3:, -1]

											assert image.shape == (1, 1_2_8, 1_2_8, 3)
											__lowerCAmelCase           =	np.array([0.6_50_72, 0.5_84_92, 0.4_82_19, 0.5_55_21, 0.5_31_80, 0.5_59_39, 0.5_06_97, 0.3_98_00, 0.4_64_55]      )

											assert np.abs(image_slice.flatten() - expected_slice      ).max() < 1E-2






						def     __SCREAMING_SNAKE_CASE(       self      ):




											"""simple docstring"""



											__lowerCAmelCase           =	OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint		,      provider="CPUExecutionProvider"      )
											__lowerCAmelCase           =	PNDMScheduler.from_config(pipe.scheduler.config		,      skip_prk_steps=_A      )
											pipe.set_progress_bar_config(disable=_A      )

											__lowerCAmelCase           =	self.get_dummy_inputs()
											__lowerCAmelCase           =	pipe(**_A      ).images
											__lowerCAmelCase           =	image[0, -3:, -3:, -1]

											assert image.shape == (1, 1_2_8, 1_2_8, 3)
											__lowerCAmelCase           =	np.array([0.6_58_63, 0.5_94_25, 0.4_93_26, 0.5_63_13, 0.5_38_75, 0.5_66_27, 0.5_10_65, 0.3_97_77, 0.4_63_30]      )

											assert np.abs(image_slice.flatten() - expected_slice      ).max() < 1E-2






						def     __SCREAMING_SNAKE_CASE(       self      ):




											"""simple docstring"""



											__lowerCAmelCase           =	OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint		,      provider="CPUExecutionProvider"      )
											__lowerCAmelCase           =	LMSDiscreteScheduler.from_config(pipe.scheduler.config      )
											pipe.set_progress_bar_config(disable=_A      )

											__lowerCAmelCase           =	self.get_dummy_inputs()
											__lowerCAmelCase           =	pipe(**_A      ).images
											__lowerCAmelCase           =	image[0, -3:, -3:, -1]

											assert image.shape == (1, 1_2_8, 1_2_8, 3)
											__lowerCAmelCase           =	np.array([0.5_37_55, 0.6_07_86, 0.4_74_02, 0.4_94_88, 0.5_18_69, 0.4_98_19, 0.4_79_85, 0.3_89_57, 0.4_42_79]      )

											assert np.abs(image_slice.flatten() - expected_slice      ).max() < 1E-2






						def     __SCREAMING_SNAKE_CASE(       self      ):




											"""simple docstring"""



											__lowerCAmelCase           =	OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint		,      provider="CPUExecutionProvider"      )
											__lowerCAmelCase           =	EulerDiscreteScheduler.from_config(pipe.scheduler.config      )
											pipe.set_progress_bar_config(disable=_A      )

											__lowerCAmelCase           =	self.get_dummy_inputs()
											__lowerCAmelCase           =	pipe(**_A      ).images
											__lowerCAmelCase           =	image[0, -3:, -3:, -1]

											assert image.shape == (1, 1_2_8, 1_2_8, 3)
											__lowerCAmelCase           =	np.array([0.5_37_55, 0.6_07_86, 0.4_74_02, 0.4_94_88, 0.5_18_69, 0.4_98_19, 0.4_79_85, 0.3_89_57, 0.4_42_79]      )

											assert np.abs(image_slice.flatten() - expected_slice      ).max() < 1E-2






						def     __SCREAMING_SNAKE_CASE(       self      ):




											"""simple docstring"""



											__lowerCAmelCase           =	OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint		,      provider="CPUExecutionProvider"      )
											__lowerCAmelCase           =	EulerAncestralDiscreteScheduler.from_config(pipe.scheduler.config      )
											pipe.set_progress_bar_config(disable=_A      )

											__lowerCAmelCase           =	self.get_dummy_inputs()
											__lowerCAmelCase           =	pipe(**_A      ).images
											__lowerCAmelCase           =	image[0, -3:, -3:, -1]

											assert image.shape == (1, 1_2_8, 1_2_8, 3)
											__lowerCAmelCase           =	np.array([0.5_38_17, 0.6_08_12, 0.4_73_84, 0.4_95_30, 0.5_18_94, 0.4_98_14, 0.4_79_84, 0.3_89_58, 0.4_42_71]      )

											assert np.abs(image_slice.flatten() - expected_slice      ).max() < 1E-2






						def     __SCREAMING_SNAKE_CASE(       self      ):




											"""simple docstring"""



											__lowerCAmelCase           =	OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint		,      provider="CPUExecutionProvider"      )
											__lowerCAmelCase           =	DPMSolverMultistepScheduler.from_config(pipe.scheduler.config      )
											pipe.set_progress_bar_config(disable=_A      )

											__lowerCAmelCase           =	self.get_dummy_inputs()
											__lowerCAmelCase           =	pipe(**_A      ).images
											__lowerCAmelCase           =	image[0, -3:, -3:, -1]

											assert image.shape == (1, 1_2_8, 1_2_8, 3)
											__lowerCAmelCase           =	np.array([0.5_38_95, 0.6_08_08, 0.4_79_33, 0.4_96_08, 0.5_18_86, 0.4_99_50, 0.4_80_53, 0.3_89_57, 0.4_42_00]      )

											assert np.abs(image_slice.flatten() - expected_slice      ).max() < 1E-2






						def     __SCREAMING_SNAKE_CASE(       self      ):




											"""simple docstring"""



											__lowerCAmelCase           =	OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint		,      provider="CPUExecutionProvider"      )
											pipe.set_progress_bar_config(disable=_A      )

											__lowerCAmelCase           =	self.get_dummy_inputs()
											__lowerCAmelCase           =	3 * [inputs["prompt"]]

											# forward
											__lowerCAmelCase           =	pipe(**_A      )
											__lowerCAmelCase           =	output.images[0, -3:, -3:, -1]

											__lowerCAmelCase           =	self.get_dummy_inputs()
											__lowerCAmelCase           =	3 * [inputs.pop("prompt"      )]

											__lowerCAmelCase           =	pipe.tokenizer(
											    _A		,      padding="max_length"		,      max_length=pipe.tokenizer.model_max_length		,      truncation=_A		,      return_tensors="np"		,      )
											__lowerCAmelCase           =	text_inputs["input_ids"]

											__lowerCAmelCase           =	pipe.text_encoder(input_ids=text_inputs.astype(np.intaa      )      )[0]

											__lowerCAmelCase           =	prompt_embeds

											# forward
											__lowerCAmelCase           =	pipe(**_A      )
											__lowerCAmelCase           =	output.images[0, -3:, -3:, -1]

											assert np.abs(image_slice_a.flatten() - image_slice_a.flatten()      ).max() < 1E-4






						def     __SCREAMING_SNAKE_CASE(       self      ):




											"""simple docstring"""



											__lowerCAmelCase           =	OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint		,      provider="CPUExecutionProvider"      )
											pipe.set_progress_bar_config(disable=_A      )

											__lowerCAmelCase           =	self.get_dummy_inputs()
											__lowerCAmelCase           =	3 * ["this is a negative prompt"]
											__lowerCAmelCase           =	negative_prompt
											__lowerCAmelCase           =	3 * [inputs["prompt"]]

											# forward
											__lowerCAmelCase           =	pipe(**_A      )
											__lowerCAmelCase           =	output.images[0, -3:, -3:, -1]

											__lowerCAmelCase           =	self.get_dummy_inputs()
											__lowerCAmelCase           =	3 * [inputs.pop("prompt"      )]

											__lowerCAmelCase           =	[]
											for p in [prompt, negative_prompt]:
																__lowerCAmelCase           =	pipe.tokenizer(
																    _A		,      padding="max_length"		,      max_length=pipe.tokenizer.model_max_length		,      truncation=_A		,      return_tensors="np"		,      )
																__lowerCAmelCase           =	text_inputs["input_ids"]

																embeds.append(pipe.text_encoder(input_ids=text_inputs.astype(np.intaa      )      )[0]      )

											__lowerCAmelCase       ,							__lowerCAmelCase           =	embeds

											# forward
											__lowerCAmelCase           =	pipe(**_A      )
											__lowerCAmelCase           =	output.images[0, -3:, -3:, -1]

											assert np.abs(image_slice_a.flatten() - image_slice_a.flatten()      ).max() < 1E-4






@nightly
@require_onnxruntime
@require_torch_gpu
class       a__   ( unittest.TestCase				):






						@property
						def     __SCREAMING_SNAKE_CASE(       self      ):




											"""simple docstring"""



											return (
											    "CUDAExecutionProvider",
											    {
											        "gpu_mem_limit": "15000000000",  # 15GB
											        "arena_extend_strategy": "kSameAsRequested",
											    },
											)






						@property
						def     __SCREAMING_SNAKE_CASE(       self      ):




											"""simple docstring"""



											__lowerCAmelCase           =	ort.SessionOptions()
											__lowerCAmelCase           =	False
											return options






						def     __SCREAMING_SNAKE_CASE(       self      ):




											"""simple docstring"""



											__lowerCAmelCase           =	OnnxStableDiffusionPipeline.from_pretrained(
											    "CompVis/stable-diffusion-v1-4"		,      revision="onnx"		,      safety_checker=_A		,      feature_extractor=_A		,      provider=self.gpu_provider		,      sess_options=self.gpu_options		,      )
											sd_pipe.set_progress_bar_config(disable=_A      )

											__lowerCAmelCase           =	"A painting of a squirrel eating a burger"
											np.random.seed(0      )
											__lowerCAmelCase           =	sd_pipe([prompt]		,      guidance_scale=6.0		,      num_inference_steps=1_0		,      output_type="np"      )
											__lowerCAmelCase           =	output.images

											__lowerCAmelCase           =	image[0, -3:, -3:, -1]

											assert image.shape == (1, 5_1_2, 5_1_2, 3)
											__lowerCAmelCase           =	np.array([0.04_52, 0.03_90, 0.00_87, 0.03_50, 0.06_17, 0.03_64, 0.05_44, 0.05_23, 0.07_20]      )

											assert np.abs(image_slice.flatten() - expected_slice      ).max() < 1E-3






						def     __SCREAMING_SNAKE_CASE(       self      ):




											"""simple docstring"""



											__lowerCAmelCase           =	DDIMScheduler.from_pretrained(
											    "runwayml/stable-diffusion-v1-5"		,      subfolder="scheduler"		,      revision="onnx"      )
											__lowerCAmelCase           =	OnnxStableDiffusionPipeline.from_pretrained(
											    "runwayml/stable-diffusion-v1-5"		,      revision="onnx"		,      scheduler=_A		,      safety_checker=_A		,      feature_extractor=_A		,      provider=self.gpu_provider		,      sess_options=self.gpu_options		,      )
											sd_pipe.set_progress_bar_config(disable=_A      )

											__lowerCAmelCase           =	"open neural network exchange"
											__lowerCAmelCase           =	np.random.RandomState(0      )
											__lowerCAmelCase           =	sd_pipe([prompt]		,      guidance_scale=7.5		,      num_inference_steps=1_0		,      generator=_A		,      output_type="np"      )
											__lowerCAmelCase           =	output.images
											__lowerCAmelCase           =	image[0, -3:, -3:, -1]

											assert image.shape == (1, 5_1_2, 5_1_2, 3)
											__lowerCAmelCase           =	np.array([0.28_67, 0.19_74, 0.14_81, 0.72_94, 0.72_51, 0.66_67, 0.41_94, 0.56_42, 0.64_86]      )

											assert np.abs(image_slice.flatten() - expected_slice      ).max() < 1E-3






						def     __SCREAMING_SNAKE_CASE(       self      ):




											"""simple docstring"""



											__lowerCAmelCase           =	LMSDiscreteScheduler.from_pretrained(
											    "runwayml/stable-diffusion-v1-5"		,      subfolder="scheduler"		,      revision="onnx"      )
											__lowerCAmelCase           =	OnnxStableDiffusionPipeline.from_pretrained(
											    "runwayml/stable-diffusion-v1-5"		,      revision="onnx"		,      scheduler=_A		,      safety_checker=_A		,      feature_extractor=_A		,      provider=self.gpu_provider		,      sess_options=self.gpu_options		,      )
											sd_pipe.set_progress_bar_config(disable=_A      )

											__lowerCAmelCase           =	"open neural network exchange"
											__lowerCAmelCase           =	np.random.RandomState(0      )
											__lowerCAmelCase           =	sd_pipe([prompt]		,      guidance_scale=7.5		,      num_inference_steps=1_0		,      generator=_A		,      output_type="np"      )
											__lowerCAmelCase           =	output.images
											__lowerCAmelCase           =	image[0, -3:, -3:, -1]

											assert image.shape == (1, 5_1_2, 5_1_2, 3)
											__lowerCAmelCase           =	np.array([0.23_06, 0.19_59, 0.15_93, 0.65_49, 0.63_94, 0.54_08, 0.50_65, 0.60_10, 0.61_61]      )

											assert np.abs(image_slice.flatten() - expected_slice      ).max() < 1E-3






						def     __SCREAMING_SNAKE_CASE(       self      ):




											"""simple docstring"""



											__lowerCAmelCase           =	0

											def test_callback_fn(_A		,      _A		,      _A      ) -> None:
																__lowerCAmelCase           =	True
																nonlocal number_of_steps
																number_of_steps += 1
																if step == 0:
																					assert latents.shape == (1, 4, 6_4, 6_4)
																					__lowerCAmelCase           =	latents[0, -3:, -3:, -1]
																					__lowerCAmelCase           =	np.array(
																					    [-0.67_72, -0.38_35, -1.24_56, 0.19_05, -1.09_74, 0.69_67, -1.93_53, 0.01_78, 1.01_67]      )

																					assert np.abs(latents_slice.flatten() - expected_slice      ).max() < 1E-3
																elif step == 5:
																					assert latents.shape == (1, 4, 6_4, 6_4)
																					__lowerCAmelCase           =	latents[0, -3:, -3:, -1]
																					__lowerCAmelCase           =	np.array(
																					    [-0.33_51, 0.22_41, -0.18_37, -0.23_25, -0.65_77, 0.33_93, -0.02_41, 0.58_99, 1.38_75]      )

																					assert np.abs(latents_slice.flatten() - expected_slice      ).max() < 1E-3

											__lowerCAmelCase           =	False

											__lowerCAmelCase           =	OnnxStableDiffusionPipeline.from_pretrained(
											    "runwayml/stable-diffusion-v1-5"		,      revision="onnx"		,      safety_checker=_A		,      feature_extractor=_A		,      provider=self.gpu_provider		,      sess_options=self.gpu_options		,      )
											pipe.set_progress_bar_config(disable=_A      )

											__lowerCAmelCase           =	"Andromeda galaxy in a bottle"

											__lowerCAmelCase           =	np.random.RandomState(0      )
											pipe(
											    prompt=_A		,      num_inference_steps=5		,      guidance_scale=7.5		,      generator=_A		,      callback=_A		,      callback_steps=1		,      )
											assert test_callback_fn.has_been_called
											assert number_of_steps == 6






						def     __SCREAMING_SNAKE_CASE(       self      ):




											"""simple docstring"""



											__lowerCAmelCase           =	OnnxStableDiffusionPipeline.from_pretrained(
											    "runwayml/stable-diffusion-v1-5"		,      revision="onnx"		,      safety_checker=_A		,      feature_extractor=_A		,      provider=self.gpu_provider		,      sess_options=self.gpu_options		,      )
											assert isinstance(_A		,      _A      )
											assert pipe.safety_checker is None

											__lowerCAmelCase           =	pipe("example prompt"		,      num_inference_steps=2      ).images[0]
											assert image is not None

											# check that there's no error when saving a pipeline with one of the models being None
											with tempfile.TemporaryDirectory() as tmpdirname:
																pipe.save_pretrained(_A      )
																__lowerCAmelCase           =	OnnxStableDiffusionPipeline.from_pretrained(_A      )

											# sanity check that the pipeline still works
											assert pipe.safety_checker is None
											__lowerCAmelCase           =	pipe("example prompt"		,      num_inference_steps=2      ).images[0]
											assert image is not None










from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..utils import maybe_allow_in_graph
from .activations import get_activation
from .attention_processor import Attention
from .embeddings import CombinedTimestepLabelEmbeddings
@maybe_allow_in_graph
class       a__   ( nn.Module				):






						def __init__(       self		,      _A		,      _A		,      _A		,      _A=0.0		,      _A = None		,      _A = "geglu"		,      _A = None		,      _A = False		,      _A = False		,      _A = False		,      _A = False		,      _A = True		,      _A = "layer_norm"		,      _A = False		,      ):




											"""simple docstring"""



											super().__init__()
											__lowerCAmelCase           =	only_cross_attention

											__lowerCAmelCase           =	(num_embeds_ada_norm is not None) and norm_type == "ada_norm_zero"
											__lowerCAmelCase           =	(num_embeds_ada_norm is not None) and norm_type == "ada_norm"

											if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None:
																raise ValueError(
																    f"""`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to"""
																    f""" define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}."""      )

											# Define 3 blocks. Each block has its own normalization layer.
											# 1. Self-Attn
											if self.use_ada_layer_norm:
																__lowerCAmelCase           =	AdaLayerNorm(_A		,      _A      )
											elif self.use_ada_layer_norm_zero:
																__lowerCAmelCase           =	AdaLayerNormZero(_A		,      _A      )
											else:
																__lowerCAmelCase           =	nn.LayerNorm(_A		,      elementwise_affine=_A      )
											__lowerCAmelCase           =	Attention(
											    query_dim=_A		,      heads=_A		,      dim_head=_A		,      dropout=_A		,      bias=_A		,      cross_attention_dim=cross_attention_dim if only_cross_attention else None		,      upcast_attention=_A		,      )

											# 2. Cross-Attn
											if cross_attention_dim is not None or double_self_attention:
																# We currently only use AdaLayerNormZero for self attention where there will only be one attention block.
																# I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during
																# the second cross attention block.
																__lowerCAmelCase           =	(
																    AdaLayerNorm(_A		,      _A      )
																    if self.use_ada_layer_norm
																    else nn.LayerNorm(_A		,      elementwise_affine=_A      )
																)
																__lowerCAmelCase           =	Attention(
																    query_dim=_A		,      cross_attention_dim=cross_attention_dim if not double_self_attention else None		,      heads=_A		,      dim_head=_A		,      dropout=_A		,      bias=_A		,      upcast_attention=_A		,      )  # is self-attn if encoder_hidden_states is none
											else:
																__lowerCAmelCase           =	None
																__lowerCAmelCase           =	None

											# 3. Feed-forward
											__lowerCAmelCase           =	nn.LayerNorm(_A		,      elementwise_affine=_A      )
											__lowerCAmelCase           =	FeedForward(_A		,      dropout=_A		,      activation_fn=_A		,      final_dropout=_A      )

											# let chunk size default to None
											__lowerCAmelCase           =	None
											__lowerCAmelCase           =	0






						def     __SCREAMING_SNAKE_CASE(       self		,      _A		,      _A      ):




											"""simple docstring"""



											__lowerCAmelCase           =	chunk_size
											__lowerCAmelCase           =	dim






						def     __SCREAMING_SNAKE_CASE(       self		,      _A		,      _A = None		,      _A = None		,      _A = None		,      _A = None		,      _A = None		,      _A = None		,      ):




											"""simple docstring"""



											if self.use_ada_layer_norm:
																__lowerCAmelCase           =	self.norma(_A		,      _A      )
											elif self.use_ada_layer_norm_zero:
																__lowerCAmelCase       ,							__lowerCAmelCase       ,							__lowerCAmelCase       ,							__lowerCAmelCase       ,							__lowerCAmelCase           =	self.norma(
																    _A		,      _A		,      _A		,      hidden_dtype=hidden_states.dtype      )
											else:
																__lowerCAmelCase           =	self.norma(_A      )

											__lowerCAmelCase           =	cross_attention_kwargs if cross_attention_kwargs is not None else {}

											__lowerCAmelCase           =	self.attna(
											    _A		,      encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None		,      attention_mask=_A		,      **_A		,      )
											if self.use_ada_layer_norm_zero:
																__lowerCAmelCase           =	gate_msa.unsqueeze(1      ) * attn_output
											__lowerCAmelCase           =	attn_output + hidden_states

											# 2. Cross-Attention
											if self.attna is not None:
																__lowerCAmelCase           =	(
																    self.norma(_A		,      _A      ) if self.use_ada_layer_norm else self.norma(_A      )
																)

																__lowerCAmelCase           =	self.attna(
																    _A		,      encoder_hidden_states=_A		,      attention_mask=_A		,      **_A		,      )
																__lowerCAmelCase           =	attn_output + hidden_states

											# 3. Feed-forward
											__lowerCAmelCase           =	self.norma(_A      )

											if self.use_ada_layer_norm_zero:
																__lowerCAmelCase           =	norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]

											if self._chunk_size is not None:
																# "feed_forward_chunk_size" can be used to save memory
																if norm_hidden_states.shape[self._chunk_dim] % self._chunk_size != 0:
																					raise ValueError(
																					    f"""`hidden_states` dimension to be chunked: {norm_hidden_states.shape[self._chunk_dim]} has to be divisible by chunk size: {self._chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`."""      )

																__lowerCAmelCase           =	norm_hidden_states.shape[self._chunk_dim] // self._chunk_size
																__lowerCAmelCase           =	torch.cat(
																    [self.ff(_A      ) for hid_slice in norm_hidden_states.chunk(_A		,      dim=self._chunk_dim      )]		,      dim=self._chunk_dim		,      )
											else:
																__lowerCAmelCase           =	self.ff(_A      )

											if self.use_ada_layer_norm_zero:
																__lowerCAmelCase           =	gate_mlp.unsqueeze(1      ) * ff_output

											__lowerCAmelCase           =	ff_output + hidden_states

											return hidden_states






class       a__   ( nn.Module				):






						def __init__(       self		,      _A		,      _A = None		,      _A = 4		,      _A = 0.0		,      _A = "geglu"		,      _A = False		,      ):




											"""simple docstring"""



											super().__init__()
											__lowerCAmelCase           =	int(dim * mult      )
											__lowerCAmelCase           =	dim_out if dim_out is not None else dim

											if activation_fn == "gelu":
																__lowerCAmelCase           =	GELU(_A		,      _A      )
											if activation_fn == "gelu-approximate":
																__lowerCAmelCase           =	GELU(_A		,      _A		,      approximate="tanh"      )
											elif activation_fn == "geglu":
																__lowerCAmelCase           =	GEGLU(_A		,      _A      )
											elif activation_fn == "geglu-approximate":
																__lowerCAmelCase           =	ApproximateGELU(_A		,      _A      )

											__lowerCAmelCase           =	nn.ModuleList([]      )
											# project in
											self.net.append(_A      )
											# project dropout
											self.net.append(nn.Dropout(_A      )      )
											# project out
											self.net.append(nn.Linear(_A		,      _A      )      )
											# FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout
											if final_dropout:
																self.net.append(nn.Dropout(_A      )      )






						def     __SCREAMING_SNAKE_CASE(       self		,      _A      ):




											"""simple docstring"""



											for module in self.net:
																__lowerCAmelCase           =	module(_A      )
											return hidden_states






class       a__   ( nn.Module				):






						def __init__(       self		,      _A		,      _A		,      _A = "none"      ):




											"""simple docstring"""



											super().__init__()
											__lowerCAmelCase           =	nn.Linear(_A		,      _A      )
											__lowerCAmelCase           =	approximate






						def     __SCREAMING_SNAKE_CASE(       self		,      _A      ):




											"""simple docstring"""



											if gate.device.type != "mps":
																return F.gelu(_A		,      approximate=self.approximate      )
											# mps: gelu is not implemented for float16
											return F.gelu(gate.to(dtype=torch.floataa      )		,      approximate=self.approximate      ).to(dtype=gate.dtype      )






						def     __SCREAMING_SNAKE_CASE(       self		,      _A      ):




											"""simple docstring"""



											__lowerCAmelCase           =	self.proj(_A      )
											__lowerCAmelCase           =	self.gelu(_A      )
											return hidden_states






class       a__   ( nn.Module				):






						def __init__(       self		,      _A		,      _A      ):




											"""simple docstring"""



											super().__init__()
											__lowerCAmelCase           =	nn.Linear(_A		,      dim_out * 2      )






						def     __SCREAMING_SNAKE_CASE(       self		,      _A      ):




											"""simple docstring"""



											if gate.device.type != "mps":
																return F.gelu(_A      )
											# mps: gelu is not implemented for float16
											return F.gelu(gate.to(dtype=torch.floataa      )      ).to(dtype=gate.dtype      )






						def     __SCREAMING_SNAKE_CASE(       self		,      _A      ):




											"""simple docstring"""



											__lowerCAmelCase       ,							__lowerCAmelCase           =	self.proj(_A      ).chunk(2		,      dim=-1      )
											return hidden_states * self.gelu(_A      )






class       a__   ( nn.Module				):






						def __init__(       self		,      _A		,      _A      ):




											"""simple docstring"""



											super().__init__()
											__lowerCAmelCase           =	nn.Linear(_A		,      _A      )






						def     __SCREAMING_SNAKE_CASE(       self		,      _A      ):




											"""simple docstring"""



											__lowerCAmelCase           =	self.proj(_A      )
											return x * torch.sigmoid(1.7_02 * x      )






class       a__   ( nn.Module				):






						def __init__(       self		,      _A		,      _A      ):




											"""simple docstring"""



											super().__init__()
											__lowerCAmelCase           =	nn.Embedding(_A		,      _A      )
											__lowerCAmelCase           =	nn.SiLU()
											__lowerCAmelCase           =	nn.Linear(_A		,      embedding_dim * 2      )
											__lowerCAmelCase           =	nn.LayerNorm(_A		,      elementwise_affine=_A      )






						def     __SCREAMING_SNAKE_CASE(       self		,      _A		,      _A      ):




											"""simple docstring"""



											__lowerCAmelCase           =	self.linear(self.silu(self.emb(_A      )      )      )
											__lowerCAmelCase       ,							__lowerCAmelCase           =	torch.chunk(_A		,      2      )
											__lowerCAmelCase           =	self.norm(_A      ) * (1 + scale) + shift
											return x






class       a__   ( nn.Module				):






						def __init__(       self		,      _A		,      _A      ):




											"""simple docstring"""



											super().__init__()

											__lowerCAmelCase           =	CombinedTimestepLabelEmbeddings(_A		,      _A      )

											__lowerCAmelCase           =	nn.SiLU()
											__lowerCAmelCase           =	nn.Linear(_A		,      6 * embedding_dim		,      bias=_A      )
											__lowerCAmelCase           =	nn.LayerNorm(_A		,      elementwise_affine=_A		,      eps=1E-6      )






						def     __SCREAMING_SNAKE_CASE(       self		,      _A		,      _A		,      _A		,      _A=None      ):




											"""simple docstring"""



											__lowerCAmelCase           =	self.linear(self.silu(self.emb(_A		,      _A		,      hidden_dtype=_A      )      )      )
											__lowerCAmelCase       ,							__lowerCAmelCase       ,							__lowerCAmelCase       ,							__lowerCAmelCase       ,							__lowerCAmelCase       ,							__lowerCAmelCase           =	emb.chunk(6		,      dim=1      )
											__lowerCAmelCase           =	self.norm(_A      ) * (1 + scale_msa[:, None]) + shift_msa[:, None]
											return x, gate_msa, shift_mlp, scale_mlp, gate_mlp






class       a__   ( nn.Module				):






						def __init__(       self		,      _A		,      _A		,      _A		,      _A = None		,      _A = 1E-5      ):




											"""simple docstring"""



											super().__init__()
											__lowerCAmelCase           =	num_groups
											__lowerCAmelCase           =	eps

											if act_fn is None:
																__lowerCAmelCase           =	None
											else:
																__lowerCAmelCase           =	get_activation(_A      )

											__lowerCAmelCase           =	nn.Linear(_A		,      out_dim * 2      )






						def     __SCREAMING_SNAKE_CASE(       self		,      _A		,      _A      ):




											"""simple docstring"""



											if self.act:
																__lowerCAmelCase           =	self.act(_A      )
											__lowerCAmelCase           =	self.linear(_A      )
											__lowerCAmelCase           =	emb[:, :, None, None]
											__lowerCAmelCase       ,							__lowerCAmelCase           =	emb.chunk(2		,      dim=1      )

											__lowerCAmelCase           =	F.group_norm(_A		,      self.num_groups		,      eps=self.eps      )
											__lowerCAmelCase           =	x * (1 + scale) + shift
											return x




