




"""simple docstring"""


import inspect
from typing import Optional, Union

import numpy as np
import PIL
import torch
from torch.nn import functional as F
from torchvision import transforms
from transformers import CLIPFeatureExtractor, CLIPModel, CLIPTextModel, CLIPTokenizer

from diffusers import (
    AutoencoderKL,
    DDIMScheduler,
    DiffusionPipeline,
    DPMSolverMultistepScheduler,
    LMSDiscreteScheduler,
    PNDMScheduler,
    UNetaDConditionModel,
)
from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion import StableDiffusionPipelineOutput
from diffusers.utils import (
    PIL_INTERPOLATION,
    randn_tensor,
)


def  UpperCAmelCase__	(snake_case__							: int		,       snake_case__							: Optional[int]		,       snake_case__							: Tuple   ):
  """simple docstring"""

  if isinstance(snake_case__		,       torch.Tensor   ):
    return image
  elif isinstance(snake_case__		,       PIL.Image.Image   ):
    _snake_case							:		Tuple								=  [image]

  if isinstance(image[0]		,       PIL.Image.Image   ):
    _snake_case							:		Union[str, Any]								=  [np.array(i.resize((w, h)		,       resample=PIL_INTERPOLATION["""lanczos"""]   )   )[None, :] for i in image]
    _snake_case							:		Tuple								=  np.concatenate(snake_case__		,       axis=0   )
    _snake_case							:		Union[str, Any]								=  np.array(snake_case__   ).astype(np.floataa   ) / 2_55.0
    _snake_case							:		str								=  image.transpose(0		,       3		,       1		,       2   )
    _snake_case							:		Optional[Any]								=  2.0 * image - 1.0
    _snake_case							:		List[str]								=  torch.from_numpy(snake_case__   )
  elif isinstance(image[0]		,       torch.Tensor   ):
    _snake_case							:		Any								=  torch.cat(snake_case__		,       dim=0   )
  return image


def  UpperCAmelCase__	(snake_case__							: Union[str, Any]		,       snake_case__							: List[str]		,       snake_case__							: Optional[int]		,       snake_case__							: str=0.99_95   ):
  """simple docstring"""

  if not isinstance(snake_case__		,       np.ndarray   ):
    _snake_case							:		Any								=  True
    _snake_case							:		List[str]								=  va.device
    _snake_case							:		Optional[Any]								=  va.cpu().numpy()
    _snake_case							:		Union[str, Any]								=  va.cpu().numpy()

  _snake_case							:		str								=  np.sum(va * va / (np.linalg.norm(snake_case__   ) * np.linalg.norm(snake_case__   ))   )
  if np.abs(snake_case__   ) > DOT_THRESHOLD:
    _snake_case							:		Tuple								=  (1 - t) * va + t * va
  else:
    _snake_case							:		Tuple								=  np.arccos(snake_case__   )
    _snake_case							:		List[Any]								=  np.sin(snake_case__   )
    _snake_case							:		Optional[int]								=  theta_a * t
    _snake_case							:		Optional[int]								=  np.sin(snake_case__   )
    _snake_case							:		str								=  np.sin(theta_a - theta_t   ) / sin_theta_a
    _snake_case							:		Tuple								=  sin_theta_t / sin_theta_a
    _snake_case							:		Union[str, Any]								=  sa * va + sa * va

  if inputs_are_torch:
    _snake_case							:		int								=  torch.from_numpy(snake_case__   ).to(snake_case__   )

  return va


def  UpperCAmelCase__	(snake_case__							: List[Any]		,       snake_case__							: int   ):
  """simple docstring"""

  _snake_case							:		Union[str, Any]								=  F.normalize(snake_case__		,       dim=-1   )
  _snake_case							:		Optional[Any]								=  F.normalize(snake_case__		,       dim=-1   )
  return (x - y).norm(dim=-1   ).div(2   ).arcsin().pow(2   ).mul(2   )


def  UpperCAmelCase__	(snake_case__							: Any		,       snake_case__							: str   ):
  """simple docstring"""

  for param in model.parameters():
    _snake_case							:		Dict								=  value






class  lowercase(					__a			):







 '''simple docstring'''





 def __init__(    self:		Any,				a_:		AutoencoderKL,				a_:		CLIPTextModel,				a_:		CLIPModel,				a_:		CLIPTokenizer,				a_:		UNetaDConditionModel,				a_:		Union[PNDMScheduler, LMSDiscreteScheduler, DDIMScheduler, DPMSolverMultistepScheduler],				a_:		CLIPFeatureExtractor,				a_:		int=None,				a_:		Optional[int]=None,				a_:		List[str]=None,				):




   '''simple docstring'''



   super().__init__()
   self.register_modules(
       vae=a_,				text_encoder=a_,				clip_model=a_,				tokenizer=a_,				unet=a_,				scheduler=a_,				feature_extractor=a_,				coca_model=a_,				coca_tokenizer=a_,				coca_transform=a_,				)
   _snake_case							:		Any								=  (
       feature_extractor.size
       if isinstance(feature_extractor.size,				a_		)
       else feature_extractor.size["""shortest_edge"""]
   )
   _snake_case							:		List[Any]								=  transforms.Normalize(mean=feature_extractor.image_mean,				std=feature_extractor.image_std		)
   set_requires_grad(self.text_encoder,				a_		)
   set_requires_grad(self.clip_model,				a_		)


 def 			UpperCamelCase_       (    self:		Union[str, Any],				a_:		Optional[Union[str, int]] = "auto"		):




   '''simple docstring'''



   if slice_size == "auto":
     # half the attention head size is usually a good trade-off between
     # speed and memory
     _snake_case							:		str								=  self.unet.config.attention_head_dim // 2
   self.unet.set_attention_slice(a_		)


 def 			UpperCamelCase_       (    self:		List[str]		):




   '''simple docstring'''



   self.enable_attention_slicing(a_		)


 def 			UpperCamelCase_       (    self:		List[Any]		):




   '''simple docstring'''



   set_requires_grad(self.vae,				a_		)


 def 			UpperCamelCase_       (    self:		Optional[Any]		):




   '''simple docstring'''



   set_requires_grad(self.vae,				a_		)


 def 			UpperCamelCase_       (    self:		List[Any]		):




   '''simple docstring'''



   set_requires_grad(self.unet,				a_		)


 def 			UpperCamelCase_       (    self:		List[str]		):




   '''simple docstring'''



   set_requires_grad(self.unet,				a_		)


 def 			UpperCamelCase_       (    self:		Optional[Any],				a_:		Optional[int],				a_:		Union[str, Any],				a_:		List[Any]		):




   '''simple docstring'''



   _snake_case							:		Any								=  min(int(num_inference_steps * strength		),				a_		)

   _snake_case							:		Union[str, Any]								=  max(num_inference_steps - init_timestep,				0		)
   _snake_case							:		int								=  self.scheduler.timesteps[t_start:]

   return timesteps, num_inference_steps - t_start


 def 			UpperCamelCase_       (    self:		Union[str, Any],				a_:		List[Any],				a_:		int,				a_:		Tuple,				a_:		List[Any],				a_:		Tuple,				a_:		str=None		):




   '''simple docstring'''



   if not isinstance(a_,				torch.Tensor		):
     raise ValueError(f"`image` has to be of type `torch.Tensor` but is {type(a_		)}"		)

   _snake_case							:		Tuple								=  image.to(device=a_,				dtype=a_		)

   if isinstance(a_,				a_		):
     _snake_case							:		Union[str, Any]								=  [
         self.vae.encode(image[i : i + 1]		).latent_dist.sample(generator[i]		) for i in range(a_		)
     ]
     _snake_case							:		int								=  torch.cat(a_,				dim=0		)
   else:
     _snake_case							:		int								=  self.vae.encode(a_		).latent_dist.sample(a_		)

   # Hardcode 0.18215 because stable-diffusion-2-base has not self.vae.config.scaling_factor
   _snake_case							:		Dict								=  0.18_215 * init_latents
   _snake_case							:		str								=  init_latents.repeat_interleave(a_,				dim=0		)

   _snake_case							:		str								=  randn_tensor(init_latents.shape,				generator=a_,				device=a_,				dtype=a_		)

   # get latents
   _snake_case							:		int								=  self.scheduler.add_noise(a_,				a_,				a_		)
   _snake_case							:		Tuple								=  init_latents

   return latents


 def 			UpperCamelCase_       (    self:		Tuple,				a_:		Any		):




   '''simple docstring'''



   _snake_case							:		Union[str, Any]								=  self.coca_transform(a_		).unsqueeze(0		)
   with torch.no_grad(), torch.cuda.amp.autocast():
     _snake_case							:		Tuple								=  self.coca_model.generate(transformed_image.to(device=self.device,				dtype=self.coca_model.dtype		)		)
   _snake_case							:		Union[str, Any]								=  self.coca_tokenizer.decode(generated[0].cpu().numpy()		)
   return generated.split("""<end_of_text>"""		)[0].replace("""<start_of_text>""",				""""""		).rstrip(""" .,"""		)


 def 			UpperCamelCase_       (    self:		int,				a_:		str,				a_:		Tuple		):




   '''simple docstring'''



   _snake_case							:		List[str]								=  self.feature_extractor.preprocess(a_		)
   _snake_case							:		str								=  torch.from_numpy(clip_image_input["""pixel_values"""][0]		).unsqueeze(0		).to(self.device		).half()
   _snake_case							:		int								=  self.clip_model.get_image_features(a_		)
   _snake_case							:		Optional[Any]								=  image_embeddings_clip / image_embeddings_clip.norm(p=2,				dim=-1,				keepdim=a_		)
   _snake_case							:		Union[str, Any]								=  image_embeddings_clip.repeat_interleave(a_,				dim=0		)
   return image_embeddings_clip


 @torch.enable_grad()
 def 			UpperCamelCase_       (    self:		Any,				a_:		Optional[int],				a_:		Tuple,				a_:		Dict,				a_:		Tuple,				a_:		str,				a_:		List[str],				a_:		str,				):




   '''simple docstring'''



   _snake_case							:		Optional[Any]								=  latents.detach().requires_grad_()

   _snake_case							:		Optional[Any]								=  self.scheduler.scale_model_input(a_,				a_		)

   # predict the noise residual
   _snake_case							:		str								=  self.unet(a_,				a_,				encoder_hidden_states=a_		).sample

   if isinstance(self.scheduler,				(PNDMScheduler, DDIMScheduler, DPMSolverMultistepScheduler)		):
     _snake_case							:		int								=  self.scheduler.alphas_cumprod[timestep]
     _snake_case							:		Optional[Any]								=  1 - alpha_prod_t
     # compute predicted original sample from predicted noise also called
     # "predicted x_0" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf
     _snake_case							:		List[str]								=  (latents - beta_prod_t ** 0.5 * noise_pred) / alpha_prod_t ** 0.5

     _snake_case							:		Union[str, Any]								=  torch.sqrt(a_		)
     _snake_case							:		Optional[Any]								=  pred_original_sample * (fac) + latents * (1 - fac)
   elif isinstance(self.scheduler,				a_		):
     _snake_case							:		Optional[int]								=  self.scheduler.sigmas[index]
     _snake_case							:		int								=  latents - sigma * noise_pred
   else:
     raise ValueError(f"scheduler type {type(self.scheduler		)} not supported"		)

   # Hardcode 0.18215 because stable-diffusion-2-base has not self.vae.config.scaling_factor
   _snake_case							:		List[Any]								=  1 / 0.18_215 * sample
   _snake_case							:		List[str]								=  self.vae.decode(a_		).sample
   _snake_case							:		Tuple								=  (image / 2 + 0.5).clamp(0,				1		)

   _snake_case							:		int								=  transforms.Resize(self.feature_extractor_size		)(a_		)
   _snake_case							:		str								=  self.normalize(a_		).to(latents.dtype		)

   _snake_case							:		Dict								=  self.clip_model.get_image_features(a_		)
   _snake_case							:		Union[str, Any]								=  image_embeddings_clip / image_embeddings_clip.norm(p=2,				dim=-1,				keepdim=a_		)

   _snake_case							:		Any								=  spherical_dist_loss(a_,				a_		).mean() * clip_guidance_scale

   _snake_case							:		str								=  -torch.autograd.grad(a_,				a_		)[0]

   if isinstance(self.scheduler,				a_		):
     _snake_case							:		Dict								=  latents.detach() + grads * (sigma**2)
     _snake_case							:		List[str]								=  noise_pred_original
   else:
     _snake_case							:		List[Any]								=  noise_pred_original - torch.sqrt(a_		) * grads
   return noise_pred, latents


 @torch.no_grad()
 def __call__(    self:		List[Any],				a_:		Union[torch.FloatTensor, PIL.Image.Image],				a_:		Union[torch.FloatTensor, PIL.Image.Image],				a_:		Optional[str] = None,				a_:		Optional[str] = None,				a_:		Optional[int] = 512,				a_:		Optional[int] = 512,				a_:		float = 0.6,				a_:		Optional[int] = 50,				a_:		Optional[float] = 7.5,				a_:		Optional[int] = 1,				a_:		float = 0.0,				a_:		Optional[float] = 100,				a_:		Optional[torch.Generator] = None,				a_:		Optional[str] = "pil",				a_:		bool = True,				a_:		float = 0.8,				a_:		float = 0.1,				a_:		float = 0.1,				):




   '''simple docstring'''



   if isinstance(a_,				a_		) and len(a_		) != batch_size:
     raise ValueError(f"You have passed {batch_size} batch_size, but only {len(a_		)} generators."		)

   if height % 8 != 0 or width % 8 != 0:
     raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}."		)

   if isinstance(a_,				torch.Generator		) and batch_size > 1:
     _snake_case							:		Union[str, Any]								=  [generator] + [None] * (batch_size - 1)

   _snake_case							:		List[str]								=  [
       ("""model""", self.coca_model is None),
       ("""tokenizer""", self.coca_tokenizer is None),
       ("""transform""", self.coca_transform is None),
   ]
   _snake_case							:		List[str]								=  [x[0] for x in coca_is_none if x[1]]
   _snake_case							:		Dict								=  """, """.join(a_		)
   # generate prompts with coca model if prompt is None
   if content_prompt is None:
     if len(a_		):
       raise ValueError(
           f"Content prompt is None and CoCa [{coca_is_none_str}] is None."
           f"Set prompt or pass Coca [{coca_is_none_str}] to DiffusionPipeline."		)
     _snake_case							:		Union[str, Any]								=  self.get_image_description(a_		)
   if style_prompt is None:
     if len(a_		):
       raise ValueError(
           f"Style prompt is None and CoCa [{coca_is_none_str}] is None."
           f" Set prompt or pass Coca [{coca_is_none_str}] to DiffusionPipeline."		)
     _snake_case							:		Union[str, Any]								=  self.get_image_description(a_		)

   # get prompt text embeddings for content and style
   _snake_case							:		Union[str, Any]								=  self.tokenizer(
       a_,				padding="""max_length""",				max_length=self.tokenizer.model_max_length,				truncation=a_,				return_tensors="""pt""",				)
   _snake_case							:		Optional[Any]								=  self.text_encoder(content_text_input.input_ids.to(self.device		)		)[0]

   _snake_case							:		Optional[int]								=  self.tokenizer(
       a_,				padding="""max_length""",				max_length=self.tokenizer.model_max_length,				truncation=a_,				return_tensors="""pt""",				)
   _snake_case							:		Union[str, Any]								=  self.text_encoder(style_text_input.input_ids.to(self.device		)		)[0]

   _snake_case							:		Dict								=  slerp(a_,				a_,				a_		)

   # duplicate text embeddings for each generation per prompt
   _snake_case							:		List[Any]								=  text_embeddings.repeat_interleave(a_,				dim=0		)

   # set timesteps
   _snake_case							:		List[Any]								=  """offset""" in set(inspect.signature(self.scheduler.set_timesteps		).parameters.keys()		)
   _snake_case							:		Dict								=  {}
   if accepts_offset:
     _snake_case							:		Optional[Any]								=  1

   self.scheduler.set_timesteps(a_,				**a_		)
   # Some schedulers like PNDM have timesteps as arrays
   # It's more optimized to move all timesteps to correct device beforehand
   self.scheduler.timesteps.to(self.device		)

   _snake_case		, _snake_case							:		Optional[Any]								=  self.get_timesteps(a_,				a_,				self.device		)
   _snake_case							:		List[str]								=  timesteps[:1].repeat(a_		)

   # Preprocess image
   _snake_case							:		List[Any]								=  preprocess(a_,				a_,				a_		)
   _snake_case							:		Tuple								=  self.prepare_latents(
       a_,				a_,				a_,				text_embeddings.dtype,				self.device,				a_		)

   _snake_case							:		List[Any]								=  preprocess(a_,				a_,				a_		)
   _snake_case							:		Tuple								=  self.prepare_latents(
       a_,				a_,				a_,				text_embeddings.dtype,				self.device,				a_		)

   _snake_case							:		str								=  slerp(a_,				a_,				a_		)

   if clip_guidance_scale > 0:
     _snake_case							:		int								=  self.get_clip_image_embeddings(a_,				a_		)
     _snake_case							:		str								=  self.get_clip_image_embeddings(a_,				a_		)
     _snake_case							:		str								=  slerp(
         a_,				a_,				a_		)

   # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)
   # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`
   # corresponds to doing no classifier free guidance.
   _snake_case							:		List[str]								=  guidance_scale > 1.0
   # get unconditional embeddings for classifier free guidance
   if do_classifier_free_guidance:
     _snake_case							:		str								=  content_text_input.input_ids.shape[-1]
     _snake_case							:		Optional[int]								=  self.tokenizer([""""""],				padding="""max_length""",				max_length=a_,				return_tensors="""pt"""		)
     _snake_case							:		List[str]								=  self.text_encoder(uncond_input.input_ids.to(self.device		)		)[0]
     # duplicate unconditional embeddings for each generation per prompt
     _snake_case							:		List[str]								=  uncond_embeddings.repeat_interleave(a_,				dim=0		)

     # For classifier free guidance, we need to do two forward passes.
     # Here we concatenate the unconditional and text embeddings into a single batch
     # to avoid doing two forward passes
     _snake_case							:		Dict								=  torch.cat([uncond_embeddings, text_embeddings]		)

   # get the initial random noise unless the user supplied it

   # Unlike in other pipelines, latents need to be generated in the target device
   # for 1-to-1 results reproducibility with the CompVis implementation.
   # However this currently doesn't work in `mps`.
   _snake_case							:		str								=  (batch_size, self.unet.config.in_channels, height // 8, width // 8)
   _snake_case							:		List[Any]								=  text_embeddings.dtype
   if latents is None:
     if self.device.type == "mps":
       # randn does not work reproducibly on mps
       _snake_case							:		Dict								=  torch.randn(a_,				generator=a_,				device="""cpu""",				dtype=a_		).to(
           self.device		)
     else:
       _snake_case							:		Any								=  torch.randn(a_,				generator=a_,				device=self.device,				dtype=a_		)
   else:
     if latents.shape != latents_shape:
       raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {latents_shape}"		)
     _snake_case							:		Union[str, Any]								=  latents.to(self.device		)

   # scale the initial noise by the standard deviation required by the scheduler
   _snake_case							:		Union[str, Any]								=  latents * self.scheduler.init_noise_sigma

   # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature
   # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.
   # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502
   # and should be between [0, 1]
   _snake_case							:		Any								=  """eta""" in set(inspect.signature(self.scheduler.step		).parameters.keys()		)
   _snake_case							:		Tuple								=  {}
   if accepts_eta:
     _snake_case							:		Optional[int]								=  eta

   # check if the scheduler accepts generator
   _snake_case							:		Tuple								=  """generator""" in set(inspect.signature(self.scheduler.step		).parameters.keys()		)
   if accepts_generator:
     _snake_case							:		Any								=  generator

   with self.progress_bar(total=a_		):
     for i, t in enumerate(a_		):
       # expand the latents if we are doing classifier free guidance
       _snake_case							:		Dict								=  torch.cat([latents] * 2		) if do_classifier_free_guidance else latents
       _snake_case							:		Optional[Any]								=  self.scheduler.scale_model_input(a_,				a_		)

       # predict the noise residual
       _snake_case							:		Tuple								=  self.unet(a_,				a_,				encoder_hidden_states=a_		).sample

       # perform classifier free guidance
       if do_classifier_free_guidance:
         _snake_case		, _snake_case							:		List[str]								=  noise_pred.chunk(2		)
         _snake_case							:		Any								=  noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)

       # perform clip guidance
       if clip_guidance_scale > 0:
         _snake_case							:		int								=  (
             text_embeddings.chunk(2		)[1] if do_classifier_free_guidance else text_embeddings
         )
         _snake_case		, _snake_case							:		Any								=  self.cond_fn(
             a_,				a_,				a_,				a_,				a_,				a_,				a_,				)

       # compute the previous noisy sample x_t -> x_t-1
       _snake_case							:		int								=  self.scheduler.step(a_,				a_,				a_,				**a_		).prev_sample

        # Hardcode 0.18215 because stable-diffusion-2-base has not self.vae.config.scaling_factor
   _snake_case							:		Optional[Any]								=  1 / 0.18_215 * latents
   _snake_case							:		Tuple								=  self.vae.decode(a_		).sample

   _snake_case							:		Optional[int]								=  (image / 2 + 0.5).clamp(0,				1		)
   _snake_case							:		List[str]								=  image.cpu().permute(0,				2,				3,				1		).numpy()

   if output_type == "pil":
     _snake_case							:		List[Any]								=  self.numpy_to_pil(a_		)

   if not return_dict:
     return (image, None)

   return StableDiffusionPipelineOutput(images=a_,				nsfw_content_detected=a_		)








"""simple docstring"""


import inspect
import unittest

import torch
import torch.nn as nn

from accelerate.hooks import (
    AlignDevicesHook,
    ModelHook,
    SequentialHook,
    add_hook_to_module,
    attach_align_device_hook,
    remove_hook_from_module,
    remove_hook_from_submodules,
)
from accelerate.test_utils import require_multi_gpu






class  lowercase(					nn.Module			):







 '''simple docstring'''





 def __init__(    self:		Optional[int]		):




   '''simple docstring'''



   super().__init__()
   _snake_case							:		List[str]								=  nn.Linear(3,				4		)
   _snake_case							:		int								=  nn.BatchNormad(4		)
   _snake_case							:		List[str]								=  nn.Linear(4,				5		)


 def 			UpperCamelCase_       (    self:		Any,				a_:		Union[str, Any]		):




   '''simple docstring'''



   return self.lineara(self.batchnorm(self.lineara(a_		)		)		)







class  lowercase(					__a			):







 '''simple docstring'''





 def 			UpperCamelCase_       (    self:		Any,				a_:		int,				*a_:		Dict,				**a_:		Dict		):




   '''simple docstring'''



   return (args[0] + 1,) + args[1:], kwargs







class  lowercase(					__a			):







 '''simple docstring'''





 def 			UpperCamelCase_       (    self:		str,				a_:		Tuple,				a_:		Union[str, Any]		):




   '''simple docstring'''



   return output + 1







class  lowercase(					unittest.TestCase			):







 '''simple docstring'''





 def 			UpperCamelCase_       (    self:		Union[str, Any]		):




   '''simple docstring'''



   _snake_case							:		Tuple								=  ModelForTest()
   _snake_case							:		List[str]								=  ModelHook()

   add_hook_to_module(a_,				a_		)
   self.assertEqual(test_model._hf_hook,				a_		)
   self.assertTrue(hasattr(a_,				"""_old_forward"""		)		)

   # Check adding the hook did not change the name or the signature
   self.assertEqual(test_model.forward.__name__,				"""forward"""		)
   self.assertListEqual(list(inspect.signature(test_model.forward		).parameters		),				["""x"""]		)

   remove_hook_from_module(a_		)
   self.assertFalse(hasattr(a_,				"""_hf_hook"""		)		)
   self.assertFalse(hasattr(a_,				"""_old_forward"""		)		)


 def 			UpperCamelCase_       (    self:		Tuple		):




   '''simple docstring'''



   _snake_case							:		Dict								=  ModelForTest()
   _snake_case							:		List[Any]								=  ModelHook()

   add_hook_to_module(a_,				a_		)
   add_hook_to_module(a_,				a_,				append=a_		)

   self.assertEqual(isinstance(test_model._hf_hook,				a_		),				a_		)
   self.assertEqual(len(test_model._hf_hook.hooks		),				2		)
   self.assertTrue(hasattr(a_,				"""_old_forward"""		)		)

   # Check adding the hook did not change the name or the signature
   self.assertEqual(test_model.forward.__name__,				"""forward"""		)
   self.assertListEqual(list(inspect.signature(test_model.forward		).parameters		),				["""x"""]		)

   remove_hook_from_module(a_		)
   self.assertFalse(hasattr(a_,				"""_hf_hook"""		)		)
   self.assertFalse(hasattr(a_,				"""_old_forward"""		)		)


 def 			UpperCamelCase_       (    self:		Tuple		):




   '''simple docstring'''



   _snake_case							:		Optional[Any]								=  ModelForTest()
   _snake_case							:		Optional[Any]								=  torch.randn(2,				3		)
   _snake_case							:		List[Any]								=  test_model(x + 1		)
   _snake_case							:		List[str]								=  test_model(x + 2		)

   _snake_case							:		Any								=  PreForwardHook()
   add_hook_to_module(a_,				a_		)
   _snake_case							:		List[Any]								=  test_model(a_		)
   self.assertTrue(torch.allclose(a_,				a_,				atol=1E-5		)		)

   # Attaching a hook to a model when it already has one replaces, does not chain
   _snake_case							:		List[str]								=  PreForwardHook()
   add_hook_to_module(a_,				a_		)
   _snake_case							:		Tuple								=  test_model(a_		)
   self.assertTrue(torch.allclose(a_,				a_,				atol=1E-5		)		)

   # You need to use the sequential hook to chain two or more hooks
   _snake_case							:		str								=  SequentialHook(PreForwardHook(),				PreForwardHook()		)
   add_hook_to_module(a_,				a_		)

   _snake_case							:		str								=  test_model(a_		)
   assert torch.allclose(a_,				a_,				atol=1E-5		)


 def 			UpperCamelCase_       (    self:		Optional[int]		):




   '''simple docstring'''



   _snake_case							:		Optional[int]								=  ModelForTest()
   _snake_case							:		List[Any]								=  torch.randn(2,				3		)
   _snake_case							:		List[str]								=  test_model(a_		)

   _snake_case							:		List[Any]								=  PostForwardHook()
   add_hook_to_module(a_,				a_		)
   _snake_case							:		Union[str, Any]								=  test_model(a_		)
   self.assertTrue(torch.allclose(a_,				output + 1,				atol=1E-5		)		)

   # Attaching a hook to a model when it already has one replaces, does not chain
   _snake_case							:		Tuple								=  PostForwardHook()
   add_hook_to_module(a_,				a_		)
   _snake_case							:		Optional[Any]								=  test_model(a_		)
   self.assertTrue(torch.allclose(a_,				output + 1,				atol=1E-5		)		)

   # You need to use the sequential hook to chain two or more hooks
   _snake_case							:		Dict								=  SequentialHook(PostForwardHook(),				PostForwardHook()		)
   add_hook_to_module(a_,				a_		)

   _snake_case							:		List[str]								=  test_model(a_		)
   assert torch.allclose(a_,				output + 2,				atol=1E-5		)


 def 			UpperCamelCase_       (    self:		Optional[Any]		):




   '''simple docstring'''



   _snake_case							:		str								=  ModelForTest()
   _snake_case							:		Any								=  torch.randn(2,				3		)
   _snake_case							:		List[str]								=  test_model(a_		)

   _snake_case							:		List[Any]								=  PostForwardHook()
   add_hook_to_module(a_,				a_		)
   _snake_case							:		Dict								=  test_model(a_		)
   self.assertTrue(torch.allclose(a_,				output + 1		)		)
   self.assertTrue(outputa.requires_grad		)

   _snake_case							:		Union[str, Any]								=  True
   _snake_case							:		Dict								=  test_model(a_		)
   self.assertFalse(outputa.requires_grad		)


 @require_multi_gpu
 def 			UpperCamelCase_       (    self:		Tuple		):




   '''simple docstring'''



   _snake_case							:		Dict								=  ModelForTest()
   # Everything is on CPU
   self.assertEqual(model.lineara.weight.device,				torch.device("""cpu"""		)		)
   self.assertEqual(model.batchnorm.weight.device,				torch.device("""cpu"""		)		)
   self.assertEqual(model.lineara.weight.device,				torch.device("""cpu"""		)		)

   # This will move each submodule on different devices
   add_hook_to_module(model.lineara,				AlignDevicesHook(execution_device=0		)		)
   add_hook_to_module(model.batchnorm,				AlignDevicesHook(execution_device=0		)		)
   add_hook_to_module(model.lineara,				AlignDevicesHook(execution_device=1		)		)

   self.assertEqual(model.lineara.weight.device,				torch.device(0		)		)
   self.assertEqual(model.batchnorm.weight.device,				torch.device(0		)		)
   self.assertEqual(model.batchnorm.running_mean.device,				torch.device(0		)		)
   self.assertEqual(model.lineara.weight.device,				torch.device(1		)		)

   # We can still make a forward pass. The input does not need to be on any particular device
   _snake_case							:		Any								=  torch.randn(2,				3		)
   _snake_case							:		Any								=  model(a_		)
   self.assertEqual(output.device,				torch.device(1		)		)

   # We can add a general hook to put back output on same device as input.
   add_hook_to_module(a_,				AlignDevicesHook(io_same_device=a_		)		)
   _snake_case							:		int								=  torch.randn(2,				3		).to(0		)
   _snake_case							:		Optional[Any]								=  model(a_		)
   self.assertEqual(output.device,				torch.device(0		)		)


 def 			UpperCamelCase_       (    self:		Union[str, Any]		):




   '''simple docstring'''



   _snake_case							:		int								=  ModelForTest()

   # Everything is on CPU
   self.assertEqual(model.lineara.weight.device,				torch.device("""cpu"""		)		)
   self.assertEqual(model.batchnorm.weight.device,				torch.device("""cpu"""		)		)
   self.assertEqual(model.lineara.weight.device,				torch.device("""cpu"""		)		)

   # This will move each submodule on different devices
   _snake_case							:		Optional[int]								=  {"""execution_device""": 0 if torch.cuda.is_available() else """cpu""", """offload""": True}

   add_hook_to_module(model.lineara,				AlignDevicesHook(**a_		)		)
   add_hook_to_module(model.batchnorm,				AlignDevicesHook(**a_		)		)
   add_hook_to_module(model.lineara,				AlignDevicesHook(**a_		)		)

   # Parameters have been offloaded, so on the meta device
   self.assertEqual(model.lineara.weight.device,				torch.device("""meta"""		)		)
   self.assertEqual(model.batchnorm.weight.device,				torch.device("""meta"""		)		)
   self.assertEqual(model.lineara.weight.device,				torch.device("""meta"""		)		)
   # Buffers are not included in the offload by default, so are on the execution device
   _snake_case							:		Union[str, Any]								=  torch.device(hook_kwargs["""execution_device"""]		)
   self.assertEqual(model.batchnorm.running_mean.device,				a_		)

   _snake_case							:		str								=  torch.randn(2,				3		)
   _snake_case							:		Optional[int]								=  model(a_		)
   self.assertEqual(output.device,				a_		)

   # Removing hooks loads back the weights in the model.
   remove_hook_from_module(model.lineara		)
   remove_hook_from_module(model.batchnorm		)
   remove_hook_from_module(model.lineara		)
   self.assertEqual(model.lineara.weight.device,				torch.device("""cpu"""		)		)
   self.assertEqual(model.batchnorm.weight.device,				torch.device("""cpu"""		)		)
   self.assertEqual(model.lineara.weight.device,				torch.device("""cpu"""		)		)

   # Now test with buffers included in the offload
   _snake_case							:		Any								=  {
       """execution_device""": 0 if torch.cuda.is_available() else """cpu""",
       """offload""": True,
       """offload_buffers""": True,
   }

   add_hook_to_module(model.lineara,				AlignDevicesHook(**a_		)		)
   add_hook_to_module(model.batchnorm,				AlignDevicesHook(**a_		)		)
   add_hook_to_module(model.lineara,				AlignDevicesHook(**a_		)		)

   # Parameters have been offloaded, so on the meta device, buffers included
   self.assertEqual(model.lineara.weight.device,				torch.device("""meta"""		)		)
   self.assertEqual(model.batchnorm.weight.device,				torch.device("""meta"""		)		)
   self.assertEqual(model.lineara.weight.device,				torch.device("""meta"""		)		)
   self.assertEqual(model.batchnorm.running_mean.device,				torch.device("""meta"""		)		)

   _snake_case							:		List[Any]								=  torch.randn(2,				3		)
   _snake_case							:		Any								=  model(a_		)
   self.assertEqual(output.device,				a_		)

   # Removing hooks loads back the weights in the model.
   remove_hook_from_module(model.lineara		)
   remove_hook_from_module(model.batchnorm		)
   remove_hook_from_module(model.lineara		)
   self.assertEqual(model.lineara.weight.device,				torch.device("""cpu"""		)		)
   self.assertEqual(model.batchnorm.weight.device,				torch.device("""cpu"""		)		)
   self.assertEqual(model.lineara.weight.device,				torch.device("""cpu"""		)		)


 def 			UpperCamelCase_       (    self:		str		):




   '''simple docstring'''



   _snake_case							:		Any								=  ModelForTest()

   # Everything is on CPU
   self.assertEqual(model.lineara.weight.device,				torch.device("""cpu"""		)		)
   self.assertEqual(model.batchnorm.weight.device,				torch.device("""cpu"""		)		)
   self.assertEqual(model.lineara.weight.device,				torch.device("""cpu"""		)		)

   # This will move each submodule on different devices
   _snake_case							:		Tuple								=  0 if torch.cuda.is_available() else """cpu"""
   attach_align_device_hook(a_,				execution_device=a_,				offload=a_		)

   # Parameters have been offloaded, so on the meta device
   self.assertEqual(model.lineara.weight.device,				torch.device("""meta"""		)		)
   self.assertEqual(model.batchnorm.weight.device,				torch.device("""meta"""		)		)
   self.assertEqual(model.lineara.weight.device,				torch.device("""meta"""		)		)
   # Buffers are not included in the offload by default, so are on the execution device
   _snake_case							:		Optional[int]								=  torch.device(a_		)
   self.assertEqual(model.batchnorm.running_mean.device,				a_		)

   _snake_case							:		List[Any]								=  torch.randn(2,				3		)
   _snake_case							:		List[Any]								=  model(a_		)
   self.assertEqual(output.device,				a_		)

   # Removing hooks loads back the weights in the model.
   remove_hook_from_submodules(a_		)
   self.assertEqual(model.lineara.weight.device,				torch.device("""cpu"""		)		)
   self.assertEqual(model.batchnorm.weight.device,				torch.device("""cpu"""		)		)
   self.assertEqual(model.lineara.weight.device,				torch.device("""cpu"""		)		)

   # Now test with buffers included in the offload
   attach_align_device_hook(a_,				execution_device=a_,				offload=a_,				offload_buffers=a_		)

   # Parameters have been offloaded, so on the meta device, buffers included
   self.assertEqual(model.lineara.weight.device,				torch.device("""meta"""		)		)
   self.assertEqual(model.batchnorm.weight.device,				torch.device("""meta"""		)		)
   self.assertEqual(model.lineara.weight.device,				torch.device("""meta"""		)		)
   self.assertEqual(model.batchnorm.running_mean.device,				torch.device("""meta"""		)		)

   _snake_case							:		str								=  torch.randn(2,				3		)
   _snake_case							:		List[Any]								=  model(a_		)
   self.assertEqual(output.device,				a_		)

   # Removing hooks loads back the weights in the model.
   remove_hook_from_submodules(a_		)
   self.assertEqual(model.lineara.weight.device,				torch.device("""cpu"""		)		)
   self.assertEqual(model.batchnorm.weight.device,				torch.device("""cpu"""		)		)
   self.assertEqual(model.lineara.weight.device,				torch.device("""cpu"""		)		)


 def 			UpperCamelCase_       (    self:		Any		):




   '''simple docstring'''



   _snake_case							:		List[str]								=  ModelForTest()

   # Everything is on CPU
   self.assertEqual(model.lineara.weight.device,				torch.device("""cpu"""		)		)
   self.assertEqual(model.batchnorm.weight.device,				torch.device("""cpu"""		)		)
   self.assertEqual(model.lineara.weight.device,				torch.device("""cpu"""		)		)

   # This will move each submodule on different devices
   _snake_case							:		Any								=  0 if torch.cuda.is_available() else """cpu"""
   attach_align_device_hook(
       a_,				execution_device=a_,				offload=a_,				weights_map=model.state_dict()		)

   # Parameters have been offloaded, so on the meta device
   self.assertEqual(model.lineara.weight.device,				torch.device("""meta"""		)		)
   self.assertEqual(model.batchnorm.weight.device,				torch.device("""meta"""		)		)
   self.assertEqual(model.lineara.weight.device,				torch.device("""meta"""		)		)
   # Buffers are not included in the offload by default, so are on the execution device
   _snake_case							:		Optional[Any]								=  torch.device(a_		)
   self.assertEqual(model.batchnorm.running_mean.device,				a_		)

   _snake_case							:		int								=  torch.randn(2,				3		)
   _snake_case							:		str								=  model(a_		)
   self.assertEqual(output.device,				a_		)

   # Removing hooks loads back the weights in the model.
   remove_hook_from_submodules(a_		)
   self.assertEqual(model.lineara.weight.device,				torch.device("""cpu"""		)		)
   self.assertEqual(model.batchnorm.weight.device,				torch.device("""cpu"""		)		)
   self.assertEqual(model.lineara.weight.device,				torch.device("""cpu"""		)		)

   # Now test with buffers included in the offload
   attach_align_device_hook(
       a_,				execution_device=a_,				offload=a_,				weights_map=model.state_dict(),				offload_buffers=a_,				)

   # Parameters have been offloaded, so on the meta device, buffers included
   self.assertEqual(model.lineara.weight.device,				torch.device("""meta"""		)		)
   self.assertEqual(model.batchnorm.weight.device,				torch.device("""meta"""		)		)
   self.assertEqual(model.lineara.weight.device,				torch.device("""meta"""		)		)
   self.assertEqual(model.batchnorm.running_mean.device,				torch.device("""meta"""		)		)

   _snake_case							:		Optional[int]								=  torch.randn(2,				3		)
   _snake_case							:		Any								=  model(a_		)
   self.assertEqual(output.device,				a_		)

   # Removing hooks loads back the weights in the model.
   remove_hook_from_submodules(a_		)
   self.assertEqual(model.lineara.weight.device,				torch.device("""cpu"""		)		)
   self.assertEqual(model.batchnorm.weight.device,				torch.device("""cpu"""		)		)
   self.assertEqual(model.lineara.weight.device,				torch.device("""cpu"""		)		)


