




'''simple docstring'''





import gc
import unittest

import numpy as np
import torch

from diffusers import AutoencoderKL, DDIMScheduler, DiTPipeline, DPMSolverMultistepScheduler, TransformeraDModel
from diffusers.utils import is_xformers_available, load_numpy, slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu

from ..pipeline_params import (
    CLASS_CONDITIONED_IMAGE_GENERATION_BATCH_PARAMS,
    CLASS_CONDITIONED_IMAGE_GENERATION_PARAMS,
)
from ..test_pipelines_common import PipelineTesterMixin


enable_full_determinism()




class 			_snake_case		(     __snake_case						,						unittest.TestCase							):
   lowerCAmelCase_					:				Any							=   DiTPipeline
   lowerCAmelCase_					:				Optional[Any]							=   CLASS_CONDITIONED_IMAGE_GENERATION_PARAMS
   lowerCAmelCase_					:				Optional[Any]							=   PipelineTesterMixin.required_optional_params - {
       "latents",
       "num_images_per_prompt",
       "callback",
       "callback_steps",
   }
   lowerCAmelCase_					:				List[Any]							=   CLASS_CONDITIONED_IMAGE_GENERATION_BATCH_PARAMS
   lowerCAmelCase_					:				Optional[Any]							=   False




   def        lowerCAmelCase__    ( self							)    ->    int:

          '''simple docstring'''




          torch.manual_seed(0							)
          snake_case_      =				TransformeraDModel(
              sample_size=16    ,							num_layers=2    ,							patch_size=4    ,							attention_head_dim=8    ,							num_attention_heads=2    ,							in_channels=4    ,							out_channels=8    ,							attention_bias=lowerCamelCase_    ,							activation_fn="gelu-approximate"    ,							num_embeds_ada_norm=1_000    ,							norm_type="ada_norm_zero"    ,							norm_elementwise_affine=lowerCamelCase_    ,							)
          snake_case_      =				AutoencoderKL()
          snake_case_      =				DDIMScheduler()
          snake_case_      =				{"""transformer""": transformer.eval(), """vae""": vae.eval(), """scheduler""": scheduler}
          return components




   def        lowerCAmelCase__    ( self    ,							a__    ,							a__=0							)    ->    int:

          '''simple docstring'''




          if str(lowerCamelCase_							).startswith("mps"							):
                 snake_case_      =				torch.manual_seed(lowerCamelCase_							)
          else:
                 snake_case_      =				torch.Generator(device=lowerCamelCase_							).manual_seed(lowerCamelCase_							)
          snake_case_      =				{
              """class_labels""": [1],
              """generator""": generator,
              """num_inference_steps""": 2,
              """output_type""": """numpy""",
          }
          return inputs




   def        lowerCAmelCase__    ( self							)    ->    Optional[Any]:

          '''simple docstring'''




          snake_case_      =				"""cpu"""

          snake_case_      =				self.get_dummy_components()
          snake_case_      =				self.pipeline_class(**lowerCamelCase_							)
          pipe.to(lowerCamelCase_							)
          pipe.set_progress_bar_config(disable=lowerCamelCase_							)

          snake_case_      =				self.get_dummy_inputs(lowerCamelCase_							)
          snake_case_      =				pipe(**lowerCamelCase_							).images
          snake_case_      =				image[0, -3:, -3:, -1]

          self.assertEqual(image.shape    ,							(1, 16, 16, 3)							)
          snake_case_      =				np.array([0.2_9_4_6, 0.6_6_0_1, 0.4_3_2_9, 0.3_2_9_6, 0.4_1_4_4, 0.5_3_1_9, 0.7_2_7_3, 0.5_0_1_3, 0.4_4_5_7]							)
          snake_case_      =				np.abs(image_slice.flatten() - expected_slice							).max()
          self.assertLessEqual(lowerCamelCase_    ,							1e-3							)




   def        lowerCAmelCase__    ( self							)    ->    Optional[int]:

          '''simple docstring'''




          self._test_inference_batch_single_identical(relax_max_difference=lowerCamelCase_    ,							expected_max_diff=1e-3							)




   @unittest.skipIf(
       torch_device != "cuda" or not is_xformers_available()    ,							reason="XFormers attention is only available with CUDA and `xformers` installed"    ,							)
   def        lowerCAmelCase__    ( self							)    ->    Any:

          '''simple docstring'''




          self._test_xformers_attention_forwardGenerator_pass(expected_max_diff=1e-3							)




@require_torch_gpu
@slow
class 			_snake_case		(     unittest.TestCase							):




   def        lowerCAmelCase__    ( self							)    ->    str:

          '''simple docstring'''




          super().tearDown()
          gc.collect()
          torch.cuda.empty_cache()




   def        lowerCAmelCase__    ( self							)    ->    List[str]:

          '''simple docstring'''




          snake_case_      =				torch.manual_seed(0							)

          snake_case_      =				DiTPipeline.from_pretrained("facebook/DiT-XL-2-256"							)
          pipe.to("cuda"							)

          snake_case_      =				["""vase""", """umbrella""", """white shark""", """white wolf"""]
          snake_case_      =				pipe.get_label_ids(lowerCamelCase_							)

          snake_case_      =				pipe(lowerCamelCase_    ,							generator=lowerCamelCase_    ,							num_inference_steps=40    ,							output_type="np"							).images

          for word, image in zip(lowerCamelCase_    ,							lowerCamelCase_							):
                 snake_case_      =				load_numpy(
                     F'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/dit/{word}.npy'							)
                 assert np.abs((expected_image - image).max()							) < 1e-2




   def        lowerCAmelCase__    ( self							)    ->    Optional[int]:

          '''simple docstring'''




          snake_case_      =				DiTPipeline.from_pretrained("facebook/DiT-XL-2-512"							)
          snake_case_      =				DPMSolverMultistepScheduler.from_config(pipe.scheduler.config							)
          pipe.to("cuda"							)

          snake_case_      =				["""vase""", """umbrella"""]
          snake_case_      =				pipe.get_label_ids(lowerCamelCase_							)

          snake_case_      =				torch.manual_seed(0							)
          snake_case_      =				pipe(lowerCamelCase_    ,							generator=lowerCamelCase_    ,							num_inference_steps=25    ,							output_type="np"							).images

          for word, image in zip(lowerCamelCase_    ,							lowerCamelCase_							):
                 snake_case_      =				load_numpy(
                     "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main"
                     F'/dit/{word}_512.npy'							)

                 assert np.abs((expected_image - image).max()							) < 1e-1










'''simple docstring'''





import os

import torch

from ..logging import get_logger
from .constants import FSDP_PYTORCH_VERSION, MODEL_NAME, OPTIMIZER_NAME
from .versions import is_torch_version


if is_torch_version(">=", FSDP_PYTORCH_VERSION):
		import torch.distributed.checkpoint as dist_cp
		from torch.distributed.checkpoint.default_planner import DefaultLoadPlanner, DefaultSavePlanner
		from torch.distributed.checkpoint.optimizer import load_sharded_optimizer_state_dict
		from torch.distributed.fsdp.fully_sharded_data_parallel import FullyShardedDataParallel as FSDP
		from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType


_SCREAMING_SNAKE_CASE							:       Tuple						=			get_logger(__name__)


def 					UpperCamelCase_(     snake_case							:	Optional[Any]							,    snake_case							:	Optional[Any]							,    snake_case							:	Optional[Any]							,    snake_case							:	int							,    snake_case							:	List[Any]=0					):

							'''simple docstring'''






							os.makedirs(snake_case							,    exist_ok=snake_case					)
							with FSDP.state_dict_type(
							    snake_case							,    fsdp_plugin.state_dict_type							,    fsdp_plugin.state_dict_config							,    fsdp_plugin.optim_state_dict_config					):
														snake_case_      =				model.state_dict()
														if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
																					snake_case_      =				f'{MODEL_NAME}.bin' if model_index == 0 else f'{MODEL_NAME}_{model_index}.bin'
																					snake_case_      =				os.path.join(snake_case							,    snake_case					)
																					if accelerator.process_index == 0:
																												logger.info(f'Saving model to {output_model_file}'					)
																												torch.save(snake_case							,    snake_case					)
																												logger.info(f'Model saved to {output_model_file}'					)
														elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT:
																					snake_case_      =				(
																					    f'{MODEL_NAME}_rank{accelerator.process_index}.bin'
																					    if model_index == 0
																					    else f'{MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin'
																					)
																					snake_case_      =				os.path.join(snake_case							,    snake_case					)
																					logger.info(f'Saving model to {output_model_file}'					)
																					torch.save(snake_case							,    snake_case					)
																					logger.info(f'Model saved to {output_model_file}'					)
														elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT:
																					snake_case_      =				os.path.join(snake_case							,    f'{MODEL_NAME}_{model_index}'					)
																					os.makedirs(snake_case							,    exist_ok=snake_case					)
																					logger.info(f'Saving model to {ckpt_dir}'					)
																					snake_case_      =				{"model": state_dict}

																					dist_cp.save_state_dict(
																					    state_dict=snake_case							,    storage_writer=dist_cp.FileSystemWriter(snake_case					)							,    planner=DefaultSavePlanner()							,    )
																					logger.info(f'Model saved to {ckpt_dir}'					)


def 					UpperCamelCase_(     snake_case							:	Optional[int]							,    snake_case							:	Optional[Any]							,    snake_case							:	int							,    snake_case							:	Union[str, Any]							,    snake_case							:	Any=0					):

							'''simple docstring'''






							accelerator.wait_for_everyone()
							with FSDP.state_dict_type(
							    snake_case							,    fsdp_plugin.state_dict_type							,    fsdp_plugin.state_dict_config							,    fsdp_plugin.optim_state_dict_config					):
														if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
																					if type(snake_case					) != FSDP and accelerator.process_index != 0:
																												if not fsdp_plugin.sync_module_states:
																																			raise ValueError(
																																			    "Set the `sync_module_states` flag to `True` so that model states are synced across processes when "
																																			    "initializing FSDP object"					)
																												return
																					snake_case_      =				f'{MODEL_NAME}.bin' if model_index == 0 else f'{MODEL_NAME}_{model_index}.bin'
																					snake_case_      =				os.path.join(snake_case							,    snake_case					)
																					logger.info(f'Loading model from {input_model_file}'					)
																					snake_case_      =				torch.load(snake_case					)
																					logger.info(f'Model loaded from {input_model_file}'					)
														elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT:
																					snake_case_      =				(
																					    f'{MODEL_NAME}_rank{accelerator.process_index}.bin'
																					    if model_index == 0
																					    else f'{MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin'
																					)
																					snake_case_      =				os.path.join(snake_case							,    snake_case					)
																					logger.info(f'Loading model from {input_model_file}'					)
																					snake_case_      =				torch.load(snake_case					)
																					logger.info(f'Model loaded from {input_model_file}'					)
														elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT:
																					snake_case_      =				(
																					    os.path.join(snake_case							,    f'{MODEL_NAME}_{model_index}'					)
																					    if f'{MODEL_NAME}' not in input_dir
																					    else input_dir
																					)
																					logger.info(f'Loading model from {ckpt_dir}'					)
																					snake_case_      =				{"model": model.state_dict()}
																					dist_cp.load_state_dict(
																					    state_dict=snake_case							,    storage_reader=dist_cp.FileSystemReader(snake_case					)							,    planner=DefaultLoadPlanner()							,    )
																					snake_case_      =				state_dict["model"]
																					logger.info(f'Model loaded from {ckpt_dir}'					)
														model.load_state_dict(snake_case					)


def 					UpperCamelCase_(     snake_case							:	str							,    snake_case							:	List[str]							,    snake_case							:	Any							,    snake_case							:	Tuple							,    snake_case							:	Optional[Any]							,    snake_case							:	Tuple=0					):

							'''simple docstring'''






							os.makedirs(snake_case							,    exist_ok=snake_case					)
							with FSDP.state_dict_type(
							    snake_case							,    fsdp_plugin.state_dict_type							,    fsdp_plugin.state_dict_config							,    fsdp_plugin.optim_state_dict_config					):
														snake_case_      =				FSDP.optim_state_dict(snake_case							,    snake_case					)
														if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
																					if accelerator.process_index == 0:
																												snake_case_      =				(
																												    f'{OPTIMIZER_NAME}.bin' if optimizer_index == 0 else f'{OPTIMIZER_NAME}_{optimizer_index}.bin'
																												)
																												snake_case_      =				os.path.join(snake_case							,    snake_case					)
																												logger.info(f'Saving Optimizer state to {output_optimizer_file}'					)
																												torch.save(snake_case							,    snake_case					)
																												logger.info(f'Optimizer state saved in {output_optimizer_file}'					)
														else:
																					snake_case_      =				os.path.join(snake_case							,    f'{OPTIMIZER_NAME}_{optimizer_index}'					)
																					os.makedirs(snake_case							,    exist_ok=snake_case					)
																					logger.info(f'Saving Optimizer state to {ckpt_dir}'					)
																					dist_cp.save_state_dict(
																					    state_dict={"optimizer": optim_state}							,    storage_writer=dist_cp.FileSystemWriter(snake_case					)							,    planner=DefaultSavePlanner()							,    )
																					logger.info(f'Optimizer state saved in {ckpt_dir}'					)




def 					UpperCamelCase_(     snake_case							:	Optional[Any]							,    snake_case							:	List[str]							,    snake_case							:	Union[str, Any]							,    snake_case							:	int							,    snake_case							:	Optional[int]							,    snake_case							:	Union[str, Any]=0					):

							'''simple docstring'''






							accelerator.wait_for_everyone()
							with FSDP.state_dict_type(
							    snake_case							,    fsdp_plugin.state_dict_type							,    fsdp_plugin.state_dict_config							,    fsdp_plugin.optim_state_dict_config					):
														if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
																					snake_case_      =				None
																					# below check should work but currently it isn't working (mostly opytorch issue),
																					# in the meantime disabling it at the cost of excess memory usage
																					# if accelerator.process_index == 0 or not fsdp_plugin.optim_state_dict_config.rank0_only:
																					snake_case_      =				(
																					    f'{OPTIMIZER_NAME}.bin' if optimizer_index == 0 else f'{OPTIMIZER_NAME}_{optimizer_index}.bin'
																					)
																					snake_case_      =				os.path.join(snake_case							,    snake_case					)
																					logger.info(f'Loading Optimizer state from {input_optimizer_file}'					)
																					snake_case_      =				torch.load(snake_case					)
																					logger.info(f'Optimizer state loaded from {input_optimizer_file}'					)
														else:
																					snake_case_      =				(
																					    os.path.join(snake_case							,    f'{OPTIMIZER_NAME}_{optimizer_index}'					)
																					    if f'{OPTIMIZER_NAME}' not in input_dir
																					    else input_dir
																					)
																					logger.info(f'Loading Optimizer from {ckpt_dir}'					)
																					snake_case_      =				load_sharded_optimizer_state_dict(
																					    model_state_dict=model.state_dict()							,    optimizer_key="optimizer"							,    storage_reader=dist_cp.FileSystemReader(snake_case					)							,    )
																					snake_case_      =				optim_state["optimizer"]
																					logger.info(f'Optimizer loaded from {ckpt_dir}'					)
														snake_case_      =				FSDP.optim_state_dict_to_load(snake_case							,    snake_case							,    snake_case					)
														optimizer.load_state_dict(snake_case					)




