
import inspect
import os

import torch
from transformers import AutoModel
from transformers.testing_utils import mockenv_context
from transformers.trainer_utils import set_seed

import accelerate
from accelerate.accelerator import Accelerator
from accelerate.state import AcceleratorState
from accelerate.test_utils.testing import (
    AccelerateTestCase,
    TempDirTestCase,
    execute_subprocess_async,
    require_cuda,
    require_fsdp,
    require_multi_gpu,
    slow,
)
from accelerate.utils.constants import (
    FSDP_AUTO_WRAP_POLICY,
    FSDP_BACKWARD_PREFETCH,
    FSDP_SHARDING_STRATEGY,
    FSDP_STATE_DICT_TYPE,
)
from accelerate.utils.dataclasses import FullyShardedDataParallelPlugin
from accelerate.utils.other import patch_environment


set_seed(42)

_UpperCAmelCase :			Optional[Any]              =	"bert-base-cased"
_UpperCAmelCase :			Dict              =	"fp16"
_UpperCAmelCase :			int              =	"bf16"
_UpperCAmelCase :			Optional[Any]              =	[FPaa, BFaa]


@require_fsdp
@require_cuda
class     lowercase (		_SCREAMING_SNAKE_CASE       ):



		def __UpperCamelCase			(			self	)  -> str:
				"""simple docstring"""
				super().setUp()

				UpperCamelCase          =					dict(
				    ACCELERATE_USE_FSDP='true'    ,				MASTER_ADDR='localhost'    ,				MASTER_PORT='10999'    ,				RANK='0'    ,				LOCAL_RANK='0'    ,				WORLD_SIZE='1'    ,				)



		def __UpperCamelCase			(			self	)  -> List[str]:
				"""simple docstring"""
				from torch.distributed.fsdp.fully_sharded_data_parallel import ShardingStrategy

				for i, strategy in enumerate(A_	):
						UpperCamelCase          =					self.dist_env.copy()
						UpperCamelCase          =					F'''{i + 1}'''
						UpperCamelCase          =					strategy
						with mockenv_context(**A_	):
								UpperCamelCase          =					FullyShardedDataParallelPlugin()
								self.assertEqual(fsdp_plugin.sharding_strategy    ,				ShardingStrategy(i + 1	)	)



		def __UpperCamelCase			(			self	)  -> str:
				"""simple docstring"""
				from torch.distributed.fsdp.fully_sharded_data_parallel import BackwardPrefetch

				for i, prefetch_policy in enumerate(A_	):
						UpperCamelCase          =					self.dist_env.copy()
						UpperCamelCase          =					prefetch_policy
						with mockenv_context(**A_	):
								UpperCamelCase          =					FullyShardedDataParallelPlugin()
								if prefetch_policy == "NO_PREFETCH":
										self.assertIsNone(fsdp_plugin.backward_prefetch	)
								else:
										self.assertEqual(fsdp_plugin.backward_prefetch    ,				BackwardPrefetch(i + 1	)	)



		def __UpperCamelCase			(			self	)  -> Optional[Any]:
				"""simple docstring"""
				from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType

				for i, state_dict_type in enumerate(A_	):
						UpperCamelCase          =					self.dist_env.copy()
						UpperCamelCase          =					state_dict_type
						with mockenv_context(**A_	):
								UpperCamelCase          =					FullyShardedDataParallelPlugin()
								self.assertEqual(fsdp_plugin.state_dict_type    ,				StateDictType(i + 1	)	)
								if state_dict_type == "FULL_STATE_DICT":
										self.assertTrue(fsdp_plugin.state_dict_config.offload_to_cpu	)
										self.assertTrue(fsdp_plugin.state_dict_config.ranka_only	)



		def __UpperCamelCase			(			self	)  -> Any:
				"""simple docstring"""
				UpperCamelCase          =					AutoModel.from_pretrained(A_	)
				for policy in FSDP_AUTO_WRAP_POLICY:
						UpperCamelCase          =					self.dist_env.copy()
						UpperCamelCase          =					policy
						if policy == "TRANSFORMER_BASED_WRAP":
								UpperCamelCase          =					'BertLayer'
						elif policy == "SIZE_BASED_WRAP":
								UpperCamelCase          =					'2000'
						with mockenv_context(**A_	):
								UpperCamelCase          =					FullyShardedDataParallelPlugin()
								fsdp_plugin.set_auto_wrap_policy(A_	)
								if policy == "NO_WRAP":
										self.assertIsNone(fsdp_plugin.auto_wrap_policy	)
								else:
										self.assertIsNotNone(fsdp_plugin.auto_wrap_policy	)

				UpperCamelCase          =					self.dist_env.copy()
				UpperCamelCase          =					'TRANSFORMER_BASED_WRAP'
				UpperCamelCase          =					'T5Layer'
				with mockenv_context(**A_	):
						UpperCamelCase          =					FullyShardedDataParallelPlugin()
						with self.assertRaises(A_	) as cm:
								fsdp_plugin.set_auto_wrap_policy(A_	)
						self.assertTrue('Could not find the transformer layer class to wrap in the model.' in str(cm.exception	)	)

				UpperCamelCase          =					self.dist_env.copy()
				UpperCamelCase          =					'SIZE_BASED_WRAP'
				UpperCamelCase          =					'0'
				with mockenv_context(**A_	):
						UpperCamelCase          =					FullyShardedDataParallelPlugin()
						fsdp_plugin.set_auto_wrap_policy(A_	)
						self.assertIsNone(fsdp_plugin.auto_wrap_policy	)



		def __UpperCamelCase			(			self	)  -> List[Any]:
				"""simple docstring"""
				from torch.distributed.fsdp.fully_sharded_data_parallel import MixedPrecision
				from torch.distributed.fsdp.sharded_grad_scaler import ShardedGradScaler

				for mp_dtype in dtypes:
						UpperCamelCase          =					self.dist_env.copy()
						UpperCamelCase          =					mp_dtype
						with mockenv_context(**A_	):
								UpperCamelCase          =					Accelerator()
								if mp_dtype == "fp16":
										UpperCamelCase          =					torch.floataa
								elif mp_dtype == "bf16":
										UpperCamelCase          =					torch.bfloataa
								UpperCamelCase          =					MixedPrecision(param_dtype=A_    ,				reduce_dtype=A_    ,				buffer_dtype=A_	)
								self.assertEqual(accelerator.state.fsdp_plugin.mixed_precision_policy    ,				A_	)
								if mp_dtype == FPaa:
										self.assertTrue(isinstance(accelerator.scaler    ,				A_	)	)
								elif mp_dtype == BFaa:
										self.assertIsNone(accelerator.scaler	)
								AcceleratorState._reset_state(A_	)



		def __UpperCamelCase			(			self	)  -> Union[str, Any]:
				"""simple docstring"""
				from torch.distributed.fsdp.fully_sharded_data_parallel import CPUOffload

				for flag in [True, False]:
						UpperCamelCase          =					self.dist_env.copy()
						UpperCamelCase          =					str(A_	).lower()
						with mockenv_context(**A_	):
								UpperCamelCase          =					FullyShardedDataParallelPlugin()
								self.assertEqual(fsdp_plugin.cpu_offload    ,				CPUOffload(offload_params=A_	)	)






@require_fsdp
@require_multi_gpu
@slow
class     lowercase (		_SCREAMING_SNAKE_CASE       ):



		def __UpperCamelCase			(			self	)  -> Union[str, Any]:
				"""simple docstring"""
				super().setUp()
				UpperCamelCase          =					0.82
				UpperCamelCase          =					[
				    'fsdp_shard_grad_op_transformer_based_wrap',
				    'fsdp_full_shard_transformer_based_wrap',
				]
				UpperCamelCase          =					{
				    'multi_gpu_fp16': 3_200,
				    'fsdp_shard_grad_op_transformer_based_wrap_fp16': 2_000,
				    'fsdp_full_shard_transformer_based_wrap_fp16': 1_900,
				    # Disabling below test as it overwhelms the RAM memory usage
				    # on CI self-hosted runner leading to tests getting killed.
				    # "fsdp_full_shard_cpu_offload_transformer_based_wrap_fp32": 1500,  # fp16 was leading to indefinite hang
				}
				UpperCamelCase          =					160
				UpperCamelCase          =					160

				UpperCamelCase          =					inspect.getfile(accelerate.test_utils	)
				UpperCamelCase          =					os.path.sep.join(mod_file.split(os.path.sep	)[:-1] + ['scripts', 'external_deps']	)



		def __UpperCamelCase			(			self	)  -> Any:
				"""simple docstring"""
				UpperCamelCase          =					os.path.join(self.test_scripts_folder    ,				'test_performance.py'	)
				UpperCamelCase          =					['accelerate', 'launch', '--num_processes=2', '--num_machines=1', '--machine_rank=0', '--use_fsdp']
				for config in self.performance_configs:
						UpperCamelCase          =					cmd.copy()
						for i, strategy in enumerate(A_	):
								if strategy.lower() in config:
										cmd_config.append(F'''--fsdp_sharding_strategy={i+1}'''	)
										break

						if "fp32" in config:
								cmd_config.append('--mixed_precision=no'	)
						else:
								cmd_config.append('--mixed_precision=fp16'	)

						if "cpu_offload" in config:
								cmd_config.append('--fsdp_offload_params=True'	)

						for policy in FSDP_AUTO_WRAP_POLICY:
								if policy.lower() in config:
										cmd_config.append(F'''--fsdp_auto_wrap_policy={policy}'''	)
										break

						if policy == "TRANSFORMER_BASED_WRAP":
								cmd_config.append('--fsdp_transformer_layer_cls_to_wrap=BertLayer'	)
						elif policy == "SIZE_BASED_WRAP":
								cmd_config.append('--fsdp_min_num_params=2000'	)

						cmd_config.extend(
						    [
						        self.test_file_path,
						        F'''--output_dir={self.tmpdir}''',
						        F'''--performance_lower_bound={self.performance_lower_bound}''',
						    ]	)
						with patch_environment(omp_num_threads=1	):
								execute_subprocess_async(A_    ,				env=os.environ.copy()	)



		def __UpperCamelCase			(			self	)  -> List[str]:
				"""simple docstring"""
				UpperCamelCase          =					os.path.join(self.test_scripts_folder    ,				'test_checkpointing.py'	)
				UpperCamelCase          =					[
				    'accelerate',
				    'launch',
				    '--num_processes=2',
				    '--num_machines=1',
				    '--machine_rank=0',
				    '--use_fsdp',
				    '--mixed_precision=fp16',
				    '--fsdp_transformer_layer_cls_to_wrap=BertLayer',
				]

				for i, strategy in enumerate(A_	):
						UpperCamelCase          =					cmd.copy()
						cmd_config.append(F'''--fsdp_sharding_strategy={i+1}'''	)
						if strategy != "FULL_SHARD":
								continue
						UpperCamelCase          =					len(A_	)
						for state_dict_type in FSDP_STATE_DICT_TYPE:
								UpperCamelCase          =					cmd_config[:state_dict_config_index]
								cmd_config.append(F'''--fsdp_state_dict_type={state_dict_type}'''	)
								cmd_config.extend(
								    [
								        self.test_file_path,
								        F'''--output_dir={self.tmpdir}''',
								        '--partial_train_epoch=1',
								    ]	)
								with patch_environment(omp_num_threads=1	):
										execute_subprocess_async(A_    ,				env=os.environ.copy()	)

								UpperCamelCase          =					cmd_config[:-1]
								UpperCamelCase          =					os.path.join(self.tmpdir    ,				'epoch_0'	)
								cmd_config.extend(
								    [
								        F'''--resume_from_checkpoint={resume_from_checkpoint}''',
								    ]	)
								with patch_environment(omp_num_threads=1	):
										execute_subprocess_async(A_    ,				env=os.environ.copy()	)



		def __UpperCamelCase			(			self	)  -> Union[str, Any]:
				"""simple docstring"""
				UpperCamelCase          =					os.path.join(self.test_scripts_folder    ,				'test_peak_memory_usage.py'	)
				UpperCamelCase          =					[
				    'accelerate',
				    'launch',
				    '--num_processes=2',
				    '--num_machines=1',
				    '--machine_rank=0',
				]
				for spec, peak_mem_upper_bound in self.peak_memory_usage_upper_bound.items():
						UpperCamelCase          =					cmd.copy()
						if "fp16" in spec:
								cmd_config.extend(['--mixed_precision=fp16']	)
						else:
								cmd_config.extend(['--mixed_precision=no']	)

						if "multi_gpu" in spec:
								continue
						else:
								cmd_config.extend(['--use_fsdp']	)
								for i, strategy in enumerate(A_	):
										if strategy.lower() in spec:
												cmd_config.append(F'''--fsdp_sharding_strategy={i+1}'''	)
												break

								if "cpu_offload" in spec:
										cmd_config.append('--fsdp_offload_params=True'	)

								for policy in FSDP_AUTO_WRAP_POLICY:
										if policy.lower() in spec:
												cmd_config.append(F'''--fsdp_auto_wrap_policy={policy}'''	)
												break

								if policy == "TRANSFORMER_BASED_WRAP":
										cmd_config.append('--fsdp_transformer_layer_cls_to_wrap=BertLayer'	)
								elif policy == "SIZE_BASED_WRAP":
										cmd_config.append('--fsdp_min_num_params=2000'	)

						cmd_config.extend(
						    [
						        self.test_file_path,
						        F'''--output_dir={self.tmpdir}''',
						        F'''--peak_memory_upper_bound={peak_mem_upper_bound}''',
						        F'''--n_train={self.n_train}''',
						        F'''--n_val={self.n_val}''',
						    ]	)
						with patch_environment(omp_num_threads=1	):
								execute_subprocess_async(A_    ,				env=os.environ.copy()	)


import gc
import random
import unittest

import numpy as np
import torch
from PIL import Image

from diffusers import (
    DDIMScheduler,
    KandinskyVaaControlnetImgaImgPipeline,
    KandinskyVaaPriorEmbaEmbPipeline,
    UNetaDConditionModel,
    VQModel,
)
from diffusers.utils import floats_tensor, load_image, load_numpy, slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu

from ..test_pipelines_common import PipelineTesterMixin, assert_mean_pixel_difference


enable_full_determinism()


class     lowercase (		_SCREAMING_SNAKE_CASE  ,       unittest.TestCase       ):
		__lowercase	:	List[Any]     					=    KandinskyVaaControlnetImgaImgPipeline
		__lowercase	:	Optional[Any]     					=    ["image_embeds", "negative_image_embeds", "image", "hint"]
		__lowercase	:	Any     					=    ["image_embeds", "negative_image_embeds", "image", "hint"]
		__lowercase	:	Union[str, Any]     					=    [
		    "generator",
		    "height",
		    "width",
		    "strength",
		    "guidance_scale",
		    "num_inference_steps",
		    "return_dict",
		    "guidance_scale",
		    "num_images_per_prompt",
		    "output_type",
		    "return_dict",
		]
		__lowercase	:	Optional[int]     					=    False



		@property
		def __UpperCamelCase			(			self	)  -> Optional[int]:
				"""simple docstring"""
				return 32



		@property
		def __UpperCamelCase			(			self	)  -> str:
				"""simple docstring"""
				return 32



		@property
		def __UpperCamelCase			(			self	)  -> int:
				"""simple docstring"""
				return self.time_input_dim



		@property
		def __UpperCamelCase			(			self	)  -> int:
				"""simple docstring"""
				return self.time_input_dim * 4



		@property
		def __UpperCamelCase			(			self	)  -> Optional[Any]:
				"""simple docstring"""
				return 100



		@property
		def __UpperCamelCase			(			self	)  -> str:
				"""simple docstring"""
				torch.manual_seed(0	)

				UpperCamelCase          =					{
				    'in_channels': 8,
				    # Out channels is double in channels because predicts mean and variance
				    'out_channels': 8,
				    'addition_embed_type': 'image_hint',
				    'down_block_types': ('ResnetDownsampleBlock2D', 'SimpleCrossAttnDownBlock2D'),
				    'up_block_types': ('SimpleCrossAttnUpBlock2D', 'ResnetUpsampleBlock2D'),
				    'mid_block_type': 'UNetMidBlock2DSimpleCrossAttn',
				    'block_out_channels': (self.block_out_channels_a, self.block_out_channels_a * 2),
				    'layers_per_block': 1,
				    'encoder_hid_dim': self.text_embedder_hidden_size,
				    'encoder_hid_dim_type': 'image_proj',
				    'cross_attention_dim': self.cross_attention_dim,
				    'attention_head_dim': 4,
				    'resnet_time_scale_shift': 'scale_shift',
				    'class_embed_type': None,
				}

				UpperCamelCase          =					UNetaDConditionModel(**A_	)
				return model



		@property
		def __UpperCamelCase			(			self	)  -> int:
				"""simple docstring"""
				return {
				    "block_out_channels": [32, 32, 64, 64],
				    "down_block_types": [
				        "DownEncoderBlock2D",
				        "DownEncoderBlock2D",
				        "DownEncoderBlock2D",
				        "AttnDownEncoderBlock2D",
				    ],
				    "in_channels": 3,
				    "latent_channels": 4,
				    "layers_per_block": 1,
				    "norm_num_groups": 8,
				    "norm_type": "spatial",
				    "num_vq_embeddings": 12,
				    "out_channels": 3,
				    "up_block_types": ["AttnUpDecoderBlock2D", "UpDecoderBlock2D", "UpDecoderBlock2D", "UpDecoderBlock2D"],
				    "vq_embed_dim": 4,
				}



		@property
		def __UpperCamelCase			(			self	)  -> Optional[Any]:
				"""simple docstring"""
				torch.manual_seed(0	)
				UpperCamelCase          =					VQModel(**self.dummy_movq_kwargs	)
				return model



		def __UpperCamelCase			(			self	)  -> Optional[int]:
				"""simple docstring"""
				UpperCamelCase          =					self.dummy_unet
				UpperCamelCase          =					self.dummy_movq

				UpperCamelCase          =					{
				    'num_train_timesteps': 1_000,
				    'beta_schedule': 'linear',
				    'beta_start': 0.0_0085,
				    'beta_end': 0.012,
				    'clip_sample': False,
				    'set_alpha_to_one': False,
				    'steps_offset': 0,
				    'prediction_type': 'epsilon',
				    'thresholding': False,
				}

				UpperCamelCase          =					DDIMScheduler(**A_	)

				UpperCamelCase          =					{
				    'unet': unet,
				    'scheduler': scheduler,
				    'movq': movq,
				}

				return components



		def __UpperCamelCase			(			self    ,				A_    ,				A_=0	)  -> List[Any]:
				"""simple docstring"""
				UpperCamelCase          =					floats_tensor((1, self.text_embedder_hidden_size)    ,				rng=random.Random(A_	)	).to(A_	)
				UpperCamelCase          =					floats_tensor((1, self.text_embedder_hidden_size)    ,				rng=random.Random(seed + 1	)	).to(
				    A_	)
				# create init_image
				UpperCamelCase          =					floats_tensor((1, 3, 64, 64)    ,				rng=random.Random(A_	)	).to(A_	)
				UpperCamelCase          =					image.cpu().permute(0    ,				2    ,				3    ,				1	)[0]
				UpperCamelCase          =					Image.fromarray(np.uinta(A_	)	).convert('RGB'	).resize((256, 256)	)
				# create hint
				UpperCamelCase          =					floats_tensor((1, 3, 64, 64)    ,				rng=random.Random(A_	)	).to(A_	)

				if str(A_	).startswith('mps'	):
						UpperCamelCase          =					torch.manual_seed(A_	)
				else:
						UpperCamelCase          =					torch.Generator(device=A_	).manual_seed(A_	)
				UpperCamelCase          =					{
				    'image': init_image,
				    'image_embeds': image_embeds,
				    'negative_image_embeds': negative_image_embeds,
				    'hint': hint,
				    'generator': generator,
				    'height': 64,
				    'width': 64,
				    'num_inference_steps': 10,
				    'guidance_scale': 7.0,
				    'strength': 0.2,
				    'output_type': 'np',
				}
				return inputs



		def __UpperCamelCase			(			self	)  -> Any:
				"""simple docstring"""
				UpperCamelCase          =					'cpu'

				UpperCamelCase          =					self.get_dummy_components()

				UpperCamelCase          =					self.pipeline_class(**A_	)
				UpperCamelCase          =					pipe.to(A_	)

				pipe.set_progress_bar_config(disable=A_	)

				UpperCamelCase          =					pipe(**self.get_dummy_inputs(A_	)	)
				UpperCamelCase          =					output.images

				UpperCamelCase          =					pipe(
				    **self.get_dummy_inputs(A_	)    ,				return_dict=A_    ,				)[0]

				UpperCamelCase          =					image[0, -3:, -3:, -1]
				UpperCamelCase          =					image_from_tuple[0, -3:, -3:, -1]

				assert image.shape == (1, 64, 64, 3)

				UpperCamelCase          =					np.array(
				    [0.5498_5034, 0.5550_9365, 0.5256_1504, 0.557_0494, 0.559_3818, 0.526_3979, 0.5028_5643, 0.506_9846, 0.5119_6736]	)
				assert (
				    np.abs(image_slice.flatten() - expected_slice	).max() < 1e-2
				), F''' expected_slice {expected_slice}, but got {image_slice.flatten()}'''
				assert (
				    np.abs(image_from_tuple_slice.flatten() - expected_slice	).max() < 1e-2
				), F''' expected_slice {expected_slice}, but got {image_from_tuple_slice.flatten()}'''






@slow
@require_torch_gpu
class     lowercase (		unittest.TestCase       ):



		def __UpperCamelCase			(			self	)  -> Dict:
				"""simple docstring"""
				# clean up the VRAM after each test
				super().tearDown()
				gc.collect()
				torch.cuda.empty_cache()



		def __UpperCamelCase			(			self	)  -> Tuple:
				"""simple docstring"""
				UpperCamelCase          =					load_numpy(
				    'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main'
				    '/kandinskyv22/kandinskyv22_controlnet_img2img_robotcat_fp16.npy'	)

				UpperCamelCase          =					load_image(
				    'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main' '/kandinsky/cat.png'	)
				UpperCamelCase          =					init_image.resize((512, 512)	)

				UpperCamelCase          =					load_image(
				    'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main'
				    '/kandinskyv22/hint_image_cat.png'	)
				UpperCamelCase          =					torch.from_numpy(np.array(A_	)	).float() / 255.0
				UpperCamelCase          =					hint.permute(2    ,				0    ,				1	).unsqueeze(0	)

				UpperCamelCase          =					'A robot, 4k photo'

				UpperCamelCase          =					KandinskyVaaPriorEmbaEmbPipeline.from_pretrained(
				    'kandinsky-community/kandinsky-2-2-prior'    ,				torch_dtype=torch.floataa	)
				pipe_prior.to(A_	)

				UpperCamelCase          =					KandinskyVaaControlnetImgaImgPipeline.from_pretrained(
				    'kandinsky-community/kandinsky-2-2-controlnet-depth'    ,				torch_dtype=torch.floataa	)
				UpperCamelCase          =					pipeline.to(A_	)

				pipeline.set_progress_bar_config(disable=A_	)

				UpperCamelCase          =					torch.Generator(device='cpu'	).manual_seed(0	)

				UpperCamelCase     ,  UpperCamelCase          =					pipe_prior(
				    A_    ,				image=A_    ,				strength=0.85    ,				generator=A_    ,				negative_prompt=''    ,				).to_tuple()

				UpperCamelCase          =					pipeline(
				    image=A_    ,				image_embeds=A_    ,				negative_image_embeds=A_    ,				hint=A_    ,				generator=A_    ,				num_inference_steps=100    ,				height=512    ,				width=512    ,				strength=0.5    ,				output_type='np'    ,				)

				UpperCamelCase          =					output.images[0]

				assert image.shape == (512, 512, 3)

				assert_mean_pixel_difference(A_    ,				A_	)
