

import itertools
import random
import unittest

import numpy as np

from transformers import is_speech_available
from transformers.testing_utils import require_torch, require_torchaudio

from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin


if is_speech_available():
     from transformers import SpeechaTextFeatureExtractor

_lowerCamelCase    :       Any        =							random.Random()

def       a__					(		UpperCAmelCase							:      Dict    ,			UpperCAmelCase							:      List[str]=1.0    ,			UpperCAmelCase							:      int=None    ,			UpperCAmelCase							:      List[str]=None      )  ->						List[str]:
  if rng is None:
    UpperCAmelCase	:							Dict						=		global_rng

  UpperCAmelCase	:							Any						=		[]
  for batch_idx in range(shape[0]      ):
    values.append([]      )
    for _ in range(shape[1]      ):
      values[-1].append(rng.random() * scale      )

  return values





@require_torch
@require_torchaudio
class  __UpperCAmelCase					(	unittest.TestCase						):



   def __init__(							self							:					Union[str, Any],		__A							:					List[str],		__A							:					Optional[Any]=7,		__A							:					Optional[int]=4_0_0,		__A							:					Tuple=2_0_0_0,		__A							:					Union[str, Any]=2_4,		__A							:					Tuple=2_4,		__A							:					Optional[int]=0.0,		__A							:					List[str]=1_6_0_0_0,		__A							:					Any=True,		__A							:					List[Any]=True,		):
     UpperCAmelCase	:							Any						=		parent
     UpperCAmelCase	:							Any						=		batch_size
     UpperCAmelCase	:							Dict						=		min_seq_length
     UpperCAmelCase	:							Union[str, Any]						=		max_seq_length
     UpperCAmelCase	:							Union[str, Any]						=		(self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
     UpperCAmelCase	:							Optional[int]						=		feature_size
     UpperCAmelCase	:							Optional[int]						=		num_mel_bins
     UpperCAmelCase	:							Union[str, Any]						=		padding_value
     UpperCAmelCase	:							Union[str, Any]						=		sampling_rate
     UpperCAmelCase	:							Any						=		return_attention_mask
     UpperCAmelCase	:							Any						=		do_normalize



   def 	__magic_name__   (							self							:					List[str]						):
     return {
         "feature_size": self.feature_size,
         "num_mel_bins": self.num_mel_bins,
         "padding_value": self.padding_value,
         "sampling_rate": self.sampling_rate,
         "return_attention_mask": self.return_attention_mask,
         "do_normalize": self.do_normalize,
     }





   def 	__magic_name__   (							self							:					str,		__A							:					Dict=False,		__A							:					List[Any]=False						):



     def _flatten(__A							:					Any						):
       return list(itertools.chain(*_A						)						)

     if equal_length:
       UpperCAmelCase	:							Optional[int]						=		[floats_list((self.max_seq_length, self.feature_size)						) for _ in range(self.batch_size						)]
     else:
       # make sure that inputs increase in size
       UpperCAmelCase	:							Optional[int]						=		[
           floats_list((x, self.feature_size)						)
           for x in range(self.min_seq_length,		self.max_seq_length,		self.seq_length_diff						)
       ]
     if numpify:
       UpperCAmelCase	:							Optional[int]						=		[np.asarray(_A						) for x in speech_inputs]
     return speech_inputs






@require_torch
@require_torchaudio
class  __UpperCAmelCase					(	snake_case__      ,  unittest.TestCase						):
   UpperCamelCase       					=					SpeechaTextFeatureExtractor if is_speech_available() else None



   def 	__magic_name__   (							self							:					Dict						):
     UpperCAmelCase	:							Tuple						=		SpeechaTextFeatureExtractionTester(self						)



   def 	__magic_name__   (							self							:					List[Any],		__A							:					Optional[Any]						):
     self.assertTrue(np.all(np.mean(_A,		axis=0						) < 1E-3						)						)
     self.assertTrue(np.all(np.abs(np.var(_A,		axis=0						) - 1						) < 1E-3						)						)



   def 	__magic_name__   (							self							:					Optional[int]						):
     UpperCAmelCase	:							Tuple						=		self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()						)
     # create three inputs of length 800, 1000, and 1200
     UpperCAmelCase	:							Optional[Any]						=		[floats_list((1, x)						)[0] for x in range(8_0_0,		1_4_0_0,		2_0_0						)]
     UpperCAmelCase	:							Any						=		[np.asarray(_A						) for speech_input in speech_inputs]

     # Test feature size
     UpperCAmelCase	:							Any						=		feature_extractor(_A,		padding=_A,		return_tensors='''np'''						).input_features
     self.assertTrue(input_features.ndim == 3						)
     self.assertTrue(input_features.shape[-1] == feature_extractor.feature_size						)

     # Test not batched input
     UpperCAmelCase	:							List[str]						=		feature_extractor(speech_inputs[0],		return_tensors='''np'''						).input_features
     UpperCAmelCase	:							Optional[int]						=		feature_extractor(np_speech_inputs[0],		return_tensors='''np'''						).input_features
     self.assertTrue(np.allclose(_A,		_A,		atol=1E-3						)						)

     # Test batched
     UpperCAmelCase	:							List[str]						=		feature_extractor(_A,		return_tensors='''np'''						).input_features
     UpperCAmelCase	:							List[Any]						=		feature_extractor(_A,		return_tensors='''np'''						).input_features
     for enc_seq_a, enc_seq_a in zip(_A,		_A						):
       self.assertTrue(np.allclose(_A,		_A,		atol=1E-3						)						)

     # Test 2-D numpy arrays are batched.
     UpperCAmelCase	:							Union[str, Any]						=		[floats_list((1, x)						)[0] for x in (8_0_0, 8_0_0, 8_0_0)]
     UpperCAmelCase	:							Optional[Any]						=		np.asarray(_A						)
     UpperCAmelCase	:							Optional[int]						=		feature_extractor(_A,		return_tensors='''np'''						).input_features
     UpperCAmelCase	:							int						=		feature_extractor(_A,		return_tensors='''np'''						).input_features
     for enc_seq_a, enc_seq_a in zip(_A,		_A						):
       self.assertTrue(np.allclose(_A,		_A,		atol=1E-3						)						)



   def 	__magic_name__   (							self							:					List[Any]						):
     UpperCAmelCase	:							Optional[int]						=		self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()						)
     UpperCAmelCase	:							Optional[Any]						=		[floats_list((1, x)						)[0] for x in range(8_0_0,		1_4_0_0,		2_0_0						)]

     UpperCAmelCase	:							Tuple						=		['''longest''', '''max_length''', '''do_not_pad''']
     UpperCAmelCase	:							Union[str, Any]						=		[None, 1_6, None]
     for max_length, padding in zip(_A,		_A						):
       UpperCAmelCase	:							List[Any]						=		feature_extractor(
           _A,		padding=_A,		max_length=_A,		return_attention_mask=_A						)
       UpperCAmelCase	:							Union[str, Any]						=		inputs.input_features
       UpperCAmelCase	:							Optional[int]						=		inputs.attention_mask
       UpperCAmelCase	:							Tuple						=		[np.sum(_A						) for x in attention_mask]

       self._check_zero_mean_unit_variance(input_features[0][: fbank_feat_lengths[0]]						)
       self._check_zero_mean_unit_variance(input_features[1][: fbank_feat_lengths[1]]						)
       self._check_zero_mean_unit_variance(input_features[2][: fbank_feat_lengths[2]]						)



   def 	__magic_name__   (							self							:					List[str]						):
     UpperCAmelCase	:							Optional[Any]						=		self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()						)
     UpperCAmelCase	:							List[Any]						=		[floats_list((1, x)						)[0] for x in range(8_0_0,		1_4_0_0,		2_0_0						)]

     UpperCAmelCase	:							Any						=		['''longest''', '''max_length''', '''do_not_pad''']
     UpperCAmelCase	:							Dict						=		[None, 1_6, None]
     for max_length, padding in zip(_A,		_A						):
       UpperCAmelCase	:							Any						=		feature_extractor(
           _A,		max_length=_A,		padding=_A,		return_tensors='''np''',		return_attention_mask=_A						)
       UpperCAmelCase	:							List[str]						=		inputs.input_features
       UpperCAmelCase	:							List[str]						=		inputs.attention_mask
       UpperCAmelCase	:							Optional[int]						=		[np.sum(_A						) for x in attention_mask]

       self._check_zero_mean_unit_variance(input_features[0][: fbank_feat_lengths[0]]						)
       self.assertTrue(input_features[0][fbank_feat_lengths[0] :].sum() < 1E-6						)
       self._check_zero_mean_unit_variance(input_features[1][: fbank_feat_lengths[1]]						)
       self.assertTrue(input_features[0][fbank_feat_lengths[1] :].sum() < 1E-6						)
       self._check_zero_mean_unit_variance(input_features[2][: fbank_feat_lengths[2]]						)



   def 	__magic_name__   (							self							:					Dict						):
     UpperCAmelCase	:							str						=		self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()						)
     UpperCAmelCase	:							List[Any]						=		[floats_list((1, x)						)[0] for x in range(8_0_0,		1_4_0_0,		2_0_0						)]
     UpperCAmelCase	:							Union[str, Any]						=		feature_extractor(
         _A,		padding='''max_length''',		max_length=4,		truncation=_A,		return_tensors='''np''',		return_attention_mask=_A,		)
     UpperCAmelCase	:							Optional[int]						=		inputs.input_features
     UpperCAmelCase	:							Dict						=		inputs.attention_mask
     UpperCAmelCase	:							Any						=		np.sum(attention_mask == 1,		axis=1						)

     self._check_zero_mean_unit_variance(input_features[0, : fbank_feat_lengths[0]]						)
     self._check_zero_mean_unit_variance(input_features[1]						)
     self._check_zero_mean_unit_variance(input_features[2]						)



   def 	__magic_name__   (							self							:					Optional[int]						):
     UpperCAmelCase	:							Tuple						=		self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()						)
     UpperCAmelCase	:							str						=		[floats_list((1, x)						)[0] for x in range(8_0_0,		1_4_0_0,		2_0_0						)]
     UpperCAmelCase	:							Optional[Any]						=		feature_extractor(
         _A,		padding='''longest''',		max_length=4,		truncation=_A,		return_tensors='''np''',		return_attention_mask=_A,		)
     UpperCAmelCase	:							int						=		inputs.input_features
     UpperCAmelCase	:							Union[str, Any]						=		inputs.attention_mask
     UpperCAmelCase	:							Tuple						=		np.sum(attention_mask == 1,		axis=1						)

     self._check_zero_mean_unit_variance(input_features[0, : fbank_feat_lengths[0]]						)
     self._check_zero_mean_unit_variance(input_features[1, : fbank_feat_lengths[1]]						)
     self._check_zero_mean_unit_variance(input_features[2]						)

     # make sure that if max_length < longest -> then pad to max_length
     self.assertEqual(input_features.shape,		(3, 4, 2_4)						)

     UpperCAmelCase	:							int						=		[floats_list((1, x)						)[0] for x in range(8_0_0,		1_4_0_0,		2_0_0						)]
     UpperCAmelCase	:							Optional[Any]						=		feature_extractor(
         _A,		padding='''longest''',		max_length=1_6,		truncation=_A,		return_tensors='''np''',		return_attention_mask=_A,		)
     UpperCAmelCase	:							Optional[Any]						=		inputs.input_features
     UpperCAmelCase	:							int						=		inputs.attention_mask
     UpperCAmelCase	:							Optional[int]						=		np.sum(attention_mask == 1,		axis=1						)

     self._check_zero_mean_unit_variance(input_features[0, : fbank_feat_lengths[0]]						)
     self._check_zero_mean_unit_variance(input_features[1, : fbank_feat_lengths[1]]						)
     self._check_zero_mean_unit_variance(input_features[2]						)

     # make sure that if max_length < longest -> then pad to max_length
     self.assertEqual(input_features.shape,		(3, 6, 2_4)						)



   def 	__magic_name__   (							self							:					Optional[int]						):
     import torch

     UpperCAmelCase	:							List[str]						=		self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()						)
     UpperCAmelCase	:							Any						=		np.random.rand(1_0_0,		3_2						).astype(np.floataa						)
     UpperCAmelCase	:							int						=		np_speech_inputs.tolist()

     for inputs in [py_speech_inputs, np_speech_inputs]:
       UpperCAmelCase	:							Tuple						=		feature_extractor.pad([{'''input_features''': inputs}],		return_tensors='''np'''						)
       self.assertTrue(np_processed.input_features.dtype == np.floataa						)
       UpperCAmelCase	:							Optional[Any]						=		feature_extractor.pad([{'''input_features''': inputs}],		return_tensors='''pt'''						)
       self.assertTrue(pt_processed.input_features.dtype == torch.floataa						)



   def 	__magic_name__   (							self							:					int,		__A							:					Any						):
     from datasets import load_dataset

     UpperCAmelCase	:							str						=		load_dataset('''hf-internal-testing/librispeech_asr_dummy''',		'''clean''',		split='''validation'''						)
     # automatic decoding with librispeech
     UpperCAmelCase	:							int						=		ds.sort('''id'''						).select(range(_A						)						)[:num_samples]['''audio''']

     return [x["array"] for x in speech_samples]





   def 	__magic_name__   (							self							:					Dict						):
     UpperCAmelCase	:							Optional[int]						=		np.array([
         -1.5_7_4_5, -1.7_7_1_3, -1.7_0_2_0, -1.6_0_6_9, -1.2_2_5_0, -1.1_1_0_5, -0.9_0_7_2, -0.8_2_4_1,
         -1.2_3_1_0, -0.8_0_9_8, -0.3_3_2_0, -0.4_1_0_1, -0.7_9_8_5, -0.4_9_9_6, -0.8_2_1_3, -0.9_1_2_8,
         -1.0_4_2_0, -1.1_2_8_6, -1.0_4_4_0, -0.7_9_9_9, -0.8_4_0_5, -1.2_2_7_5, -1.5_4_4_3, -1.4_6_2_5,
     ]						)
     # fmt: on

     UpperCAmelCase	:							str						=		self._load_datasamples(1						)
     UpperCAmelCase	:							List[Any]						=		self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()						)
     UpperCAmelCase	:							Any						=		feature_extractor(_A,		return_tensors='''pt'''						).input_features
     self.assertEquals(input_features.shape,		(1, 5_8_4, 2_4)						)
     self.assertTrue(np.allclose(input_features[0, 0, :3_0],		_A,		atol=1E-4						)						)








import inspect
import os

import torch
from transformers import AutoModel
from transformers.testing_utils import mockenv_context
from transformers.trainer_utils import set_seed

import accelerate
from accelerate.accelerator import Accelerator
from accelerate.state import AcceleratorState
from accelerate.test_utils.testing import (
    AccelerateTestCase,
    TempDirTestCase,
    execute_subprocess_async,
    require_cuda,
    require_fsdp,
    require_multi_gpu,
    slow,
)
from accelerate.utils.constants import (
    FSDP_AUTO_WRAP_POLICY,
    FSDP_BACKWARD_PREFETCH,
    FSDP_SHARDING_STRATEGY,
    FSDP_STATE_DICT_TYPE,
)
from accelerate.utils.dataclasses import FullyShardedDataParallelPlugin
from accelerate.utils.other import patch_environment


set_seed(4_2)

_lowerCamelCase    :       List[str]        =							"bert-base-cased"
_lowerCamelCase    :       str        =							"fp16"
_lowerCamelCase    :       Optional[int]        =							"bf16"
_lowerCamelCase    :       List[str]        =							[FPaa, BFaa]





@require_fsdp
@require_cuda
class  __UpperCAmelCase					(	lowerCamelCase__						):



   def 	__magic_name__   (							self							:					Tuple						):
     super().setUp()

     UpperCAmelCase	:							Optional[Any]						=			dict(
         ACCELERATE_USE_FSDP='''true''',		MASTER_ADDR='''localhost''',		MASTER_PORT='''10999''',		RANK='''0''',		LOCAL_RANK='''0''',		WORLD_SIZE='''1''',		)



   def 	__magic_name__   (							self							:					int						):
     from torch.distributed.fsdp.fully_sharded_data_parallel import ShardingStrategy

     for i, strategy in enumerate(__A						):
       UpperCAmelCase	:							List[Any]						=			self.dist_env.copy()
       UpperCAmelCase	:							Union[str, Any]						=			F'''{i + 1}'''
       UpperCAmelCase	:							Union[str, Any]						=			strategy
       with mockenv_context(**__A						):
         UpperCAmelCase	:							Union[str, Any]						=			FullyShardedDataParallelPlugin()
         self.assertEqual(fsdp_plugin.sharding_strategy,		ShardingStrategy(i + 1						)						)



   def 	__magic_name__   (							self							:					Dict						):
     from torch.distributed.fsdp.fully_sharded_data_parallel import BackwardPrefetch

     for i, prefetch_policy in enumerate(__A						):
       UpperCAmelCase	:							int						=			self.dist_env.copy()
       UpperCAmelCase	:							Dict						=			prefetch_policy
       with mockenv_context(**__A						):
         UpperCAmelCase	:							Optional[int]						=			FullyShardedDataParallelPlugin()
         if prefetch_policy == "NO_PREFETCH":
           self.assertIsNone(fsdp_plugin.backward_prefetch						)
         else:
           self.assertEqual(fsdp_plugin.backward_prefetch,		BackwardPrefetch(i + 1						)						)



   def 	__magic_name__   (							self							:					Any						):
     from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType

     for i, state_dict_type in enumerate(__A						):
       UpperCAmelCase	:							Any						=			self.dist_env.copy()
       UpperCAmelCase	:							int						=			state_dict_type
       with mockenv_context(**__A						):
         UpperCAmelCase	:							str						=			FullyShardedDataParallelPlugin()
         self.assertEqual(fsdp_plugin.state_dict_type,		StateDictType(i + 1						)						)
         if state_dict_type == "FULL_STATE_DICT":
           self.assertTrue(fsdp_plugin.state_dict_config.offload_to_cpu						)
           self.assertTrue(fsdp_plugin.state_dict_config.ranka_only						)



   def 	__magic_name__   (							self							:					int						):
     UpperCAmelCase	:							Optional[int]						=			AutoModel.from_pretrained(__A						)
     for policy in FSDP_AUTO_WRAP_POLICY:
       UpperCAmelCase	:							Any						=			self.dist_env.copy()
       UpperCAmelCase	:							List[Any]						=			policy
       if policy == "TRANSFORMER_BASED_WRAP":
         UpperCAmelCase	:							Tuple						=			'''BertLayer'''
       elif policy == "SIZE_BASED_WRAP":
         UpperCAmelCase	:							List[str]						=			'''2000'''
       with mockenv_context(**__A						):
         UpperCAmelCase	:							Optional[Any]						=			FullyShardedDataParallelPlugin()
         fsdp_plugin.set_auto_wrap_policy(__A						)
         if policy == "NO_WRAP":
           self.assertIsNone(fsdp_plugin.auto_wrap_policy						)
         else:
           self.assertIsNotNone(fsdp_plugin.auto_wrap_policy						)

     UpperCAmelCase	:							List[str]						=			self.dist_env.copy()
     UpperCAmelCase	:							Tuple						=			'''TRANSFORMER_BASED_WRAP'''
     UpperCAmelCase	:							Optional[Any]						=			'''T5Layer'''
     with mockenv_context(**__A						):
       UpperCAmelCase	:							int						=			FullyShardedDataParallelPlugin()
       with self.assertRaises(__A						) as cm:
         fsdp_plugin.set_auto_wrap_policy(__A						)
       self.assertTrue('''Could not find the transformer layer class to wrap in the model.''' in str(cm.exception						)						)

     UpperCAmelCase	:							List[Any]						=			self.dist_env.copy()
     UpperCAmelCase	:							str						=			'''SIZE_BASED_WRAP'''
     UpperCAmelCase	:							str						=			'''0'''
     with mockenv_context(**__A						):
       UpperCAmelCase	:							Optional[int]						=			FullyShardedDataParallelPlugin()
       fsdp_plugin.set_auto_wrap_policy(__A						)
       self.assertIsNone(fsdp_plugin.auto_wrap_policy						)



   def 	__magic_name__   (							self							:					int						):
     from torch.distributed.fsdp.fully_sharded_data_parallel import MixedPrecision
     from torch.distributed.fsdp.sharded_grad_scaler import ShardedGradScaler

     for mp_dtype in dtypes:
       UpperCAmelCase	:							List[Any]						=			self.dist_env.copy()
       UpperCAmelCase	:							int						=			mp_dtype
       with mockenv_context(**__A						):
         UpperCAmelCase	:							int						=			Accelerator()
         if mp_dtype == "fp16":
           UpperCAmelCase	:							Any						=			torch.floataa
         elif mp_dtype == "bf16":
           UpperCAmelCase	:							Any						=			torch.bfloataa
         UpperCAmelCase	:							Optional[Any]						=			MixedPrecision(param_dtype=__A,		reduce_dtype=__A,		buffer_dtype=__A						)
         self.assertEqual(accelerator.state.fsdp_plugin.mixed_precision_policy,		__A						)
         if mp_dtype == FPaa:
           self.assertTrue(isinstance(accelerator.scaler,		__A						)						)
         elif mp_dtype == BFaa:
           self.assertIsNone(accelerator.scaler						)
         AcceleratorState._reset_state(__A						)





   def 	__magic_name__   (							self							:					Optional[int]						):
     from torch.distributed.fsdp.fully_sharded_data_parallel import CPUOffload

     for flag in [True, False]:
       UpperCAmelCase	:							Any						=			self.dist_env.copy()
       UpperCAmelCase	:							int						=			str(__A						).lower()
       with mockenv_context(**__A						):
         UpperCAmelCase	:							Union[str, Any]						=			FullyShardedDataParallelPlugin()
         self.assertEqual(fsdp_plugin.cpu_offload,		CPUOffload(offload_params=__A						)						)






@require_fsdp
@require_multi_gpu
@slow
class  __UpperCAmelCase					(	lowerCamelCase__						):



   def 	__magic_name__   (							self							:					List[Any]						):
     super().setUp()
     UpperCAmelCase	:							int						=			0.8_2
     UpperCAmelCase	:							List[str]						=			[
         '''fsdp_shard_grad_op_transformer_based_wrap''',
         '''fsdp_full_shard_transformer_based_wrap''',
     ]
     UpperCAmelCase	:							int						=			{
         '''multi_gpu_fp16''': 3_2_0_0,
         '''fsdp_shard_grad_op_transformer_based_wrap_fp16''': 2_0_0_0,
         '''fsdp_full_shard_transformer_based_wrap_fp16''': 1_9_0_0,
         # Disabling below test as it overwhelms the RAM memory usage
         # on CI self-hosted runner leading to tests getting killed.
         # "fsdp_full_shard_cpu_offload_transformer_based_wrap_fp32": 1500,  # fp16 was leading to indefinite hang
     }
     UpperCAmelCase	:							List[str]						=			1_6_0
     UpperCAmelCase	:							Optional[int]						=			1_6_0

     UpperCAmelCase	:							Union[str, Any]						=			inspect.getfile(accelerate.test_utils						)
     UpperCAmelCase	:							Dict						=			os.path.sep.join(mod_file.split(os.path.sep						)[:-1] + ['''scripts''', '''external_deps''']						)



   def 	__magic_name__   (							self							:					str						):
     UpperCAmelCase	:							str						=			os.path.join(self.test_scripts_folder,		'''test_performance.py'''						)
     UpperCAmelCase	:							Any						=			['''accelerate''', '''launch''', '''--num_processes=2''', '''--num_machines=1''', '''--machine_rank=0''', '''--use_fsdp''']
     for config in self.performance_configs:
       UpperCAmelCase	:							Union[str, Any]						=			cmd.copy()
       for i, strategy in enumerate(__A						):
         if strategy.lower() in config:
           cmd_config.append(F'''--fsdp_sharding_strategy={i+1}'''						)
           break

       if "fp32" in config:
         cmd_config.append('''--mixed_precision=no'''						)
       else:
         cmd_config.append('''--mixed_precision=fp16'''						)

       if "cpu_offload" in config:
         cmd_config.append('''--fsdp_offload_params=True'''						)

       for policy in FSDP_AUTO_WRAP_POLICY:
         if policy.lower() in config:
           cmd_config.append(F'''--fsdp_auto_wrap_policy={policy}'''						)
           break

       if policy == "TRANSFORMER_BASED_WRAP":
         cmd_config.append('''--fsdp_transformer_layer_cls_to_wrap=BertLayer'''						)
       elif policy == "SIZE_BASED_WRAP":
         cmd_config.append('''--fsdp_min_num_params=2000'''						)

       cmd_config.extend(
           [
               self.test_file_path,
               F'''--output_dir={self.tmpdir}''',
               F'''--performance_lower_bound={self.performance_lower_bound}''',
           ]						)
       with patch_environment(omp_num_threads=1						):
         execute_subprocess_async(__A,		env=os.environ.copy()						)



   def 	__magic_name__   (							self							:					Tuple						):
     UpperCAmelCase	:							Dict						=			os.path.join(self.test_scripts_folder,		'''test_checkpointing.py'''						)
     UpperCAmelCase	:							Optional[int]						=			[
         '''accelerate''',
         '''launch''',
         '''--num_processes=2''',
         '''--num_machines=1''',
         '''--machine_rank=0''',
         '''--use_fsdp''',
         '''--mixed_precision=fp16''',
         '''--fsdp_transformer_layer_cls_to_wrap=BertLayer''',
     ]

     for i, strategy in enumerate(__A						):
       UpperCAmelCase	:							Optional[Any]						=			cmd.copy()
       cmd_config.append(F'''--fsdp_sharding_strategy={i+1}'''						)
       if strategy != "FULL_SHARD":
         continue
       UpperCAmelCase	:							Any						=			len(__A						)
       for state_dict_type in FSDP_STATE_DICT_TYPE:
         UpperCAmelCase	:							List[Any]						=			cmd_config[:state_dict_config_index]
         cmd_config.append(F'''--fsdp_state_dict_type={state_dict_type}'''						)
         cmd_config.extend(
             [
                 self.test_file_path,
                 F'''--output_dir={self.tmpdir}''',
                 '''--partial_train_epoch=1''',
             ]						)
         with patch_environment(omp_num_threads=1						):
           execute_subprocess_async(__A,		env=os.environ.copy()						)

         UpperCAmelCase	:							Tuple						=			cmd_config[:-1]
         UpperCAmelCase	:							List[str]						=			os.path.join(self.tmpdir,		'''epoch_0'''						)
         cmd_config.extend(
             [
                 F'''--resume_from_checkpoint={resume_from_checkpoint}''',
             ]						)
         with patch_environment(omp_num_threads=1						):
           execute_subprocess_async(__A,		env=os.environ.copy()						)





   def 	__magic_name__   (							self							:					Any						):
     UpperCAmelCase	:							Optional[Any]						=			os.path.join(self.test_scripts_folder,		'''test_peak_memory_usage.py'''						)
     UpperCAmelCase	:							str						=			[
         '''accelerate''',
         '''launch''',
         '''--num_processes=2''',
         '''--num_machines=1''',
         '''--machine_rank=0''',
     ]
     for spec, peak_mem_upper_bound in self.peak_memory_usage_upper_bound.items():
       UpperCAmelCase	:							List[Any]						=			cmd.copy()
       if "fp16" in spec:
         cmd_config.extend(['''--mixed_precision=fp16''']						)
       else:
         cmd_config.extend(['''--mixed_precision=no''']						)

       if "multi_gpu" in spec:
         continue
       else:
         cmd_config.extend(['''--use_fsdp''']						)
         for i, strategy in enumerate(__A						):
           if strategy.lower() in spec:
             cmd_config.append(F'''--fsdp_sharding_strategy={i+1}'''						)
             break

         if "cpu_offload" in spec:
           cmd_config.append('''--fsdp_offload_params=True'''						)

         for policy in FSDP_AUTO_WRAP_POLICY:
           if policy.lower() in spec:
             cmd_config.append(F'''--fsdp_auto_wrap_policy={policy}'''						)
             break

         if policy == "TRANSFORMER_BASED_WRAP":
           cmd_config.append('''--fsdp_transformer_layer_cls_to_wrap=BertLayer'''						)
         elif policy == "SIZE_BASED_WRAP":
           cmd_config.append('''--fsdp_min_num_params=2000'''						)

       cmd_config.extend(
           [
               self.test_file_path,
               F'''--output_dir={self.tmpdir}''',
               F'''--peak_memory_upper_bound={peak_mem_upper_bound}''',
               F'''--n_train={self.n_train}''',
               F'''--n_val={self.n_val}''',
           ]						)
       with patch_environment(omp_num_threads=1						):
         execute_subprocess_async(__A,		env=os.environ.copy()						)





