

import argparse
import json
import os

import tensorstore as ts
import torch
from flax import serialization
from flax.traverse_util import flatten_dict, unflatten_dict
from tensorflow.io import gfile

from transformers.modeling_utils import dtype_byte_size
from transformers.models.switch_transformers.convert_switch_transformers_original_flax_checkpoint_to_pytorch import (
    rename_keys,
)
from transformers.utils import WEIGHTS_INDEX_NAME, WEIGHTS_NAME
from transformers.utils.hub import convert_file_size_to_int

def       a__					(		UpperCAmelCase							:      str    ,			UpperCAmelCase							:      List[Any]      )  ->						List[Any]:
  if flax_key_tuple[-1] == "kernel" and flax_tensor.ndim == 3:
    # expert layer
    UpperCAmelCase	:							Dict						=			flax_key_tuple[:-1] + ('''weight''',)
    UpperCAmelCase	:							List[str]						=			torch.permute(UpperCAmelCase    ,			(0, 2, 1)      )
  elif flax_key_tuple[-1] == "kernel" and ".".join(UpperCAmelCase      ):
    # linear layer
    UpperCAmelCase	:							List[str]						=			flax_key_tuple[:-1] + ('''weight''',)
    UpperCAmelCase	:							Tuple						=			flax_tensor.T
  elif flax_key_tuple[-1] in ["scale", "embedding"]:
    UpperCAmelCase	:							Any						=			flax_key_tuple[:-1] + ('''weight''',)

  return flax_key_tuple, flax_tensor

def       a__					(		UpperCAmelCase							:      Optional[Any]    ,			UpperCAmelCase							:      Dict    ,			UpperCAmelCase							:      Tuple      )  ->						int:
  if "metadata" in layer:
    UpperCAmelCase	:							Dict						=			layer.split('''metadata'''      )
    UpperCAmelCase	:							Dict						=			''''''.join(split_layer[0]      )[:-1]
    UpperCAmelCase	:							Dict						=			[tuple(('''metadata''' + split_layer[1]).split('''/'''      )      )]
  elif "kvstore" in layer:
    UpperCAmelCase	:							Optional[Any]						=			layer.split('''kvstore'''      )
    UpperCAmelCase	:							Dict						=			''''''.join(split_layer[0]      )[:-1]
    UpperCAmelCase	:							str						=			[tuple(('''kvstore''' + split_layer[1]).split('''/'''      )      )]

  else:
    UpperCAmelCase	:							Tuple						=			layer.split('''/'''      )
    UpperCAmelCase	:							Any						=			'''/'''.join(split_layer[:-1]      )
    UpperCAmelCase	:							Any						=			(split_layer[-1],)

  if "kvstore/path" in layer:
    UpperCAmelCase	:							Optional[Any]						=			f'''{switch_checkpoint_path}/{checkpoint_info[layer]}'''
  elif "kvstore/driver" in layer:
    UpperCAmelCase	:							int						=			'''file'''
  else:
    UpperCAmelCase	:							Dict						=			checkpoint_info[layer]

  return curr_real_layer_name, split_layer, content

def       a__					(		UpperCAmelCase							:      Optional[int]    ,			UpperCAmelCase							:      Any      )  ->						Optional[Any]:
  UpperCAmelCase	:							Tuple						=			rename_keys(UpperCAmelCase      )
  UpperCAmelCase	:							List[Any]						=			{}
  for k, v in current_block.items():
    UpperCAmelCase	:							Dict						=			v
  UpperCAmelCase	:							Optional[Any]						=			new_current_block
  torch.save(UpperCAmelCase    ,			UpperCAmelCase      )

def       a__					(		UpperCAmelCase							:      Optional[Any]    ,			UpperCAmelCase							:      Union[str, Any]    ,			UpperCAmelCase							:      Tuple    ,			UpperCAmelCase							:      List[Any]    ,			UpperCAmelCase							:      str = WEIGHTS_NAME      )  ->						Dict:
  UpperCAmelCase	:							Optional[int]						=			convert_file_size_to_int(UpperCAmelCase      )
  UpperCAmelCase	:							Union[str, Any]						=			[]
  UpperCAmelCase	:							Dict						=			{}
  UpperCAmelCase	:							int						=			0
  UpperCAmelCase	:							str						=			0

  os.makedirs(UpperCAmelCase    ,			exist_ok=UpperCAmelCase      )
  with gfile.GFile(switch_checkpoint_path + '''/checkpoint'''    ,			'''rb'''      ) as fp:
    UpperCAmelCase	:							Union[str, Any]						=			serialization.msgpack_restore(fp.read()      )['''optimizer''']['''target''']
    UpperCAmelCase	:							Optional[Any]						=			flatten_dict(UpperCAmelCase    ,			sep='''/'''      )

  UpperCAmelCase	:							List[str]						=			{}
  for layer in checkpoint_info.keys():
    UpperCAmelCase			,						UpperCAmelCase			,						UpperCAmelCase	:							Any						=			get_key_and_tensorstore_dict(
        UpperCAmelCase    ,			UpperCAmelCase    ,			UpperCAmelCase      )
    if curr_real_layer_name in all_layers:
      UpperCAmelCase	:							Union[str, Any]						=			content
    else:
      UpperCAmelCase	:							Union[str, Any]						=			{split_layer[-1]: content}

  for key in all_layers.keys():
    # open tensorstore file
    UpperCAmelCase	:							Dict						=			ts.open(unflatten_dict(all_layers[key]      )      ).result().read().result()
    UpperCAmelCase	:							int						=			torch.tensor(UpperCAmelCase      )
    UpperCAmelCase	:							Tuple						=			raw_weights.numel() * dtype_byte_size(raw_weights.dtype      )

    # use the renaming pattern from the small conversion scripts
    UpperCAmelCase			,						UpperCAmelCase	:							str						=			rename_base_flax_keys(tuple(key.split('''/'''      )      )    ,			UpperCAmelCase      )
    UpperCAmelCase	:							Optional[int]						=			'''/'''.join(UpperCAmelCase      )

    # If this weight is going to tip up over the maximal size, we split.
    if current_block_size + weight_size > max_shard_size:
      UpperCAmelCase	:							Optional[Any]						=			os.path.join(
          UpperCAmelCase    ,			weights_name.replace('''.bin'''    ,			f'''-{len(UpperCAmelCase      )+1:05d}-of-???.bin'''      )      )
      rename_and_save_block(UpperCAmelCase    ,			UpperCAmelCase      )
      sharded_state_dicts.append(current_block.keys()      )
      del current_block
      UpperCAmelCase	:							Optional[Any]						=			{}
      UpperCAmelCase	:							Union[str, Any]						=			0

    UpperCAmelCase	:							Union[str, Any]						=			raw_weights.to(getattr(UpperCAmelCase    ,			UpperCAmelCase      )      )
    current_block_size += weight_size
    total_size += weight_size

  # Add the last block
  UpperCAmelCase	:							Tuple						=			os.path.join(UpperCAmelCase    ,			weights_name.replace('''.bin'''    ,			f'''-{len(UpperCAmelCase      )+1:05d}-of-???.bin'''      )      )
  rename_and_save_block(UpperCAmelCase    ,			UpperCAmelCase      )
  sharded_state_dicts.append(current_block.keys()      )

  # If we only have one shard, we return it
  if len(UpperCAmelCase      ) == 1:
    return {weights_name: sharded_state_dicts[0]}, None

  # Otherwise, let's build the index
  UpperCAmelCase	:							Any						=			{}
  UpperCAmelCase	:							Any						=			{}
  for idx, shard in enumerate(UpperCAmelCase      ):
    UpperCAmelCase	:							Union[str, Any]						=			weights_name.replace(
        '''.bin'''    ,			f'''-{idx+1:05d}-of-{len(UpperCAmelCase      ):05d}.bin'''      )  # len(sharded_state_dicts):05d}
    UpperCAmelCase	:							Dict						=			os.path.join(UpperCAmelCase    ,			weights_name.replace('''.bin'''    ,			f'''-{idx+1:05d}-of-???.bin'''      )      )
    os.rename(UpperCAmelCase    ,			os.path.join(UpperCAmelCase    ,			UpperCAmelCase      )      )
    UpperCAmelCase	:							Any						=			shard
    for key in shard:
      UpperCAmelCase	:							int						=			shard_file

    # Add the metadata
  UpperCAmelCase	:							int						=			{'''total_size''': total_size}
  UpperCAmelCase	:							Any						=			{'''metadata''': metadata, '''weight_map''': weight_map}

  with open(os.path.join(UpperCAmelCase    ,			UpperCAmelCase      )    ,			'''w'''    ,			encoding='''utf-8'''      ) as f:
    UpperCAmelCase	:							str						=			json.dumps(UpperCAmelCase    ,			indent=2    ,			sort_keys=UpperCAmelCase      ) + '''\n'''
    f.write(UpperCAmelCase      )

  return metadata, index


if __name__ == "__main__":
     _lowerCamelCase    :       List[Any]        =							argparse.ArgumentParser()
     # Required parameters
     parser.add_argument(
         "--switch_t5x_checkpoint_path",
         default="/mnt/disks/disk_switch/original_checkpoints/switch-xxl-128/checkpoint_634600",
         type=str,
         required=False,
         help="Path to a directory containing a folder per layer. Follows the original Google format.",
     )
     parser.add_argument("--max_shard_size", default="10GB", required=False, help="Max shard size")
     parser.add_argument("--dtype", default="bfloat16", type=str, required=False, help="dtype of the saved model")
     parser.add_argument(
         "--pytorch_dump_folder_path",
         default="/mnt/disks/disk_switch/original_checkpoints/switch-xxl-128-converted",
         type=str,
         required=False,
         help="Path to the output pytorch model.",
     )
     _lowerCamelCase    :       Any        =							parser.parse_args()
     shard_on_the_fly(
         args.switch_tax_checkpoint_path,
         args.pytorch_dump_folder_path,
         args.max_shard_size,
         args.dtype,
     )






def       a__					(		)  ->						str:
  from transformers import SwitchTransformersConfig, SwitchTransformersForConditionalGeneration, TaTokenizer

  UpperCAmelCase	:							Tuple						=			SwitchTransformersConfig.from_pretrained('''google/switch-base-8'''      )
  config.save_pretrained('''/home/arthur_huggingface_co/transformers/switch_converted'''      )
  UpperCAmelCase	:							Optional[int]						=			SwitchTransformersForConditionalGeneration.from_pretrained(
      '''/home/arthur_huggingface_co/transformers/switch_converted'''    ,			device_map='''auto'''      )

  UpperCAmelCase	:							Optional[int]						=			TaTokenizer.from_pretrained('''t5-small'''      )
  UpperCAmelCase	:							Optional[Any]						=			'''A <extra_id_0> walks into a bar a orders a <extra_id_1> with <extra_id_2> pinch of <extra_id_3>.'''

  UpperCAmelCase	:							List[Any]						=			tokenizer(UpperCAmelCase    ,			return_tensors='''pt'''      ).input_ids
  UpperCAmelCase	:							str						=			model.generate(UpperCAmelCase    ,			decoder_start_token_id=0      )
  print(tokenizer.decode(out[0]      )      )








import inspect
import os

import torch
from transformers import AutoModel
from transformers.testing_utils import mockenv_context
from transformers.trainer_utils import set_seed

import accelerate
from accelerate.accelerator import Accelerator
from accelerate.state import AcceleratorState
from accelerate.test_utils.testing import (
    AccelerateTestCase,
    TempDirTestCase,
    execute_subprocess_async,
    require_cuda,
    require_fsdp,
    require_multi_gpu,
    slow,
)
from accelerate.utils.constants import (
    FSDP_AUTO_WRAP_POLICY,
    FSDP_BACKWARD_PREFETCH,
    FSDP_SHARDING_STRATEGY,
    FSDP_STATE_DICT_TYPE,
)
from accelerate.utils.dataclasses import FullyShardedDataParallelPlugin
from accelerate.utils.other import patch_environment


set_seed(4_2)

_lowerCamelCase    :       List[str]        =							"bert-base-cased"
_lowerCamelCase    :       str        =							"fp16"
_lowerCamelCase    :       Optional[int]        =							"bf16"
_lowerCamelCase    :       List[str]        =							[FPaa, BFaa]





@require_fsdp
@require_cuda
class  __UpperCAmelCase					(	lowerCamelCase__						):



   def 	__magic_name__   (							self							:					Tuple						):
     super().setUp()

     UpperCAmelCase	:							Optional[Any]						=			dict(
         ACCELERATE_USE_FSDP='''true''',		MASTER_ADDR='''localhost''',		MASTER_PORT='''10999''',		RANK='''0''',		LOCAL_RANK='''0''',		WORLD_SIZE='''1''',		)



   def 	__magic_name__   (							self							:					int						):
     from torch.distributed.fsdp.fully_sharded_data_parallel import ShardingStrategy

     for i, strategy in enumerate(__A						):
       UpperCAmelCase	:							List[Any]						=			self.dist_env.copy()
       UpperCAmelCase	:							Union[str, Any]						=			F'''{i + 1}'''
       UpperCAmelCase	:							Union[str, Any]						=			strategy
       with mockenv_context(**__A						):
         UpperCAmelCase	:							Union[str, Any]						=			FullyShardedDataParallelPlugin()
         self.assertEqual(fsdp_plugin.sharding_strategy,		ShardingStrategy(i + 1						)						)



   def 	__magic_name__   (							self							:					Dict						):
     from torch.distributed.fsdp.fully_sharded_data_parallel import BackwardPrefetch

     for i, prefetch_policy in enumerate(__A						):
       UpperCAmelCase	:							int						=			self.dist_env.copy()
       UpperCAmelCase	:							Dict						=			prefetch_policy
       with mockenv_context(**__A						):
         UpperCAmelCase	:							Optional[int]						=			FullyShardedDataParallelPlugin()
         if prefetch_policy == "NO_PREFETCH":
           self.assertIsNone(fsdp_plugin.backward_prefetch						)
         else:
           self.assertEqual(fsdp_plugin.backward_prefetch,		BackwardPrefetch(i + 1						)						)



   def 	__magic_name__   (							self							:					Any						):
     from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType

     for i, state_dict_type in enumerate(__A						):
       UpperCAmelCase	:							Any						=			self.dist_env.copy()
       UpperCAmelCase	:							int						=			state_dict_type
       with mockenv_context(**__A						):
         UpperCAmelCase	:							str						=			FullyShardedDataParallelPlugin()
         self.assertEqual(fsdp_plugin.state_dict_type,		StateDictType(i + 1						)						)
         if state_dict_type == "FULL_STATE_DICT":
           self.assertTrue(fsdp_plugin.state_dict_config.offload_to_cpu						)
           self.assertTrue(fsdp_plugin.state_dict_config.ranka_only						)



   def 	__magic_name__   (							self							:					int						):
     UpperCAmelCase	:							Optional[int]						=			AutoModel.from_pretrained(__A						)
     for policy in FSDP_AUTO_WRAP_POLICY:
       UpperCAmelCase	:							Any						=			self.dist_env.copy()
       UpperCAmelCase	:							List[Any]						=			policy
       if policy == "TRANSFORMER_BASED_WRAP":
         UpperCAmelCase	:							Tuple						=			'''BertLayer'''
       elif policy == "SIZE_BASED_WRAP":
         UpperCAmelCase	:							List[str]						=			'''2000'''
       with mockenv_context(**__A						):
         UpperCAmelCase	:							Optional[Any]						=			FullyShardedDataParallelPlugin()
         fsdp_plugin.set_auto_wrap_policy(__A						)
         if policy == "NO_WRAP":
           self.assertIsNone(fsdp_plugin.auto_wrap_policy						)
         else:
           self.assertIsNotNone(fsdp_plugin.auto_wrap_policy						)

     UpperCAmelCase	:							List[str]						=			self.dist_env.copy()
     UpperCAmelCase	:							Tuple						=			'''TRANSFORMER_BASED_WRAP'''
     UpperCAmelCase	:							Optional[Any]						=			'''T5Layer'''
     with mockenv_context(**__A						):
       UpperCAmelCase	:							int						=			FullyShardedDataParallelPlugin()
       with self.assertRaises(__A						) as cm:
         fsdp_plugin.set_auto_wrap_policy(__A						)
       self.assertTrue('''Could not find the transformer layer class to wrap in the model.''' in str(cm.exception						)						)

     UpperCAmelCase	:							List[Any]						=			self.dist_env.copy()
     UpperCAmelCase	:							str						=			'''SIZE_BASED_WRAP'''
     UpperCAmelCase	:							str						=			'''0'''
     with mockenv_context(**__A						):
       UpperCAmelCase	:							Optional[int]						=			FullyShardedDataParallelPlugin()
       fsdp_plugin.set_auto_wrap_policy(__A						)
       self.assertIsNone(fsdp_plugin.auto_wrap_policy						)



   def 	__magic_name__   (							self							:					int						):
     from torch.distributed.fsdp.fully_sharded_data_parallel import MixedPrecision
     from torch.distributed.fsdp.sharded_grad_scaler import ShardedGradScaler

     for mp_dtype in dtypes:
       UpperCAmelCase	:							List[Any]						=			self.dist_env.copy()
       UpperCAmelCase	:							int						=			mp_dtype
       with mockenv_context(**__A						):
         UpperCAmelCase	:							int						=			Accelerator()
         if mp_dtype == "fp16":
           UpperCAmelCase	:							Any						=			torch.floataa
         elif mp_dtype == "bf16":
           UpperCAmelCase	:							Any						=			torch.bfloataa
         UpperCAmelCase	:							Optional[Any]						=			MixedPrecision(param_dtype=__A,		reduce_dtype=__A,		buffer_dtype=__A						)
         self.assertEqual(accelerator.state.fsdp_plugin.mixed_precision_policy,		__A						)
         if mp_dtype == FPaa:
           self.assertTrue(isinstance(accelerator.scaler,		__A						)						)
         elif mp_dtype == BFaa:
           self.assertIsNone(accelerator.scaler						)
         AcceleratorState._reset_state(__A						)





   def 	__magic_name__   (							self							:					Optional[int]						):
     from torch.distributed.fsdp.fully_sharded_data_parallel import CPUOffload

     for flag in [True, False]:
       UpperCAmelCase	:							Any						=			self.dist_env.copy()
       UpperCAmelCase	:							int						=			str(__A						).lower()
       with mockenv_context(**__A						):
         UpperCAmelCase	:							Union[str, Any]						=			FullyShardedDataParallelPlugin()
         self.assertEqual(fsdp_plugin.cpu_offload,		CPUOffload(offload_params=__A						)						)






@require_fsdp
@require_multi_gpu
@slow
class  __UpperCAmelCase					(	lowerCamelCase__						):



   def 	__magic_name__   (							self							:					List[Any]						):
     super().setUp()
     UpperCAmelCase	:							int						=			0.8_2
     UpperCAmelCase	:							List[str]						=			[
         '''fsdp_shard_grad_op_transformer_based_wrap''',
         '''fsdp_full_shard_transformer_based_wrap''',
     ]
     UpperCAmelCase	:							int						=			{
         '''multi_gpu_fp16''': 3_2_0_0,
         '''fsdp_shard_grad_op_transformer_based_wrap_fp16''': 2_0_0_0,
         '''fsdp_full_shard_transformer_based_wrap_fp16''': 1_9_0_0,
         # Disabling below test as it overwhelms the RAM memory usage
         # on CI self-hosted runner leading to tests getting killed.
         # "fsdp_full_shard_cpu_offload_transformer_based_wrap_fp32": 1500,  # fp16 was leading to indefinite hang
     }
     UpperCAmelCase	:							List[str]						=			1_6_0
     UpperCAmelCase	:							Optional[int]						=			1_6_0

     UpperCAmelCase	:							Union[str, Any]						=			inspect.getfile(accelerate.test_utils						)
     UpperCAmelCase	:							Dict						=			os.path.sep.join(mod_file.split(os.path.sep						)[:-1] + ['''scripts''', '''external_deps''']						)



   def 	__magic_name__   (							self							:					str						):
     UpperCAmelCase	:							str						=			os.path.join(self.test_scripts_folder,		'''test_performance.py'''						)
     UpperCAmelCase	:							Any						=			['''accelerate''', '''launch''', '''--num_processes=2''', '''--num_machines=1''', '''--machine_rank=0''', '''--use_fsdp''']
     for config in self.performance_configs:
       UpperCAmelCase	:							Union[str, Any]						=			cmd.copy()
       for i, strategy in enumerate(__A						):
         if strategy.lower() in config:
           cmd_config.append(F'''--fsdp_sharding_strategy={i+1}'''						)
           break

       if "fp32" in config:
         cmd_config.append('''--mixed_precision=no'''						)
       else:
         cmd_config.append('''--mixed_precision=fp16'''						)

       if "cpu_offload" in config:
         cmd_config.append('''--fsdp_offload_params=True'''						)

       for policy in FSDP_AUTO_WRAP_POLICY:
         if policy.lower() in config:
           cmd_config.append(F'''--fsdp_auto_wrap_policy={policy}'''						)
           break

       if policy == "TRANSFORMER_BASED_WRAP":
         cmd_config.append('''--fsdp_transformer_layer_cls_to_wrap=BertLayer'''						)
       elif policy == "SIZE_BASED_WRAP":
         cmd_config.append('''--fsdp_min_num_params=2000'''						)

       cmd_config.extend(
           [
               self.test_file_path,
               F'''--output_dir={self.tmpdir}''',
               F'''--performance_lower_bound={self.performance_lower_bound}''',
           ]						)
       with patch_environment(omp_num_threads=1						):
         execute_subprocess_async(__A,		env=os.environ.copy()						)



   def 	__magic_name__   (							self							:					Tuple						):
     UpperCAmelCase	:							Dict						=			os.path.join(self.test_scripts_folder,		'''test_checkpointing.py'''						)
     UpperCAmelCase	:							Optional[int]						=			[
         '''accelerate''',
         '''launch''',
         '''--num_processes=2''',
         '''--num_machines=1''',
         '''--machine_rank=0''',
         '''--use_fsdp''',
         '''--mixed_precision=fp16''',
         '''--fsdp_transformer_layer_cls_to_wrap=BertLayer''',
     ]

     for i, strategy in enumerate(__A						):
       UpperCAmelCase	:							Optional[Any]						=			cmd.copy()
       cmd_config.append(F'''--fsdp_sharding_strategy={i+1}'''						)
       if strategy != "FULL_SHARD":
         continue
       UpperCAmelCase	:							Any						=			len(__A						)
       for state_dict_type in FSDP_STATE_DICT_TYPE:
         UpperCAmelCase	:							List[Any]						=			cmd_config[:state_dict_config_index]
         cmd_config.append(F'''--fsdp_state_dict_type={state_dict_type}'''						)
         cmd_config.extend(
             [
                 self.test_file_path,
                 F'''--output_dir={self.tmpdir}''',
                 '''--partial_train_epoch=1''',
             ]						)
         with patch_environment(omp_num_threads=1						):
           execute_subprocess_async(__A,		env=os.environ.copy()						)

         UpperCAmelCase	:							Tuple						=			cmd_config[:-1]
         UpperCAmelCase	:							List[str]						=			os.path.join(self.tmpdir,		'''epoch_0'''						)
         cmd_config.extend(
             [
                 F'''--resume_from_checkpoint={resume_from_checkpoint}''',
             ]						)
         with patch_environment(omp_num_threads=1						):
           execute_subprocess_async(__A,		env=os.environ.copy()						)





   def 	__magic_name__   (							self							:					Any						):
     UpperCAmelCase	:							Optional[Any]						=			os.path.join(self.test_scripts_folder,		'''test_peak_memory_usage.py'''						)
     UpperCAmelCase	:							str						=			[
         '''accelerate''',
         '''launch''',
         '''--num_processes=2''',
         '''--num_machines=1''',
         '''--machine_rank=0''',
     ]
     for spec, peak_mem_upper_bound in self.peak_memory_usage_upper_bound.items():
       UpperCAmelCase	:							List[Any]						=			cmd.copy()
       if "fp16" in spec:
         cmd_config.extend(['''--mixed_precision=fp16''']						)
       else:
         cmd_config.extend(['''--mixed_precision=no''']						)

       if "multi_gpu" in spec:
         continue
       else:
         cmd_config.extend(['''--use_fsdp''']						)
         for i, strategy in enumerate(__A						):
           if strategy.lower() in spec:
             cmd_config.append(F'''--fsdp_sharding_strategy={i+1}'''						)
             break

         if "cpu_offload" in spec:
           cmd_config.append('''--fsdp_offload_params=True'''						)

         for policy in FSDP_AUTO_WRAP_POLICY:
           if policy.lower() in spec:
             cmd_config.append(F'''--fsdp_auto_wrap_policy={policy}'''						)
             break

         if policy == "TRANSFORMER_BASED_WRAP":
           cmd_config.append('''--fsdp_transformer_layer_cls_to_wrap=BertLayer'''						)
         elif policy == "SIZE_BASED_WRAP":
           cmd_config.append('''--fsdp_min_num_params=2000'''						)

       cmd_config.extend(
           [
               self.test_file_path,
               F'''--output_dir={self.tmpdir}''',
               F'''--peak_memory_upper_bound={peak_mem_upper_bound}''',
               F'''--n_train={self.n_train}''',
               F'''--n_val={self.n_val}''',
           ]						)
       with patch_environment(omp_num_threads=1						):
         execute_subprocess_async(__A,		env=os.environ.copy()						)





