


import inspect
import os

import torch
from transformers import AutoModel
from transformers.testing_utils import mockenv_context
from transformers.trainer_utils import set_seed

import accelerate
from accelerate.accelerator import Accelerator
from accelerate.state import AcceleratorState
from accelerate.test_utils.testing import (
    AccelerateTestCase,
    TempDirTestCase,
    execute_subprocess_async,
    require_cuda,
    require_fsdp,
    require_multi_gpu,
    slow,
)
from accelerate.utils.constants import (
    FSDP_AUTO_WRAP_POLICY,
    FSDP_BACKWARD_PREFETCH,
    FSDP_SHARDING_STRATEGY,
    FSDP_STATE_DICT_TYPE,
)
from accelerate.utils.dataclasses import FullyShardedDataParallelPlugin
from accelerate.utils.other import patch_environment


set_seed(42)

_A			=						'bert-base-cased'
_A			=						'fp16'
_A			=						'bf16'
_A			=						[FPaa, BFaa]






@require_fsdp
@require_cuda
class    UpperCAmelCase__	( A_		):



    """simple docstring"""






    def _a (      self   )  ->	Any:
         super().setUp()

         __UpperCamelCase   							=dict(
             ACCELERATE_USE_FSDP='true' ,       MASTER_ADDR='localhost' ,       MASTER_PORT='10999' ,       RANK='0' ,       LOCAL_RANK='0' ,       WORLD_SIZE='1' ,       )






    def _a (      self   )  ->	Optional[int]:
         from torch.distributed.fsdp.fully_sharded_data_parallel import ShardingStrategy

         for i, strategy in enumerate(A_   ):
              __UpperCamelCase   							=self.dist_env.copy()
              __UpperCamelCase   							=f'{i + 1}'
              __UpperCamelCase   							=strategy
              with mockenv_context(**A_   ):
                   __UpperCamelCase   							=FullyShardedDataParallelPlugin()
                   self.assertEqual(fsdp_plugin.sharding_strategy ,       ShardingStrategy(i + 1   )   )






    def _a (      self   )  ->	Tuple:
         from torch.distributed.fsdp.fully_sharded_data_parallel import BackwardPrefetch

         for i, prefetch_policy in enumerate(A_   ):
              __UpperCamelCase   							=self.dist_env.copy()
              __UpperCamelCase   							=prefetch_policy
              with mockenv_context(**A_   ):
                   __UpperCamelCase   							=FullyShardedDataParallelPlugin()
                   if prefetch_policy == "NO_PREFETCH":
                        self.assertIsNone(fsdp_plugin.backward_prefetch   )
                   else:
                        self.assertEqual(fsdp_plugin.backward_prefetch ,       BackwardPrefetch(i + 1   )   )






    def _a (      self   )  ->	Any:
         from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType

         for i, state_dict_type in enumerate(A_   ):
              __UpperCamelCase   							=self.dist_env.copy()
              __UpperCamelCase   							=state_dict_type
              with mockenv_context(**A_   ):
                   __UpperCamelCase   							=FullyShardedDataParallelPlugin()
                   self.assertEqual(fsdp_plugin.state_dict_type ,       StateDictType(i + 1   )   )
                   if state_dict_type == "FULL_STATE_DICT":
                        self.assertTrue(fsdp_plugin.state_dict_config.offload_to_cpu   )
                        self.assertTrue(fsdp_plugin.state_dict_config.ranka_only   )






    def _a (      self   )  ->	List[Any]:
         __UpperCamelCase   							=AutoModel.from_pretrained(A_   )
         for policy in FSDP_AUTO_WRAP_POLICY:
              __UpperCamelCase   							=self.dist_env.copy()
              __UpperCamelCase   							=policy
              if policy == "TRANSFORMER_BASED_WRAP":
                   __UpperCamelCase   							='BertLayer'
              elif policy == "SIZE_BASED_WRAP":
                   __UpperCamelCase   							='2000'
              with mockenv_context(**A_   ):
                   __UpperCamelCase   							=FullyShardedDataParallelPlugin()
                   fsdp_plugin.set_auto_wrap_policy(A_   )
                   if policy == "NO_WRAP":
                        self.assertIsNone(fsdp_plugin.auto_wrap_policy   )
                   else:
                        self.assertIsNotNone(fsdp_plugin.auto_wrap_policy   )

         __UpperCamelCase   							=self.dist_env.copy()
         __UpperCamelCase   							='TRANSFORMER_BASED_WRAP'
         __UpperCamelCase   							='T5Layer'
         with mockenv_context(**A_   ):
              __UpperCamelCase   							=FullyShardedDataParallelPlugin()
              with self.assertRaises(A_   ) as cm:
                   fsdp_plugin.set_auto_wrap_policy(A_   )
              self.assertTrue('Could not find the transformer layer class to wrap in the model.' in str(cm.exception   )   )

         __UpperCamelCase   							=self.dist_env.copy()
         __UpperCamelCase   							='SIZE_BASED_WRAP'
         __UpperCamelCase   							='0'
         with mockenv_context(**A_   ):
              __UpperCamelCase   							=FullyShardedDataParallelPlugin()
              fsdp_plugin.set_auto_wrap_policy(A_   )
              self.assertIsNone(fsdp_plugin.auto_wrap_policy   )






    def _a (      self   )  ->	int:
         from torch.distributed.fsdp.fully_sharded_data_parallel import MixedPrecision
         from torch.distributed.fsdp.sharded_grad_scaler import ShardedGradScaler

         for mp_dtype in dtypes:
              __UpperCamelCase   							=self.dist_env.copy()
              __UpperCamelCase   							=mp_dtype
              with mockenv_context(**A_   ):
                   __UpperCamelCase   							=Accelerator()
                   if mp_dtype == "fp16":
                        __UpperCamelCase   							=torch.floataa
                   elif mp_dtype == "bf16":
                        __UpperCamelCase   							=torch.bfloataa
                   __UpperCamelCase   							=MixedPrecision(param_dtype=A_ ,       reduce_dtype=A_ ,       buffer_dtype=A_   )
                   self.assertEqual(accelerator.state.fsdp_plugin.mixed_precision_policy ,       A_   )
                   if mp_dtype == FPaa:
                        self.assertTrue(isinstance(accelerator.scaler ,       A_   )   )
                   elif mp_dtype == BFaa:
                        self.assertIsNone(accelerator.scaler   )
                   AcceleratorState._reset_state(A_   )






    def _a (      self   )  ->	int:
         from torch.distributed.fsdp.fully_sharded_data_parallel import CPUOffload

         for flag in [True, False]:
              __UpperCamelCase   							=self.dist_env.copy()
              __UpperCamelCase   							=str(A_   ).lower()
              with mockenv_context(**A_   ):
                   __UpperCamelCase   							=FullyShardedDataParallelPlugin()
                   self.assertEqual(fsdp_plugin.cpu_offload ,       CPUOffload(offload_params=A_   )   )






@require_fsdp
@require_multi_gpu
@slow
class    UpperCAmelCase__	( A_		):



    """simple docstring"""






    def _a (      self   )  ->	Any:
         super().setUp()
         __UpperCamelCase   							=0.82
         __UpperCamelCase   							=[
             'fsdp_shard_grad_op_transformer_based_wrap',
             'fsdp_full_shard_transformer_based_wrap',
         ]
         __UpperCamelCase   							={
             'multi_gpu_fp16': 3200,
             'fsdp_shard_grad_op_transformer_based_wrap_fp16': 2000,
             'fsdp_full_shard_transformer_based_wrap_fp16': 1900,
             # Disabling below test as it overwhelms the RAM memory usage
             # on CI self-hosted runner leading to tests getting killed.
             # "fsdp_full_shard_cpu_offload_transformer_based_wrap_fp32": 1500,  # fp16 was leading to indefinite hang
         }
         __UpperCamelCase   							=160
         __UpperCamelCase   							=160

         __UpperCamelCase   							=inspect.getfile(accelerate.test_utils   )
         __UpperCamelCase   							=os.path.sep.join(mod_file.split(os.path.sep   )[:-1] + ['scripts', 'external_deps']   )






    def _a (      self   )  ->	Union[str, Any]:
         __UpperCamelCase   							=os.path.join(self.test_scripts_folder ,       'test_performance.py'   )
         __UpperCamelCase   							=['accelerate', 'launch', '--num_processes=2', '--num_machines=1', '--machine_rank=0', '--use_fsdp']
         for config in self.performance_configs:
              __UpperCamelCase   							=cmd.copy()
              for i, strategy in enumerate(A_   ):
                   if strategy.lower() in config:
                        cmd_config.append(f'--fsdp_sharding_strategy={i+1}'   )
                        break

              if "fp32" in config:
                   cmd_config.append('--mixed_precision=no'   )
              else:
                   cmd_config.append('--mixed_precision=fp16'   )

              if "cpu_offload" in config:
                   cmd_config.append('--fsdp_offload_params=True'   )

              for policy in FSDP_AUTO_WRAP_POLICY:
                   if policy.lower() in config:
                        cmd_config.append(f'--fsdp_auto_wrap_policy={policy}'   )
                        break

              if policy == "TRANSFORMER_BASED_WRAP":
                   cmd_config.append('--fsdp_transformer_layer_cls_to_wrap=BertLayer'   )
              elif policy == "SIZE_BASED_WRAP":
                   cmd_config.append('--fsdp_min_num_params=2000'   )

              cmd_config.extend(
                  [
                      self.test_file_path,
                      f'--output_dir={self.tmpdir}',
                      f'--performance_lower_bound={self.performance_lower_bound}',
                  ]   )
              with patch_environment(omp_num_threads=1   ):
                   execute_subprocess_async(A_ ,       env=os.environ.copy()   )






    def _a (      self   )  ->	int:
         __UpperCamelCase   							=os.path.join(self.test_scripts_folder ,       'test_checkpointing.py'   )
         __UpperCamelCase   							=[
             'accelerate',
             'launch',
             '--num_processes=2',
             '--num_machines=1',
             '--machine_rank=0',
             '--use_fsdp',
             '--mixed_precision=fp16',
             '--fsdp_transformer_layer_cls_to_wrap=BertLayer',
         ]

         for i, strategy in enumerate(A_   ):
              __UpperCamelCase   							=cmd.copy()
              cmd_config.append(f'--fsdp_sharding_strategy={i+1}'   )
              if strategy != "FULL_SHARD":
                   continue
              __UpperCamelCase   							=len(A_   )
              for state_dict_type in FSDP_STATE_DICT_TYPE:
                   __UpperCamelCase   							=cmd_config[:state_dict_config_index]
                   cmd_config.append(f'--fsdp_state_dict_type={state_dict_type}'   )
                   cmd_config.extend(
                       [
                           self.test_file_path,
                           f'--output_dir={self.tmpdir}',
                           '--partial_train_epoch=1',
                       ]   )
                   with patch_environment(omp_num_threads=1   ):
                        execute_subprocess_async(A_ ,       env=os.environ.copy()   )

                   __UpperCamelCase   							=cmd_config[:-1]
                   __UpperCamelCase   							=os.path.join(self.tmpdir ,       'epoch_0'   )
                   cmd_config.extend(
                       [
                           f'--resume_from_checkpoint={resume_from_checkpoint}',
                       ]   )
                   with patch_environment(omp_num_threads=1   ):
                        execute_subprocess_async(A_ ,       env=os.environ.copy()   )






    def _a (      self   )  ->	Optional[Any]:
         __UpperCamelCase   							=os.path.join(self.test_scripts_folder ,       'test_peak_memory_usage.py'   )
         __UpperCamelCase   							=[
             'accelerate',
             'launch',
             '--num_processes=2',
             '--num_machines=1',
             '--machine_rank=0',
         ]
         for spec, peak_mem_upper_bound in self.peak_memory_usage_upper_bound.items():
              __UpperCamelCase   							=cmd.copy()
              if "fp16" in spec:
                   cmd_config.extend(['--mixed_precision=fp16']   )
              else:
                   cmd_config.extend(['--mixed_precision=no']   )

              if "multi_gpu" in spec:
                   continue
              else:
                   cmd_config.extend(['--use_fsdp']   )
                   for i, strategy in enumerate(A_   ):
                        if strategy.lower() in spec:
                             cmd_config.append(f'--fsdp_sharding_strategy={i+1}'   )
                             break

                   if "cpu_offload" in spec:
                        cmd_config.append('--fsdp_offload_params=True'   )

                   for policy in FSDP_AUTO_WRAP_POLICY:
                        if policy.lower() in spec:
                             cmd_config.append(f'--fsdp_auto_wrap_policy={policy}'   )
                             break

                   if policy == "TRANSFORMER_BASED_WRAP":
                        cmd_config.append('--fsdp_transformer_layer_cls_to_wrap=BertLayer'   )
                   elif policy == "SIZE_BASED_WRAP":
                        cmd_config.append('--fsdp_min_num_params=2000'   )

              cmd_config.extend(
                  [
                      self.test_file_path,
                      f'--output_dir={self.tmpdir}',
                      f'--peak_memory_upper_bound={peak_mem_upper_bound}',
                      f'--n_train={self.n_train}',
                      f'--n_val={self.n_val}',
                  ]   )
              with patch_environment(omp_num_threads=1   ):
                   execute_subprocess_async(A_ ,       env=os.environ.copy()   )









import sys

from .dependency_versions_table import deps
from .utils.versions import require_version, require_version_core


# define which module versions we always want to check at run time
# (usually the ones defined in `install_requires` in setup.py)
#
# order specific notes:
# - tqdm must be checked before tokenizers

_A			=						'python tqdm regex requests packaging filelock numpy tokenizers'.split()
if sys.version_info < (3, 7):
   pkgs_to_check_at_runtime.append('dataclasses')
if sys.version_info < (3, 8):
   pkgs_to_check_at_runtime.append('importlib_metadata')

for pkg in pkgs_to_check_at_runtime:
   if pkg in deps:
      if pkg == "tokenizers":
         # must be loaded here, or else tqdm check may fail
         from .utils import is_tokenizers_available

         if not is_tokenizers_available():
            continue  # not required, check version only if installed

      require_version_core(deps[pkg])
   else:
      raise ValueError(f"""can't find {pkg} in {deps.keys()}, check dependency_versions_table.py""")

def 						_UpperCAmelCase    (      SCREAMING_SNAKE_CASE__      : Union[str, Any]      ,					SCREAMING_SNAKE_CASE__      : Optional[int]=None    ):
     require_version(deps[pkg]      ,					SCREAMING_SNAKE_CASE__    )





