



import os

import torch

from ..logging import get_logger
from .constants import FSDP_PYTORCH_VERSION, MODEL_NAME, OPTIMIZER_NAME
from .versions import is_torch_version


if is_torch_version('''>=''', FSDP_PYTORCH_VERSION):
			import torch.distributed.checkpoint as dist_cp
			from torch.distributed.checkpoint.default_planner import DefaultLoadPlanner, DefaultSavePlanner
			from torch.distributed.checkpoint.optimizer import load_sharded_optimizer_state_dict
			from torch.distributed.fsdp.fully_sharded_data_parallel import FullyShardedDataParallel as FSDP
			from torch.distributed.fsdp.fully_sharded_data_parallel import StateDictType


lowerCAmelCase__		:					Optional[int]     						=get_logger(__name__)






def      __lowercase       (			a__							,       a__							,       a__							,       a__							,       a__=0       )    ->			int:
							os.makedirs(a__							,       exist_ok=a__       )
							with FSDP.state_dict_type(
							    a__							,       fsdp_plugin.state_dict_type							,       fsdp_plugin.state_dict_config							,       fsdp_plugin.optim_state_dict_config       ):
														__SCREAMING_SNAKE_CASE									= model.state_dict()
														if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
																					__SCREAMING_SNAKE_CASE									= f"""{MODEL_NAME}.bin""" if model_index == 0 else f"""{MODEL_NAME}_{model_index}.bin"""
																					__SCREAMING_SNAKE_CASE									= os.path.join(a__							,       a__       )
																					if accelerator.process_index == 0:
																												logger.info(f"""Saving model to {output_model_file}"""       )
																												torch.save(a__							,       a__       )
																												logger.info(f"""Model saved to {output_model_file}"""       )
														elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT:
																					__SCREAMING_SNAKE_CASE									= (
																					    f"""{MODEL_NAME}_rank{accelerator.process_index}.bin"""
																					    if model_index == 0
																					    else f"""{MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin"""
																					)
																					__SCREAMING_SNAKE_CASE									= os.path.join(a__							,       a__       )
																					logger.info(f"""Saving model to {output_model_file}"""       )
																					torch.save(a__							,       a__       )
																					logger.info(f"""Model saved to {output_model_file}"""       )
														elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT:
																					__SCREAMING_SNAKE_CASE									= os.path.join(a__							,       f"""{MODEL_NAME}_{model_index}"""       )
																					os.makedirs(a__							,       exist_ok=a__       )
																					logger.info(f"""Saving model to {ckpt_dir}"""       )
																					__SCREAMING_SNAKE_CASE									= {'model': state_dict}

																					dist_cp.save_state_dict(
																					    state_dict=a__							,       storage_writer=dist_cp.FileSystemWriter(a__       )							,       planner=DefaultSavePlanner()							,       )
																					logger.info(f"""Model saved to {ckpt_dir}"""       )






def      __lowercase       (			a__							,       a__							,       a__							,       a__							,       a__=0       )    ->			List[str]:
							accelerator.wait_for_everyone()
							with FSDP.state_dict_type(
							    a__							,       fsdp_plugin.state_dict_type							,       fsdp_plugin.state_dict_config							,       fsdp_plugin.optim_state_dict_config       ):
														if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
																					if type(a__       ) != FSDP and accelerator.process_index != 0:
																												if not fsdp_plugin.sync_module_states:
																																			raise ValueError(
																																			    'Set the `sync_module_states` flag to `True` so that model states are synced across processes when '
																																			    'initializing FSDP object'       )
																												return
																					__SCREAMING_SNAKE_CASE									= f"""{MODEL_NAME}.bin""" if model_index == 0 else f"""{MODEL_NAME}_{model_index}.bin"""
																					__SCREAMING_SNAKE_CASE									= os.path.join(a__							,       a__       )
																					logger.info(f"""Loading model from {input_model_file}"""       )
																					__SCREAMING_SNAKE_CASE									= torch.load(a__       )
																					logger.info(f"""Model loaded from {input_model_file}"""       )
														elif fsdp_plugin.state_dict_type == StateDictType.LOCAL_STATE_DICT:
																					__SCREAMING_SNAKE_CASE									= (
																					    f"""{MODEL_NAME}_rank{accelerator.process_index}.bin"""
																					    if model_index == 0
																					    else f"""{MODEL_NAME}_{model_index}_rank{accelerator.process_index}.bin"""
																					)
																					__SCREAMING_SNAKE_CASE									= os.path.join(a__							,       a__       )
																					logger.info(f"""Loading model from {input_model_file}"""       )
																					__SCREAMING_SNAKE_CASE									= torch.load(a__       )
																					logger.info(f"""Model loaded from {input_model_file}"""       )
														elif fsdp_plugin.state_dict_type == StateDictType.SHARDED_STATE_DICT:
																					__SCREAMING_SNAKE_CASE									= (
																					    os.path.join(a__							,       f"""{MODEL_NAME}_{model_index}"""       )
																					    if f"""{MODEL_NAME}""" not in input_dir
																					    else input_dir
																					)
																					logger.info(f"""Loading model from {ckpt_dir}"""       )
																					__SCREAMING_SNAKE_CASE									= {'model': model.state_dict()}
																					dist_cp.load_state_dict(
																					    state_dict=a__							,       storage_reader=dist_cp.FileSystemReader(a__       )							,       planner=DefaultLoadPlanner()							,       )
																					__SCREAMING_SNAKE_CASE									= state_dict['model']
																					logger.info(f"""Model loaded from {ckpt_dir}"""       )
														model.load_state_dict(a__       )






def      __lowercase       (			a__							,       a__							,       a__							,       a__							,       a__							,       a__=0       )    ->			List[Any]:
							os.makedirs(a__							,       exist_ok=a__       )
							with FSDP.state_dict_type(
							    a__							,       fsdp_plugin.state_dict_type							,       fsdp_plugin.state_dict_config							,       fsdp_plugin.optim_state_dict_config       ):
														__SCREAMING_SNAKE_CASE									= FSDP.optim_state_dict(a__							,       a__       )
														if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
																					if accelerator.process_index == 0:
																												__SCREAMING_SNAKE_CASE									= (
																												    f"""{OPTIMIZER_NAME}.bin""" if optimizer_index == 0 else f"""{OPTIMIZER_NAME}_{optimizer_index}.bin"""
																												)
																												__SCREAMING_SNAKE_CASE									= os.path.join(a__							,       a__       )
																												logger.info(f"""Saving Optimizer state to {output_optimizer_file}"""       )
																												torch.save(a__							,       a__       )
																												logger.info(f"""Optimizer state saved in {output_optimizer_file}"""       )
														else:
																					__SCREAMING_SNAKE_CASE									= os.path.join(a__							,       f"""{OPTIMIZER_NAME}_{optimizer_index}"""       )
																					os.makedirs(a__							,       exist_ok=a__       )
																					logger.info(f"""Saving Optimizer state to {ckpt_dir}"""       )
																					dist_cp.save_state_dict(
																					    state_dict={'optimizer': optim_state}							,       storage_writer=dist_cp.FileSystemWriter(a__       )							,       planner=DefaultSavePlanner()							,       )
																					logger.info(f"""Optimizer state saved in {ckpt_dir}"""       )






def      __lowercase       (			a__							,       a__							,       a__							,       a__							,       a__							,       a__=0       )    ->			str:
							accelerator.wait_for_everyone()
							with FSDP.state_dict_type(
							    a__							,       fsdp_plugin.state_dict_type							,       fsdp_plugin.state_dict_config							,       fsdp_plugin.optim_state_dict_config       ):
														if fsdp_plugin.state_dict_type == StateDictType.FULL_STATE_DICT:
																					__SCREAMING_SNAKE_CASE									= None
																					# below check should work but currently it isn't working (mostly opytorch issue),
																					# in the meantime disabling it at the cost of excess memory usage
																					# if accelerator.process_index == 0 or not fsdp_plugin.optim_state_dict_config.rank0_only:
																					__SCREAMING_SNAKE_CASE									= (
																					    f"""{OPTIMIZER_NAME}.bin""" if optimizer_index == 0 else f"""{OPTIMIZER_NAME}_{optimizer_index}.bin"""
																					)
																					__SCREAMING_SNAKE_CASE									= os.path.join(a__							,       a__       )
																					logger.info(f"""Loading Optimizer state from {input_optimizer_file}"""       )
																					__SCREAMING_SNAKE_CASE									= torch.load(a__       )
																					logger.info(f"""Optimizer state loaded from {input_optimizer_file}"""       )
														else:
																					__SCREAMING_SNAKE_CASE									= (
																					    os.path.join(a__							,       f"""{OPTIMIZER_NAME}_{optimizer_index}"""       )
																					    if f"""{OPTIMIZER_NAME}""" not in input_dir
																					    else input_dir
																					)
																					logger.info(f"""Loading Optimizer from {ckpt_dir}"""       )
																					__SCREAMING_SNAKE_CASE									= load_sharded_optimizer_state_dict(
																					    model_state_dict=model.state_dict()							,       optimizer_key='optimizer'							,       storage_reader=dist_cp.FileSystemReader(a__       )							,       )
																					__SCREAMING_SNAKE_CASE									= optim_state['optimizer']
																					logger.info(f"""Optimizer loaded from {ckpt_dir}"""       )
														__SCREAMING_SNAKE_CASE									= FSDP.optim_state_dict_to_load(a__							,       a__							,       a__       )
														optimizer.load_state_dict(a__       )











import argparse
import json

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image
from torchvision import transforms

from transformers import BitImageProcessor, FocalNetConfig, FocalNetForImageClassification
from transformers.image_utils import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD, PILImageResampling






def      __lowercase       (			a__       )    ->			Optional[Any]:
							__SCREAMING_SNAKE_CASE									= [2, 2, 6, 2] if 'tiny' in model_name else [2, 2, 18, 2]
							__SCREAMING_SNAKE_CASE									= True if 'large' in model_name or 'huge' in model_name else False
							__SCREAMING_SNAKE_CASE									= True if 'large' in model_name or 'huge' in model_name else False
							__SCREAMING_SNAKE_CASE									= True if 'large' in model_name or 'huge' in model_name else False

							if "large" in model_name or "xlarge" in model_name or "huge" in model_name:
														if "fl3" in model_name:
																					__SCREAMING_SNAKE_CASE									= [3, 3, 3, 3]
																					__SCREAMING_SNAKE_CASE									= [5, 5, 5, 5]
														elif "fl4" in model_name:
																					__SCREAMING_SNAKE_CASE									= [4, 4, 4, 4]
																					__SCREAMING_SNAKE_CASE									= [3, 3, 3, 3]

							if "tiny" in model_name or "small" in model_name or "base" in model_name:
														__SCREAMING_SNAKE_CASE									= [3, 3, 3, 3]
														if "lrf" in model_name:
																					__SCREAMING_SNAKE_CASE									= [3, 3, 3, 3]
														else:
																					__SCREAMING_SNAKE_CASE									= [2, 2, 2, 2]

							if "tiny" in model_name:
														__SCREAMING_SNAKE_CASE									= 96
							elif "small" in model_name:
														__SCREAMING_SNAKE_CASE									= 96
							elif "base" in model_name:
														__SCREAMING_SNAKE_CASE									= 1_28
							elif "large" in model_name:
														__SCREAMING_SNAKE_CASE									= 1_92
							elif "xlarge" in model_name:
														__SCREAMING_SNAKE_CASE									= 2_56
							elif "huge" in model_name:
														__SCREAMING_SNAKE_CASE									= 3_52

							# set label information
							__SCREAMING_SNAKE_CASE									= 'huggingface/label-files'
							if "large" in model_name or "huge" in model_name:
														__SCREAMING_SNAKE_CASE									= 'imagenet-22k-id2label.json'
							else:
														__SCREAMING_SNAKE_CASE									= 'imagenet-1k-id2label.json'

							__SCREAMING_SNAKE_CASE									= json.load(open(hf_hub_download(a__							,       a__							,       repo_type='dataset'       )							,       'r'       )       )
							__SCREAMING_SNAKE_CASE									= {int(a__       ): v for k, v in idalabel.items()}
							__SCREAMING_SNAKE_CASE									= {v: k for k, v in idalabel.items()}

							__SCREAMING_SNAKE_CASE									= FocalNetConfig(
							    embed_dim=a__							,       depths=a__							,       focal_levels=a__							,       focal_windows=a__							,       use_conv_embed=a__							,       idalabel=a__							,       labelaid=a__							,       use_post_layernorm=a__							,       use_layerscale=a__							,       )

							return config






def      __lowercase       (			a__       )    ->			Any:
							if "patch_embed.proj" in name:
														__SCREAMING_SNAKE_CASE									= name.replace('patch_embed.proj'							,       'embeddings.patch_embeddings.projection'       )
							if "patch_embed.norm" in name:
														__SCREAMING_SNAKE_CASE									= name.replace('patch_embed.norm'							,       'embeddings.norm'       )
							if "layers" in name:
														__SCREAMING_SNAKE_CASE									= 'encoder.' + name
							if "encoder.layers" in name:
														__SCREAMING_SNAKE_CASE									= name.replace('encoder.layers'							,       'encoder.stages'       )
							if "downsample.proj" in name:
														__SCREAMING_SNAKE_CASE									= name.replace('downsample.proj'							,       'downsample.projection'       )
							if "blocks" in name:
														__SCREAMING_SNAKE_CASE									= name.replace('blocks'							,       'layers'       )
							if "modulation.f.weight" in name or "modulation.f.bias" in name:
														__SCREAMING_SNAKE_CASE									= name.replace('modulation.f'							,       'modulation.projection_in'       )
							if "modulation.h.weight" in name or "modulation.h.bias" in name:
														__SCREAMING_SNAKE_CASE									= name.replace('modulation.h'							,       'modulation.projection_context'       )
							if "modulation.proj.weight" in name or "modulation.proj.bias" in name:
														__SCREAMING_SNAKE_CASE									= name.replace('modulation.proj'							,       'modulation.projection_out'       )

							if name == "norm.weight":
														__SCREAMING_SNAKE_CASE									= 'layernorm.weight'
							if name == "norm.bias":
														__SCREAMING_SNAKE_CASE									= 'layernorm.bias'

							if "head" in name:
														__SCREAMING_SNAKE_CASE									= name.replace('head'							,       'classifier'       )
							else:
														__SCREAMING_SNAKE_CASE									= 'focalnet.' + name

							return name






def      __lowercase       (			a__							,       a__							,       a__=False       )    ->			Dict:
							# fmt: off
							__SCREAMING_SNAKE_CASE									= {
							    'focalnet-tiny': 'https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_tiny_srf.pth',
							    'focalnet-tiny-lrf': 'https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_tiny_lrf.pth',
							    'focalnet-small': 'https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_small_srf.pth',
							    'focalnet-small-lrf': 'https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_small_lrf.pth',
							    'focalnet-base': 'https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_base_srf.pth',
							    'focalnet-base-lrf': 'https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_base_lrf.pth',
							    'focalnet-large-lrf-fl3': 'https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_large_lrf_384.pth',
							    'focalnet-large-lrf-fl4': 'https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_large_lrf_384_fl4.pth',
							    'focalnet-xlarge-lrf-fl3': 'https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_xlarge_lrf_384.pth',
							    'focalnet-xlarge-lrf-fl4': 'https://projects4jw.blob.core.windows.net/focalnet/release/classification/focalnet_xlarge_lrf_384_fl4.pth',
							}
							# fmt: on

							__SCREAMING_SNAKE_CASE									= model_name_to_url[model_name]
							print('Checkpoint URL: '							,       a__       )
							__SCREAMING_SNAKE_CASE									= torch.hub.load_state_dict_from_url(a__							,       map_location='cpu'       )['model']

							# rename keys
							for key in state_dict.copy().keys():
														__SCREAMING_SNAKE_CASE									= state_dict.pop(a__       )
														__SCREAMING_SNAKE_CASE									= val

							__SCREAMING_SNAKE_CASE									= get_focalnet_config(a__       )
							__SCREAMING_SNAKE_CASE									= FocalNetForImageClassification(a__       )
							model.eval()

							# load state dict
							model.load_state_dict(a__       )

							# verify conversion
							__SCREAMING_SNAKE_CASE									= 'http://images.cocodataset.org/val2017/000000039769.jpg'

							__SCREAMING_SNAKE_CASE									= BitImageProcessor(
							    do_resize=a__							,       size={'shortest_edge': 2_56}							,       resample=PILImageResampling.BILINEAR							,       do_center_crop=a__							,       crop_size=2_24							,       do_normalize=a__							,       image_mean=a__							,       image_std=a__							,       )
							__SCREAMING_SNAKE_CASE									= Image.open(requests.get(a__							,       stream=a__       ).raw       )
							__SCREAMING_SNAKE_CASE									= processor(images=a__							,       return_tensors='pt'       )

							__SCREAMING_SNAKE_CASE									= transforms.Compose(
							    [
							        transforms.Resize(2_56       ),
							        transforms.CenterCrop(2_24       ),
							        transforms.ToTensor(),
							        transforms.Normalize(mean=[0.485, 0.456, 0.406]							,       std=[0.229, 0.224, 0.225]       ),
							    ]       )

							__SCREAMING_SNAKE_CASE									= image_transforms(a__       ).unsqueeze(0       )

							# verify pixel_values
							assert torch.allclose(inputs.pixel_values							,       a__							,       atol=1E-4       )

							__SCREAMING_SNAKE_CASE									= model(**a__       )

							__SCREAMING_SNAKE_CASE									= outputs.logits.argmax(-1       ).item()
							print('Predicted class:'							,       model.config.idalabel[predicted_class_idx]       )

							print('First values of logits:'							,       outputs.logits[0, :3]       )

							if model_name == "focalnet-tiny":
														__SCREAMING_SNAKE_CASE									= torch.tensor([0.2166, -0.4368, 0.2191]       )
							elif model_name == "focalnet-tiny-lrf":
														__SCREAMING_SNAKE_CASE									= torch.tensor([1.1669, 0.0125, -0.1695]       )
							elif model_name == "focalnet-small":
														__SCREAMING_SNAKE_CASE									= torch.tensor([0.4917, -0.0430, 0.1341]       )
							elif model_name == "focalnet-small-lrf":
														__SCREAMING_SNAKE_CASE									= torch.tensor([-0.2588, -0.5342, -0.2331]       )
							elif model_name == "focalnet-base":
														__SCREAMING_SNAKE_CASE									= torch.tensor([-0.1655, -0.4090, -0.1730]       )
							elif model_name == "focalnet-base-lrf":
														__SCREAMING_SNAKE_CASE									= torch.tensor([0.5306, -0.0483, -0.3928]       )
							assert torch.allclose(outputs.logits[0, :3]							,       a__							,       atol=1E-4       )
							print('Looks ok!'       )

							if pytorch_dump_folder_path is not None:
														print(f"""Saving model and processor of {model_name} to {pytorch_dump_folder_path}"""       )
														model.save_pretrained(a__       )
														processor.save_pretrained(a__       )

							if push_to_hub:
														print(f"""Pushing model and processor of {model_name} to the hub..."""       )
														model.push_to_hub(f"""{model_name}"""       )
														processor.push_to_hub(f"""{model_name}"""       )


if __name__ == "__main__":
			lowerCAmelCase__		:					int							=argparse.ArgumentParser()
			# Required parameters
			parser.add_argument(
			    '''--model_name''',
			    default='''focalnet-tiny''',
			    type=str,
			    help='''Name of the FocalNet model you\'d like to convert.''',
			)
			parser.add_argument(
			    '''--pytorch_dump_folder_path''', default=None, type=str, help='''Path to the output PyTorch model directory.'''
			)
			parser.add_argument(
			    '''--push_to_hub''',
			    action='''store_true''',
			    help='''Whether to push the model and processor to the hub.''',
			)

			lowerCAmelCase__		:					List[Any]							=parser.parse_args()
			convert_focalnet_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)






