

import argparse

import requests
import torch
from PIL import Image

from transformers import ViTMAEConfig, ViTMAEForPreTraining, ViTMAEImageProcessor

def 						UpperCamelCase (lowercase_:			Tuple )					->				Union[str, Any]:
	if "cls_token" in name:
		A__							:   Dict					       =		name.replace("""cls_token"""   ,				"""vit.embeddings.cls_token""" )
	if "mask_token" in name:
		A__							:   Tuple					       =		name.replace("""mask_token"""   ,				"""decoder.mask_token""" )
	if "decoder_pos_embed" in name:
		A__							:   int					       =		name.replace("""decoder_pos_embed"""   ,				"""decoder.decoder_pos_embed""" )
	if "pos_embed" in name and "decoder" not in name:
		A__							:   Optional[int]					       =		name.replace("""pos_embed"""   ,				"""vit.embeddings.position_embeddings""" )
	if "patch_embed.proj" in name:
		A__							:   int					       =		name.replace("""patch_embed.proj"""   ,				"""vit.embeddings.patch_embeddings.projection""" )
	if "patch_embed.norm" in name:
		A__							:   Tuple					       =		name.replace("""patch_embed.norm"""   ,				"""vit.embeddings.norm""" )
	if "decoder_blocks" in name:
		A__							:   Dict					       =		name.replace("""decoder_blocks"""   ,				"""decoder.decoder_layers""" )
	if "blocks" in name:
		A__							:   List[str]					       =		name.replace("""blocks"""   ,				"""vit.encoder.layer""" )
	if "attn.proj" in name:
		A__							:   int					       =		name.replace("""attn.proj"""   ,				"""attention.output.dense""" )
	if "attn" in name:
		A__							:   Tuple					       =		name.replace("""attn"""   ,				"""attention.self""" )
	if "norm1" in name:
		A__							:   Optional[Any]					       =		name.replace("""norm1"""   ,				"""layernorm_before""" )
	if "norm2" in name:
		A__							:   Union[str, Any]					       =		name.replace("""norm2"""   ,				"""layernorm_after""" )
	if "mlp.fc1" in name:
		A__							:   str					       =		name.replace("""mlp.fc1"""   ,				"""intermediate.dense""" )
	if "mlp.fc2" in name:
		A__							:   List[Any]					       =		name.replace("""mlp.fc2"""   ,				"""output.dense""" )
	if "decoder_embed" in name:
		A__							:   Optional[Any]					       =		name.replace("""decoder_embed"""   ,				"""decoder.decoder_embed""" )
	if "decoder_norm" in name:
		A__							:   Optional[int]					       =		name.replace("""decoder_norm"""   ,				"""decoder.decoder_norm""" )
	if "decoder_pred" in name:
		A__							:   Optional[Any]					       =		name.replace("""decoder_pred"""   ,				"""decoder.decoder_pred""" )
	if "norm.weight" in name and "decoder" not in name:
		A__							:   Union[str, Any]					       =		name.replace("""norm.weight"""   ,				"""vit.layernorm.weight""" )
	if "norm.bias" in name and "decoder" not in name:
		A__							:   Any					       =		name.replace("""norm.bias"""   ,				"""vit.layernorm.bias""" )

	return name

def 						UpperCamelCase (lowercase_:			Optional[int]   ,				lowercase_:			List[Any] )					->				Union[str, Any]:
	for key in orig_state_dict.copy().keys():
		A__							:   Union[str, Any]					       =		orig_state_dict.pop(__SCREAMING_SNAKE_CASE )

		if "qkv" in key:
			A__							:   Tuple					       =		key.split(""".""" )
			A__							:   Dict					       =		int(key_split[1] )
			if "decoder_blocks" in key:
				A__							:   Optional[Any]					       =		config.decoder_hidden_size
				A__							:   Union[str, Any]					       =		"""decoder.decoder_layers."""
				if "weight" in key:
					A__							:   Optional[Any]					       =		val[:dim, :]
					A__							:   Any					       =		val[dim : dim * 2, :]
					A__							:   Tuple					       =		val[-dim:, :]
				elif "bias" in key:
					A__							:   Dict					       =		val[:dim]
					A__							:   List[Any]					       =		val[dim : dim * 2]
					A__							:   Any					       =		val[-dim:]
			else:
				A__							:   Any					       =		config.hidden_size
				A__							:   Tuple					       =		"""vit.encoder.layer."""
				if "weight" in key:
					A__							:   Tuple					       =		val[:dim, :]
					A__							:   int					       =		val[dim : dim * 2, :]
					A__							:   List[str]					       =		val[-dim:, :]
				elif "bias" in key:
					A__							:   Union[str, Any]					       =		val[:dim]
					A__							:   Tuple					       =		val[dim : dim * 2]
					A__							:   Optional[Any]					       =		val[-dim:]

		else:
			A__							:   List[str]					       =		val

	return orig_state_dict

def 						UpperCamelCase (lowercase_:			int   ,				lowercase_:			Dict )					->				List[str]:
	A__							:   Any					       =		ViTMAEConfig()
	if "large" in checkpoint_url:
		A__							:   Union[str, Any]					       =		1024
		A__							:   List[str]					       =		4096
		A__							:   Optional[int]					       =		24
		A__							:   Optional[int]					       =		16
	elif "huge" in checkpoint_url:
		A__							:   Dict					       =		14
		A__							:   str					       =		1280
		A__							:   List[str]					       =		5120
		A__							:   Any					       =		32
		A__							:   Tuple					       =		16

	A__							:   str					       =		ViTMAEForPreTraining(__SCREAMING_SNAKE_CASE )

	A__							:   Optional[int]					       =		torch.hub.load_state_dict_from_url(__SCREAMING_SNAKE_CASE   ,				map_location="""cpu""" )["""model"""]

	A__							:   List[Any]					       =		ViTMAEImageProcessor(size=config.image_size )

	A__							:   Optional[int]					       =		convert_state_dict(__SCREAMING_SNAKE_CASE   ,				__SCREAMING_SNAKE_CASE )

	model.load_state_dict(__SCREAMING_SNAKE_CASE )
	model.eval()

	A__							:   Tuple					       =		"""https://user-images.githubusercontent.com/11435359/147738734-196fd92f-9260-48d5-ba7e-bf103d29364d.jpg"""

	A__							:   Any					       =		Image.open(requests.get(__SCREAMING_SNAKE_CASE   ,				stream=__SCREAMING_SNAKE_CASE ).raw )
	A__							:   Union[str, Any]					       =		ViTMAEImageProcessor(size=config.image_size )
	A__							:   int					       =		image_processor(images=__SCREAMING_SNAKE_CASE   ,				return_tensors="""pt""" )

	# forward pass
	torch.manual_seed(2 )
	A__							:   str					       =		model(**__SCREAMING_SNAKE_CASE )
	A__							:   Union[str, Any]					       =		outputs.logits

	if "large" in checkpoint_url:
		A__							:   Any					       =		torch.tensor(
		    [[-0.7309, -0.7128, -1.0169], [-1.0161, -0.9058, -1.1878], [-1.0478, -0.9411, -1.1911]] )
	elif "huge" in checkpoint_url:
		A__							:   int					       =		torch.tensor(
		    [[-1.1599, -0.9199, -1.2221], [-1.1952, -0.9269, -1.2307], [-1.2143, -0.9337, -1.2262]] )
	else:
		A__							:   Tuple					       =		torch.tensor(
		    [[-0.9192, -0.8481, -1.1259], [-1.1349, -1.0034, -1.2599], [-1.1757, -1.0429, -1.2726]] )

	# verify logits
	assert torch.allclose(logits[0, :3, :3]   ,				__SCREAMING_SNAKE_CASE   ,				atol=1E-4 )

	print(f"""Saving model to {pytorch_dump_folder_path}""" )
	model.save_pretrained(__SCREAMING_SNAKE_CASE )

	print(f"""Saving image processor to {pytorch_dump_folder_path}""" )
	image_processor.save_pretrained(__SCREAMING_SNAKE_CASE )


if __name__ == "__main__":
	A_	:			int  	=					argparse.ArgumentParser()
	# Required parameters
	parser.add_argument(
	    '--checkpoint_url',
	    default='https://dl.fbaipublicfiles.com/mae/visualize/mae_visualize_vit_base.pth',
	    type=str,
	    help='URL of the checkpoint you\'d like to convert.',
	)
	parser.add_argument(
	    '--pytorch_dump_folder_path', default=None, type=str, help='Path to the output PyTorch model directory.'
	)

	A_	:			Optional[int]  	=					parser.parse_args()
	convert_vit_mae_checkpoint(args.checkpoint_url, args.pytorch_dump_folder_path)






from ...configuration_utils import PretrainedConfig
from ...utils import logging


A_	:			str  	=      logging.get_logger(__name__)

A_	:			Any  	=      {
    'tiiuae/falcon-40b': 'https://huggingface.co/tiiuae/falcon-40b/resolve/main/config.json',
    'tiiuae/falcon-7b': 'https://huggingface.co/tiiuae/falcon-7b/resolve/main/config.json',
}



class    _a						(__magic_name__    ):


					'''simple docstring'''






					UpperCAmelCase__:       Dict  							=	'''falcon'''
					UpperCAmelCase__:       Any  							=	['''past_key_values''']






					def __init__(			self   ,	A__=6_5024   ,	A__=4544   ,	A__=32   ,	A__=71   ,	A__=1e-5   ,	A__=0.0_2   ,	A__=True   ,	A__=0.0   ,	A__=0.0   ,	A__=None   ,	A__=False   ,	A__=False   ,	A__=True   ,	A__=True   ,	A__=False   ,	A__=11   ,	A__=11   ,	**A__   ,	):
						A__							:   Dict					       =		vocab_size
						# Backward compatibility with n_embed kwarg
						A__							:   Union[str, Any]					       =		kwargs.pop("""n_embed"""   ,	A__      )
						A__							:   Optional[Any]					       =		hidden_size if n_embed is None else n_embed
						A__							:   Any					       =		num_hidden_layers
						A__							:   Any					       =		num_attention_heads
						A__							:   Optional[Any]					       =		layer_norm_epsilon
						A__							:   Tuple					       =		initializer_range
						A__							:   Tuple					       =		use_cache
						A__							:   str					       =		hidden_dropout
						A__							:   List[str]					       =		attention_dropout

						A__							:   List[Any]					       =		bos_token_id
						A__							:   Optional[Any]					       =		eos_token_id
						A__							:   Optional[Any]					       =		num_attention_heads if num_kv_heads is None else num_kv_heads
						A__							:   List[str]					       =		alibi
						A__							:   Tuple					       =		new_decoder_architecture
						A__							:   List[str]					       =		multi_query  # Ignored when new_decoder_architecture is True
						A__							:   List[Any]					       =		parallel_attn
						A__							:   int					       =		bias

						super().__init__(bos_token_id=A__   ,	eos_token_id=A__   ,	**A__      )






					@property
					def   __A      (			self      ):
						return self.hidden_size // self.num_attention_heads






					@property
					def   __A      (			self      ):
						return not self.alibi



