

'''simple docstring'''



import argparse

import torch
from huggingface_hub import hf_hub_download

from transformers import AutoTokenizer, RobertaPreLayerNormConfig, RobertaPreLayerNormForMaskedLM
from transformers.utils import logging


logging.set_verbosity_info()
__snake_case			:     Union[str, Any]										=      logging.get_logger(__name__)
def 						__lowerCamelCase	(       __snake_case					:		str,		__snake_case					:		str	)	->							str:






						"""simple docstring"""





						A__					:       Optional[int]     =RobertaPreLayerNormConfig.from_pretrained(
						    __snake_case,		architectures=["""RobertaPreLayerNormForMaskedLM"""]	)

						# convert state_dict
						A__					:       str     =torch.load(hf_hub_download(repo_id=__snake_case,		filename="""pytorch_model.bin"""	)	)
						A__					:       Optional[int]     ={}
						for tensor_key, tensor_value in original_state_dict.items():
												# The transformer implementation gives the model a unique name, rather than overwiriting 'roberta'
												if tensor_key.startswith("""roberta."""	):
																		A__					:       Optional[Any]     ="""roberta_prelayernorm.""" + tensor_key[len("""roberta."""	) :]

												# The original implementation contains weights which are not used, remove them from the state_dict
												if tensor_key.endswith(""".self.LayerNorm.weight"""	) or tensor_key.endswith(""".self.LayerNorm.bias"""	):
																		continue

												A__					:       Tuple     =tensor_value

						A__					:       List[str]     =RobertaPreLayerNormForMaskedLM.from_pretrained(
						    pretrained_model_name_or_path=__snake_case,		config=__snake_case,		state_dict=__snake_case	)
						model.save_pretrained(__snake_case	)

						# convert tokenizer
						A__					:       Tuple     =AutoTokenizer.from_pretrained(__snake_case	)
						tokenizer.save_pretrained(__snake_case	)


if __name__ == "__main__":
		__snake_case			:     str										=      argparse.ArgumentParser()
		# Required parameters
		parser.add_argument(
		    '--checkpoint-repo',
		    default=None,
		    type=str,
		    required=True,
		    help='Path the official PyTorch dump, e.g. \'andreasmadsen/efficient_mlm_m0.40\'.',
		)
		parser.add_argument(
		    '--pytorch_dump_folder_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
		)
		__snake_case			:     str										=      parser.parse_args()
		convert_roberta_prelayernorm_checkpoint_to_pytorch(args.checkpoint_repo, args.pytorch_dump_folder_path)




'''simple docstring'''



import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
		from transformers import LlamaTokenizerFast
except ImportError as e:
		warnings.warn(e)
		warnings.warn(
		    'The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion'
		)
		__snake_case			:     Optional[Any]										=      None
__snake_case			:     Optional[Any]										=      {
    '7B': 1_1008,
    '13B': 1_3824,
    '30B': 1_7920,
    '65B': 2_2016,
    '70B': 2_8672,
}
__snake_case			:     Union[str, Any]										=      {
    '7B': 1,
    '7Bf': 1,
    '13B': 2,
    '13Bf': 2,
    '30B': 4,
    '65B': 8,
    '70B': 8,
    '70Bf': 8,
}
def 						__lowerCamelCase	(       __snake_case					:		Optional[Any],		__snake_case					:		str=1,		__snake_case					:		Tuple=256	)	->							str:






						"""simple docstring"""





						return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3	)	) + multiple_of - 1) // multiple_of)
def 						__lowerCamelCase	(       __snake_case					:		Tuple	)	->							Tuple:






						"""simple docstring"""





						with open(__snake_case,		"""r"""	) as f:
												return json.load(__snake_case	)
def 						__lowerCamelCase	(       __snake_case					:		Optional[int],		__snake_case					:		Tuple	)	->							Dict:






						"""simple docstring"""





						with open(__snake_case,		"""w"""	) as f:
												json.dump(__snake_case,		__snake_case	)
def 						__lowerCamelCase	(       __snake_case					:		List[Any],		__snake_case					:		Any,		__snake_case					:		Any,		__snake_case					:		Tuple=True	)	->							Any:






						"""simple docstring"""





						os.makedirs(__snake_case,		exist_ok=__snake_case	)
						A__					:       List[Any]     =os.path.join(__snake_case,		"""tmp"""	)
						os.makedirs(__snake_case,		exist_ok=__snake_case	)

						A__					:       Dict     =read_json(os.path.join(__snake_case,		"""params.json"""	)	)
						A__					:       Dict     =NUM_SHARDS[model_size]
						A__					:       List[str]     =params["""n_layers"""]
						A__					:       int     =params["""n_heads"""]
						A__					:       str     =n_heads // num_shards
						A__					:       Tuple     =params["""dim"""]
						A__					:       Union[str, Any]     =dim // n_heads
						A__					:       str     =1_00_00.0
						A__					:       Any     =1.0 / (base ** (torch.arange(0,		__snake_case,		2	).float() / dims_per_head))

						if "n_kv_heads" in params:
												A__					:       Optional[Any]     =params["""n_kv_heads"""]  # for GQA / MQA
												A__					:       int     =n_heads_per_shard // num_key_value_heads
												A__					:       int     =dim // num_key_value_heads
						else:  # compatibility with other checkpoints
												A__					:       List[Any]     =n_heads
												A__					:       List[str]     =n_heads_per_shard
												A__					:       Dict     =dim

						# permute for sliced rotary
						def permute(__snake_case					:		Tuple,		__snake_case					:		Optional[int]=n_heads,		__snake_case					:		int=dim,		__snake_case					:		Optional[Any]=dim	):
												return w.view(__snake_case,		dima // n_heads // 2,		2,		__snake_case	).transpose(1,		2	).reshape(__snake_case,		__snake_case	)

						print(f"Fetching all parameters from the checkpoint at {input_base_path}."	)
						# Load weights
						if model_size == "7B":
												# Not sharded
												# (The sharded implementation would also work, but this is simpler.)
												A__					:       List[str]     =torch.load(os.path.join(__snake_case,		"""consolidated.00.pth"""	),		map_location="""cpu"""	)
						else:
												# Sharded
												A__					:       Optional[Any]     =[
												    torch.load(os.path.join(__snake_case,		f"consolidated.{i:02d}.pth"	),		map_location="""cpu"""	)
												    for i in range(__snake_case	)
												]
						A__					:       Optional[Any]     =0
						A__					:       str     ={"""weight_map""": {}}
						for layer_i in range(__snake_case	):
												A__					:       Dict     =f"pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin"
												if model_size == "7B":
																		# Unsharded
																		A__					:       Dict     ={
																		    f"model.layers.{layer_i}.self_attn.q_proj.weight": permute(
																		        loaded[f"layers.{layer_i}.attention.wq.weight"]	),
																		    f"model.layers.{layer_i}.self_attn.k_proj.weight": permute(
																		        loaded[f"layers.{layer_i}.attention.wk.weight"]	),
																		    f"model.layers.{layer_i}.self_attn.v_proj.weight": loaded[f"layers.{layer_i}.attention.wv.weight"],
																		    f"model.layers.{layer_i}.self_attn.o_proj.weight": loaded[f"layers.{layer_i}.attention.wo.weight"],
																		    f"model.layers.{layer_i}.mlp.gate_proj.weight": loaded[f"layers.{layer_i}.feed_forward.w1.weight"],
																		    f"model.layers.{layer_i}.mlp.down_proj.weight": loaded[f"layers.{layer_i}.feed_forward.w2.weight"],
																		    f"model.layers.{layer_i}.mlp.up_proj.weight": loaded[f"layers.{layer_i}.feed_forward.w3.weight"],
																		    f"model.layers.{layer_i}.input_layernorm.weight": loaded[f"layers.{layer_i}.attention_norm.weight"],
																		    f"model.layers.{layer_i}.post_attention_layernorm.weight": loaded[f"layers.{layer_i}.ffn_norm.weight"],
																		}
												else:
																		# Sharded
																		# Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
																		# the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
																		# redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

																		A__					:       Any     ={
																		    f"model.layers.{layer_i}.input_layernorm.weight": loaded[0][
																		        f"layers.{layer_i}.attention_norm.weight"
																		    ].clone(),
																		    f"model.layers.{layer_i}.post_attention_layernorm.weight": loaded[0][
																		        f"layers.{layer_i}.ffn_norm.weight"
																		    ].clone(),
																		}
																		A__					:       Optional[Any]     =permute(
																		    torch.cat(
																		        [
																		            loaded[i][f"layers.{layer_i}.attention.wq.weight"].view(__snake_case,		__snake_case,		__snake_case	)
																		            for i in range(__snake_case	)
																		        ],		dim=0,		).reshape(__snake_case,		__snake_case	)	)
																		A__					:       int     =permute(
																		    torch.cat(
																		        [
																		            loaded[i][f"layers.{layer_i}.attention.wk.weight"].view(
																		                __snake_case,		__snake_case,		__snake_case	)
																		            for i in range(__snake_case	)
																		        ],		dim=0,		).reshape(__snake_case,		__snake_case	),		__snake_case,		__snake_case,		__snake_case,		)
																		A__					:       int     =torch.cat(
																		    [
																		        loaded[i][f"layers.{layer_i}.attention.wv.weight"].view(
																		            __snake_case,		__snake_case,		__snake_case	)
																		        for i in range(__snake_case	)
																		    ],		dim=0,		).reshape(__snake_case,		__snake_case	)

																		A__					:       List[str]     =torch.cat(
																		    [loaded[i][f"layers.{layer_i}.attention.wo.weight"] for i in range(__snake_case	)],		dim=1	)
																		A__					:       Optional[int]     =torch.cat(
																		    [loaded[i][f"layers.{layer_i}.feed_forward.w1.weight"] for i in range(__snake_case	)],		dim=0	)
																		A__					:       str     =torch.cat(
																		    [loaded[i][f"layers.{layer_i}.feed_forward.w2.weight"] for i in range(__snake_case	)],		dim=1	)
																		A__					:       List[str]     =torch.cat(
																		    [loaded[i][f"layers.{layer_i}.feed_forward.w3.weight"] for i in range(__snake_case	)],		dim=0	)

												A__					:       List[Any]     =inv_freq
												for k, v in state_dict.items():
																		A__					:       Optional[Any]     =filename
																		param_count += v.numel()
												torch.save(__snake_case,		os.path.join(__snake_case,		__snake_case	)	)

						A__					:       Tuple     =f"pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin"
						if model_size == "7B":
												# Unsharded
												A__					:       Tuple     ={
												    """model.embed_tokens.weight""": loaded["""tok_embeddings.weight"""],
												    """model.norm.weight""": loaded["""norm.weight"""],
												    """lm_head.weight""": loaded["""output.weight"""],
												}
						else:
												A__					:       Any     ={
												    """model.norm.weight""": loaded[0]["""norm.weight"""],
												    """model.embed_tokens.weight""": torch.cat(
												        [loaded[i]["""tok_embeddings.weight"""] for i in range(__snake_case	)],		dim=1	),
												    """lm_head.weight""": torch.cat([loaded[i]["""output.weight"""] for i in range(__snake_case	)],		dim=0	),
												}

						for k, v in state_dict.items():
												A__					:       int     =filename
												param_count += v.numel()
						torch.save(__snake_case,		os.path.join(__snake_case,		__snake_case	)	)

						# Write configs
						A__					:       Union[str, Any]     ={"""total_size""": param_count * 2}
						write_json(__snake_case,		os.path.join(__snake_case,		"""pytorch_model.bin.index.json"""	)	)
						A__					:       Optional[Any]     =params["""ffn_dim_multiplier"""] if """ffn_dim_multiplier""" in params else 1
						A__					:       List[Any]     =params["""multiple_of"""] if """multiple_of""" in params else 256
						A__					:       int     =LlamaConfig(
						    hidden_size=__snake_case,		intermediate_size=compute_intermediate_size(__snake_case,		__snake_case,		__snake_case	),		num_attention_heads=params["""n_heads"""],		num_hidden_layers=params["""n_layers"""],		rms_norm_eps=params["""norm_eps"""],		num_key_value_heads=__snake_case,		)
						config.save_pretrained(__snake_case	)

						# Make space so we can load the model properly now.
						del state_dict
						del loaded
						gc.collect()

						print("""Loading the checkpoint in a Llama model."""	)
						A__					:       List[Any]     =LlamaForCausalLM.from_pretrained(__snake_case,		torch_dtype=torch.floataa,		low_cpu_mem_usage=__snake_case	)
						# Avoid saving this as part of the config.
						del model.config._name_or_path

						print("""Saving in the Transformers format."""	)
						model.save_pretrained(__snake_case,		safe_serialization=__snake_case	)
						shutil.rmtree(__snake_case	)
def 						__lowerCamelCase	(       __snake_case					:		Union[str, Any],		__snake_case					:		Dict	)	->							Tuple:






						"""simple docstring"""





						A__					:       List[Any]     =LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
						print(f"Saving a {tokenizer_class.__name__} to {tokenizer_path}."	)
						A__					:       List[str]     =tokenizer_class(__snake_case	)
						tokenizer.save_pretrained(__snake_case	)
def 						__lowerCamelCase	(       )	->							Any:






						"""simple docstring"""





						A__					:       List[str]     =argparse.ArgumentParser()
						parser.add_argument(
						    """--input_dir""",		help="""Location of LLaMA weights, which contains tokenizer.model and model folders""",		)
						parser.add_argument(
						    """--model_size""",		choices=["""7B""", """7Bf""", """13B""", """13Bf""", """30B""", """65B""", """70B""", """70Bf""", """tokenizer_only"""],		)
						parser.add_argument(
						    """--output_dir""",		help="""Location to write HF model and tokenizer""",		)
						parser.add_argument("""--safe_serialization""",		type=__snake_case,		help="""Whether or not to save using `safetensors`."""	)
						A__					:       Any     =parser.parse_args()
						if args.model_size != "tokenizer_only":
												write_model(
												    model_path=args.output_dir,		input_base_path=os.path.join(args.input_dir,		args.model_size	),		model_size=args.model_size,		safe_serialization=args.safe_serialization,		)
						A__					:       List[Any]     =os.path.join(args.input_dir,		"""tokenizer.model"""	)
						write_tokenizer(args.output_dir,		__snake_case	)


if __name__ == "__main__":
		main()

