






"""simple docstring"""







import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
			from transformers import LlamaTokenizerFast
except ImportError as e:
			warnings.warn(e)
			warnings.warn(
			    """The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion"""
			)
			a			:    List[Any]				 =		None
a			:    Optional[Any]				 =		{
    """7B""": 11008,
    """13B""": 13824,
    """30B""": 17920,
    """65B""": 22016,
    """70B""": 28672,
}
a			:    Tuple				 =		{
    """7B""": 1,
    """7Bf""": 1,
    """13B""": 2,
    """13Bf""": 2,
    """30B""": 4,
    """65B""": 8,
    """70B""": 8,
    """70Bf""": 8,
}





def 						lowercase__(A ,		A=1 ,		A=256						)    ->List[str]:




						"""simple docstring"""



						return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3						)						) + multiple_of - 1) // multiple_of)





def 						lowercase__(A						)    ->List[str]:




						"""simple docstring"""



						with open(A ,		"r"						) as f:
												return json.load(A						)





def 						lowercase__(A ,		A						)    ->Optional[int]:




						"""simple docstring"""



						with open(A ,		"w"						) as f:
												json.dump(A ,		A						)





def 						lowercase__(A ,		A ,		A ,		A=True						)    ->Union[str, Any]:




						"""simple docstring"""



						os.makedirs(A ,		exist_ok=A						)
						lowercase__    :				List[Any]=	os.path.join(A ,		"tmp"						)
						os.makedirs(A ,		exist_ok=A						)

						lowercase__    :				int=	read_json(os.path.join(A ,		"params.json"						)						)
						lowercase__    :				Optional[int]=	NUM_SHARDS[model_size]
						lowercase__    :				Tuple=	params["n_layers"]
						lowercase__    :				str=	params["n_heads"]
						lowercase__    :				Dict=	n_heads // num_shards
						lowercase__    :				List[str]=	params["dim"]
						lowercase__    :				List[Any]=	dim // n_heads
						lowercase__    :				List[Any]=	10_000.0
						lowercase__    :				List[str]=	1.0 / (base ** (torch.arange(0 ,		A ,		2						).float() / dims_per_head))

						if "n_kv_heads" in params:
												lowercase__    :				Any=	params["n_kv_heads"]  # for GQA / MQA
												lowercase__    :				Dict=	n_heads_per_shard // num_key_value_heads
												lowercase__    :				int=	dim // num_key_value_heads
						else:  # compatibility with other checkpoints
												lowercase__    :				Optional[Any]=	n_heads
												lowercase__    :				Optional[Any]=	n_heads_per_shard
												lowercase__    :				Union[str, Any]=	dim

						# permute for sliced rotary
						def permute(A ,		A=n_heads ,		A=dim ,		A=dim						):
												return w.view(A ,		dima // n_heads // 2 ,		2 ,		A						).transpose(1 ,		2						).reshape(A ,		A						)

						print(f'''Fetching all parameters from the checkpoint at {input_base_path}.'''						)
						# Load weights
						if model_size == "7B":
												# Not sharded
												# (The sharded implementation would also work, but this is simpler.)
												lowercase__    :				List[str]=	torch.load(os.path.join(A ,		"consolidated.00.pth"						) ,		map_location="cpu"						)
						else:
												# Sharded
												lowercase__    :				Optional[Any]=	[
												    torch.load(os.path.join(A ,		f'''consolidated.{i:02d}.pth'''						) ,		map_location="cpu"						)
												    for i in range(A						)
												]
						lowercase__    :				Tuple=	0
						lowercase__    :				Tuple=	{"weight_map": {}}
						for layer_i in range(A						):
												lowercase__    :				List[Any]=	f'''pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin'''
												if model_size == "7B":
																		# Unsharded
																		lowercase__    :				Union[str, Any]=	{
																		    f'''model.layers.{layer_i}.self_attn.q_proj.weight''': permute(
																		        loaded[f'''layers.{layer_i}.attention.wq.weight''']						),
																		    f'''model.layers.{layer_i}.self_attn.k_proj.weight''': permute(
																		        loaded[f'''layers.{layer_i}.attention.wk.weight''']						),
																		    f'''model.layers.{layer_i}.self_attn.v_proj.weight''': loaded[f'''layers.{layer_i}.attention.wv.weight'''],
																		    f'''model.layers.{layer_i}.self_attn.o_proj.weight''': loaded[f'''layers.{layer_i}.attention.wo.weight'''],
																		    f'''model.layers.{layer_i}.mlp.gate_proj.weight''': loaded[f'''layers.{layer_i}.feed_forward.w1.weight'''],
																		    f'''model.layers.{layer_i}.mlp.down_proj.weight''': loaded[f'''layers.{layer_i}.feed_forward.w2.weight'''],
																		    f'''model.layers.{layer_i}.mlp.up_proj.weight''': loaded[f'''layers.{layer_i}.feed_forward.w3.weight'''],
																		    f'''model.layers.{layer_i}.input_layernorm.weight''': loaded[f'''layers.{layer_i}.attention_norm.weight'''],
																		    f'''model.layers.{layer_i}.post_attention_layernorm.weight''': loaded[f'''layers.{layer_i}.ffn_norm.weight'''],
																		}
												else:
																		# Sharded
																		# Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
																		# the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
																		# redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

																		lowercase__    :				int=	{
																		    f'''model.layers.{layer_i}.input_layernorm.weight''': loaded[0][
																		        f'''layers.{layer_i}.attention_norm.weight'''
																		    ].clone(),
																		    f'''model.layers.{layer_i}.post_attention_layernorm.weight''': loaded[0][
																		        f'''layers.{layer_i}.ffn_norm.weight'''
																		    ].clone(),
																		}
																		lowercase__    :				Optional[int]=	permute(
																		    torch.cat(
																		        [
																		            loaded[i][f'''layers.{layer_i}.attention.wq.weight'''].view(A ,		A ,		A						)
																		            for i in range(A						)
																		        ] ,		dim=0 ,		).reshape(A ,		A						)						)
																		lowercase__    :				Optional[int]=	permute(
																		    torch.cat(
																		        [
																		            loaded[i][f'''layers.{layer_i}.attention.wk.weight'''].view(
																		                A ,		A ,		A						)
																		            for i in range(A						)
																		        ] ,		dim=0 ,		).reshape(A ,		A						) ,		A ,		A ,		A ,		)
																		lowercase__    :				Any=	torch.cat(
																		    [
																		        loaded[i][f'''layers.{layer_i}.attention.wv.weight'''].view(
																		            A ,		A ,		A						)
																		        for i in range(A						)
																		    ] ,		dim=0 ,		).reshape(A ,		A						)

																		lowercase__    :				Optional[Any]=	torch.cat(
																		    [loaded[i][f'''layers.{layer_i}.attention.wo.weight'''] for i in range(A						)] ,		dim=1						)
																		lowercase__    :				Any=	torch.cat(
																		    [loaded[i][f'''layers.{layer_i}.feed_forward.w1.weight'''] for i in range(A						)] ,		dim=0						)
																		lowercase__    :				Union[str, Any]=	torch.cat(
																		    [loaded[i][f'''layers.{layer_i}.feed_forward.w2.weight'''] for i in range(A						)] ,		dim=1						)
																		lowercase__    :				Union[str, Any]=	torch.cat(
																		    [loaded[i][f'''layers.{layer_i}.feed_forward.w3.weight'''] for i in range(A						)] ,		dim=0						)

												lowercase__    :				Optional[int]=	inv_freq
												for k, v in state_dict.items():
																		lowercase__    :				Optional[int]=	filename
																		param_count += v.numel()
												torch.save(A ,		os.path.join(A ,		A						)						)

						lowercase__    :				List[Any]=	f'''pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin'''
						if model_size == "7B":
												# Unsharded
												lowercase__    :				Optional[int]=	{
												    "model.embed_tokens.weight": loaded["tok_embeddings.weight"],
												    "model.norm.weight": loaded["norm.weight"],
												    "lm_head.weight": loaded["output.weight"],
												}
						else:
												lowercase__    :				str=	{
												    "model.norm.weight": loaded[0]["norm.weight"],
												    "model.embed_tokens.weight": torch.cat(
												        [loaded[i]["tok_embeddings.weight"] for i in range(A						)] ,		dim=1						),
												    "lm_head.weight": torch.cat([loaded[i]["output.weight"] for i in range(A						)] ,		dim=0						),
												}

						for k, v in state_dict.items():
												lowercase__    :				Dict=	filename
												param_count += v.numel()
						torch.save(A ,		os.path.join(A ,		A						)						)

						# Write configs
						lowercase__    :				List[str]=	{"total_size": param_count * 2}
						write_json(A ,		os.path.join(A ,		"pytorch_model.bin.index.json"						)						)
						lowercase__    :				int=	params["ffn_dim_multiplier"] if "ffn_dim_multiplier" in params else 1
						lowercase__    :				Any=	params["multiple_of"] if "multiple_of" in params else 256
						lowercase__    :				Tuple=	LlamaConfig(
						    hidden_size=A ,		intermediate_size=compute_intermediate_size(A ,		A ,		A						) ,		num_attention_heads=params["n_heads"] ,		num_hidden_layers=params["n_layers"] ,		rms_norm_eps=params["norm_eps"] ,		num_key_value_heads=A ,		)
						config.save_pretrained(A						)

						# Make space so we can load the model properly now.
						del state_dict
						del loaded
						gc.collect()

						print("Loading the checkpoint in a Llama model."						)
						lowercase__    :				Tuple=	LlamaForCausalLM.from_pretrained(A ,		torch_dtype=torch.floataa ,		low_cpu_mem_usage=A						)
						# Avoid saving this as part of the config.
						del model.config._name_or_path

						print("Saving in the Transformers format."						)
						model.save_pretrained(A ,		safe_serialization=A						)
						shutil.rmtree(A						)





def 						lowercase__(A ,		A						)    ->Union[str, Any]:




						"""simple docstring"""



						lowercase__    :				int=	LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
						print(f'''Saving a {tokenizer_class.__name__} to {tokenizer_path}.'''						)
						lowercase__    :				str=	tokenizer_class(A						)
						tokenizer.save_pretrained(A						)





def 						lowercase__()    ->Optional[int]:




						"""simple docstring"""



						lowercase__    :				List[Any]=	argparse.ArgumentParser()
						parser.add_argument(
						    "--input_dir" ,		help="Location of LLaMA weights, which contains tokenizer.model and model folders" ,		)
						parser.add_argument(
						    "--model_size" ,		choices=["7B", "7Bf", "13B", "13Bf", "30B", "65B", "70B", "70Bf", "tokenizer_only"] ,		)
						parser.add_argument(
						    "--output_dir" ,		help="Location to write HF model and tokenizer" ,		)
						parser.add_argument("--safe_serialization" ,		type=A ,		help="Whether or not to save using `safetensors`."						)
						lowercase__    :				str=	parser.parse_args()
						if args.model_size != "tokenizer_only":
												write_model(
												    model_path=args.output_dir ,		input_base_path=os.path.join(args.input_dir ,		args.model_size						) ,		model_size=args.model_size ,		safe_serialization=args.safe_serialization ,		)
						lowercase__    :				List[str]=	os.path.join(args.input_dir ,		"tokenizer.model"						)
						write_tokenizer(args.output_dir ,		A						)


if __name__ == "__main__":
			main()














"""simple docstring"""







from __future__ import annotations





def 						lowercase__(A						)    ->int:




						"""simple docstring"""



						for i in range(1 ,		len(matrix[0]						)						):
												matrix[0][i] += matrix[0][i - 1]

						# preprocessing the first column
						for i in range(1 ,		len(A						)						):
												matrix[i][0] += matrix[i - 1][0]

						# updating the path cost for current position
						for i in range(1 ,		len(A						)						):
												for j in range(1 ,		len(matrix[0]						)						):
																		matrix[i][j] += min(matrix[i - 1][j] ,		matrix[i][j - 1]						)

						return matrix[-1][-1]


if __name__ == "__main__":
			import doctest

			doctest.testmod()






