



import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
			from transformers import LlamaTokenizerFast
except ImportError as e:
			warnings.warn(e)
			warnings.warn(
			    '''The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion'''
			)
			lowerCAmelCase__		:					Dict							=None
lowerCAmelCase__		:					Optional[Any]							={
    '''7B''': 11008,
    '''13B''': 13824,
    '''30B''': 17920,
    '''65B''': 22016,
    '''70B''': 28672,
}
lowerCAmelCase__		:					List[str]							={
    '''7B''': 1,
    '''7Bf''': 1,
    '''13B''': 2,
    '''13Bf''': 2,
    '''30B''': 4,
    '''65B''': 8,
    '''70B''': 8,
    '''70Bf''': 8,
}






def      __lowercase       (			a__							,       a__=1							,       a__=2_56       )    ->			List[str]:
							return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3       )       ) + multiple_of - 1) // multiple_of)






def      __lowercase       (			a__       )    ->			int:
							with open(a__							,       'r'       ) as f:
														return json.load(a__       )






def      __lowercase       (			a__							,       a__       )    ->			List[str]:
							with open(a__							,       'w'       ) as f:
														json.dump(a__							,       a__       )






def      __lowercase       (			a__							,       a__							,       a__							,       a__=True       )    ->			Dict:
							os.makedirs(a__							,       exist_ok=a__       )
							__SCREAMING_SNAKE_CASE									= os.path.join(a__							,       'tmp'       )
							os.makedirs(a__							,       exist_ok=a__       )

							__SCREAMING_SNAKE_CASE									= read_json(os.path.join(a__							,       'params.json'       )       )
							__SCREAMING_SNAKE_CASE									= NUM_SHARDS[model_size]
							__SCREAMING_SNAKE_CASE									= params['n_layers']
							__SCREAMING_SNAKE_CASE									= params['n_heads']
							__SCREAMING_SNAKE_CASE									= n_heads // num_shards
							__SCREAMING_SNAKE_CASE									= params['dim']
							__SCREAMING_SNAKE_CASE									= dim // n_heads
							__SCREAMING_SNAKE_CASE									= 10000.0
							__SCREAMING_SNAKE_CASE									= 1.0 / (base ** (torch.arange(0							,       a__							,       2       ).float() / dims_per_head))

							if "n_kv_heads" in params:
														__SCREAMING_SNAKE_CASE									= params['n_kv_heads']  # for GQA / MQA
														__SCREAMING_SNAKE_CASE									= n_heads_per_shard // num_key_value_heads
														__SCREAMING_SNAKE_CASE									= dim // num_key_value_heads
							else:  # compatibility with other checkpoints
														__SCREAMING_SNAKE_CASE									= n_heads
														__SCREAMING_SNAKE_CASE									= n_heads_per_shard
														__SCREAMING_SNAKE_CASE									= dim

							# permute for sliced rotary
							def permute(a__							,       a__=n_heads							,       a__=dim							,       a__=dim       ):
														return w.view(a__							,       dima // n_heads // 2							,       2							,       a__       ).transpose(1							,       2       ).reshape(a__							,       a__       )

							print(f"""Fetching all parameters from the checkpoint at {input_base_path}."""       )
							# Load weights
							if model_size == "7B":
														# Not sharded
														# (The sharded implementation would also work, but this is simpler.)
														__SCREAMING_SNAKE_CASE									= torch.load(os.path.join(a__							,       'consolidated.00.pth'       )							,       map_location='cpu'       )
							else:
														# Sharded
														__SCREAMING_SNAKE_CASE									= [
														    torch.load(os.path.join(a__							,       f"""consolidated.{i:02d}.pth"""       )							,       map_location='cpu'       )
														    for i in range(a__       )
														]
							__SCREAMING_SNAKE_CASE									= 0
							__SCREAMING_SNAKE_CASE									= {'weight_map': {}}
							for layer_i in range(a__       ):
														__SCREAMING_SNAKE_CASE									= f"""pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin"""
														if model_size == "7B":
																					# Unsharded
																					__SCREAMING_SNAKE_CASE									= {
																					    f"""model.layers.{layer_i}.self_attn.q_proj.weight""": permute(
																					        loaded[f"""layers.{layer_i}.attention.wq.weight"""]       ),
																					    f"""model.layers.{layer_i}.self_attn.k_proj.weight""": permute(
																					        loaded[f"""layers.{layer_i}.attention.wk.weight"""]       ),
																					    f"""model.layers.{layer_i}.self_attn.v_proj.weight""": loaded[f"""layers.{layer_i}.attention.wv.weight"""],
																					    f"""model.layers.{layer_i}.self_attn.o_proj.weight""": loaded[f"""layers.{layer_i}.attention.wo.weight"""],
																					    f"""model.layers.{layer_i}.mlp.gate_proj.weight""": loaded[f"""layers.{layer_i}.feed_forward.w1.weight"""],
																					    f"""model.layers.{layer_i}.mlp.down_proj.weight""": loaded[f"""layers.{layer_i}.feed_forward.w2.weight"""],
																					    f"""model.layers.{layer_i}.mlp.up_proj.weight""": loaded[f"""layers.{layer_i}.feed_forward.w3.weight"""],
																					    f"""model.layers.{layer_i}.input_layernorm.weight""": loaded[f"""layers.{layer_i}.attention_norm.weight"""],
																					    f"""model.layers.{layer_i}.post_attention_layernorm.weight""": loaded[f"""layers.{layer_i}.ffn_norm.weight"""],
																					}
														else:
																					# Sharded
																					# Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
																					# the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
																					# redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

																					__SCREAMING_SNAKE_CASE									= {
																					    f"""model.layers.{layer_i}.input_layernorm.weight""": loaded[0][
																					        f"""layers.{layer_i}.attention_norm.weight"""
																					    ].clone(),
																					    f"""model.layers.{layer_i}.post_attention_layernorm.weight""": loaded[0][
																					        f"""layers.{layer_i}.ffn_norm.weight"""
																					    ].clone(),
																					}
																					__SCREAMING_SNAKE_CASE									= permute(
																					    torch.cat(
																					        [
																					            loaded[i][f"""layers.{layer_i}.attention.wq.weight"""].view(a__							,       a__							,       a__       )
																					            for i in range(a__       )
																					        ]							,       dim=0							,       ).reshape(a__							,       a__       )       )
																					__SCREAMING_SNAKE_CASE									= permute(
																					    torch.cat(
																					        [
																					            loaded[i][f"""layers.{layer_i}.attention.wk.weight"""].view(
																					                a__							,       a__							,       a__       )
																					            for i in range(a__       )
																					        ]							,       dim=0							,       ).reshape(a__							,       a__       )							,       a__							,       a__							,       a__							,       )
																					__SCREAMING_SNAKE_CASE									= torch.cat(
																					    [
																					        loaded[i][f"""layers.{layer_i}.attention.wv.weight"""].view(
																					            a__							,       a__							,       a__       )
																					        for i in range(a__       )
																					    ]							,       dim=0							,       ).reshape(a__							,       a__       )

																					__SCREAMING_SNAKE_CASE									= torch.cat(
																					    [loaded[i][f"""layers.{layer_i}.attention.wo.weight"""] for i in range(a__       )]							,       dim=1       )
																					__SCREAMING_SNAKE_CASE									= torch.cat(
																					    [loaded[i][f"""layers.{layer_i}.feed_forward.w1.weight"""] for i in range(a__       )]							,       dim=0       )
																					__SCREAMING_SNAKE_CASE									= torch.cat(
																					    [loaded[i][f"""layers.{layer_i}.feed_forward.w2.weight"""] for i in range(a__       )]							,       dim=1       )
																					__SCREAMING_SNAKE_CASE									= torch.cat(
																					    [loaded[i][f"""layers.{layer_i}.feed_forward.w3.weight"""] for i in range(a__       )]							,       dim=0       )

														__SCREAMING_SNAKE_CASE									= inv_freq
														for k, v in state_dict.items():
																					__SCREAMING_SNAKE_CASE									= filename
																					param_count += v.numel()
														torch.save(a__							,       os.path.join(a__							,       a__       )       )

							__SCREAMING_SNAKE_CASE									= f"""pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin"""
							if model_size == "7B":
														# Unsharded
														__SCREAMING_SNAKE_CASE									= {
														    'model.embed_tokens.weight': loaded['tok_embeddings.weight'],
														    'model.norm.weight': loaded['norm.weight'],
														    'lm_head.weight': loaded['output.weight'],
														}
							else:
														__SCREAMING_SNAKE_CASE									= {
														    'model.norm.weight': loaded[0]['norm.weight'],
														    'model.embed_tokens.weight': torch.cat(
														        [loaded[i]['tok_embeddings.weight'] for i in range(a__       )]							,       dim=1       ),
														    'lm_head.weight': torch.cat([loaded[i]['output.weight'] for i in range(a__       )]							,       dim=0       ),
														}

							for k, v in state_dict.items():
														__SCREAMING_SNAKE_CASE									= filename
														param_count += v.numel()
							torch.save(a__							,       os.path.join(a__							,       a__       )       )

							# Write configs
							__SCREAMING_SNAKE_CASE									= {'total_size': param_count * 2}
							write_json(a__							,       os.path.join(a__							,       'pytorch_model.bin.index.json'       )       )
							__SCREAMING_SNAKE_CASE									= params['ffn_dim_multiplier'] if 'ffn_dim_multiplier' in params else 1
							__SCREAMING_SNAKE_CASE									= params['multiple_of'] if 'multiple_of' in params else 2_56
							__SCREAMING_SNAKE_CASE									= LlamaConfig(
							    hidden_size=a__							,       intermediate_size=compute_intermediate_size(a__							,       a__							,       a__       )							,       num_attention_heads=params['n_heads']							,       num_hidden_layers=params['n_layers']							,       rms_norm_eps=params['norm_eps']							,       num_key_value_heads=a__							,       )
							config.save_pretrained(a__       )

							# Make space so we can load the model properly now.
							del state_dict
							del loaded
							gc.collect()

							print('Loading the checkpoint in a Llama model.'       )
							__SCREAMING_SNAKE_CASE									= LlamaForCausalLM.from_pretrained(a__							,       torch_dtype=torch.floataa							,       low_cpu_mem_usage=a__       )
							# Avoid saving this as part of the config.
							del model.config._name_or_path

							print('Saving in the Transformers format.'       )
							model.save_pretrained(a__							,       safe_serialization=a__       )
							shutil.rmtree(a__       )






def      __lowercase       (			a__							,       a__       )    ->			List[Any]:
							# Initialize the tokenizer based on the `spm` model
							__SCREAMING_SNAKE_CASE									= LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
							print(f"""Saving a {tokenizer_class.__name__} to {tokenizer_path}."""       )
							__SCREAMING_SNAKE_CASE									= tokenizer_class(a__       )
							tokenizer.save_pretrained(a__       )






def      __lowercase       (			)    ->			List[str]:
							__SCREAMING_SNAKE_CASE									= argparse.ArgumentParser()
							parser.add_argument(
							    '--input_dir'							,       help='Location of LLaMA weights, which contains tokenizer.model and model folders'							,       )
							parser.add_argument(
							    '--model_size'							,       choices=['7B', '7Bf', '13B', '13Bf', '30B', '65B', '70B', '70Bf', 'tokenizer_only']							,       )
							parser.add_argument(
							    '--output_dir'							,       help='Location to write HF model and tokenizer'							,       )
							parser.add_argument('--safe_serialization'							,       type=a__							,       help='Whether or not to save using `safetensors`.'       )
							__SCREAMING_SNAKE_CASE									= parser.parse_args()
							if args.model_size != "tokenizer_only":
														write_model(
														    model_path=args.output_dir							,       input_base_path=os.path.join(args.input_dir							,       args.model_size       )							,       model_size=args.model_size							,       safe_serialization=args.safe_serialization							,       )
							__SCREAMING_SNAKE_CASE									= os.path.join(args.input_dir							,       'tokenizer.model'       )
							write_tokenizer(args.output_dir							,       a__       )


if __name__ == "__main__":
			main()











from dataclasses import dataclass, field
from typing import Optional

from transformers import AutoConfig, AutoImageProcessor, AutoTokenizer, FlaxVisionEncoderDecoderModel, HfArgumentParser

@dataclass
class        UpperCAmelCase_  :

						'''simple docstring'''




						UpperCamelCase__      :			str      =  field(
						    metadata={'''help''': '''The output directory where the model will be written.'''}   ,   )
						UpperCamelCase__      :			str      =  field(
						    metadata={
						        '''help''': (
						            '''The encoder model checkpoint for weights initialization.'''
						            '''Don\'t set if you want to train an encoder model from scratch.'''
						        )
						    }   ,   )
						UpperCamelCase__      :			str      =  field(
						    metadata={
						        '''help''': (
						            '''The decoder model checkpoint for weights initialization.'''
						            '''Don\'t set if you want to train a decoder model from scratch.'''
						        )
						    }   ,   )
						UpperCamelCase__      :			Optional[str]      =  field(
						    default=UpperCamelCase_   ,   metadata={'''help''': '''Pretrained encoder config name or path if not the same as encoder_model_name'''}	)
						UpperCamelCase__      :			Optional[str]      =  field(
						    default=UpperCamelCase_   ,   metadata={'''help''': '''Pretrained decoder config name or path if not the same as decoder_model_name'''}	)







def      __lowercase       (			)    ->			Union[str, Any]:
							__SCREAMING_SNAKE_CASE									= HfArgumentParser((ModelArguments,)       )
							((__SCREAMING_SNAKE_CASE)							,	)									= parser.parse_args_into_dataclasses()

							# Load pretrained model and tokenizer

							# Use explicit specified encoder config
							if model_args.encoder_config_name:
														__SCREAMING_SNAKE_CASE									= AutoConfig.from_pretrained(model_args.encoder_config_name       )
							# Use pretrained encoder model's config
							else:
														__SCREAMING_SNAKE_CASE									= AutoConfig.from_pretrained(model_args.encoder_model_name_or_path       )

							# Use explicit specified decoder config
							if model_args.decoder_config_name:
														__SCREAMING_SNAKE_CASE									= AutoConfig.from_pretrained(model_args.decoder_config_name       )
							# Use pretrained decoder model's config
							else:
														__SCREAMING_SNAKE_CASE									= AutoConfig.from_pretrained(model_args.decoder_model_name_or_path       )

							# necessary for `from_encoder_decoder_pretrained` when `decoder_config` is passed
							__SCREAMING_SNAKE_CASE									= True
							__SCREAMING_SNAKE_CASE									= True

							__SCREAMING_SNAKE_CASE									= FlaxVisionEncoderDecoderModel.from_encoder_decoder_pretrained(
							    encoder_pretrained_model_name_or_path=model_args.encoder_model_name_or_path							,       decoder_pretrained_model_name_or_path=model_args.decoder_model_name_or_path							,       encoder_config=a__							,       decoder_config=a__							,       )

							# GPT2 only has bos/eos tokens but not decoder_start/pad tokens
							__SCREAMING_SNAKE_CASE									= decoder_config.decoder_start_token_id
							__SCREAMING_SNAKE_CASE									= decoder_config.pad_token_id
							if decoder_start_token_id is None:
														__SCREAMING_SNAKE_CASE									= decoder_config.bos_token_id
							if pad_token_id is None:
														__SCREAMING_SNAKE_CASE									= decoder_config.eos_token_id

							# This is necessary to make Flax's generate() work
							__SCREAMING_SNAKE_CASE									= decoder_config.eos_token_id
							__SCREAMING_SNAKE_CASE									= decoder_start_token_id
							__SCREAMING_SNAKE_CASE									= pad_token_id

							__SCREAMING_SNAKE_CASE									= AutoImageProcessor.from_pretrained(model_args.encoder_model_name_or_path       )

							__SCREAMING_SNAKE_CASE									= AutoTokenizer.from_pretrained(model_args.decoder_model_name_or_path       )
							__SCREAMING_SNAKE_CASE									= tokenizer.convert_ids_to_tokens(model.config.pad_token_id       )

							model.save_pretrained(model_args.output_dir       )
							image_processor.save_pretrained(model_args.output_dir       )
							tokenizer.save_pretrained(model_args.output_dir       )


if __name__ == "__main__":
			main()






