






# Copyright (c) 2021-, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

####################################################################################################

#
# Note: If when running this conversion script you're getting an exception:
#     ModuleNotFoundError: No module named 'megatron.model.enums'
# you need to tell python where to find the clone of Megatron-LM, e.g.:
#
# cd /tmp
# git clone https://github.com/NVIDIA/Megatron-LM
# PYTHONPATH=/tmp/Megatron-LM python src/transformers/models/megatron_gpt2/convert_megatron_gpt2_checkpoint.py ...
#
# if you already have it cloned elsewhere, simply adjust the path to the existing path
#
# If the training was done using a Megatron-LM fork, e.g.,
# https://github.com/microsoft/Megatron-DeepSpeed/ then chances are that you need to have that one
# in your path, i.e., /path/to/Megatron-DeepSpeed/
#

import argparse
import os
import re
import zipfile

import torch

from transformers import AutoTokenizer, GPTaConfig

def 	__lowerCamelCase      (		snake_case__	,snake_case__	,snake_case__=0		)							->  Optional[int]:






						"""simple docstring"""





						if name is None:
												_SCREAMING_SNAKE_CASE     				=  None
						else:
												_SCREAMING_SNAKE_CASE     				=  """.""" * max(0	,spaces - 2		) + """# {:""" + str(50 - spaces		) + """s}"""
												_SCREAMING_SNAKE_CASE     				=  fmt.format(snake_case__		)

						# Print and recurse (if needed).
						if isinstance(snake_case__	,snake_case__		):
												if msg is not None:
																		print(snake_case__		)
												for k in val.keys():
																		recursive_print(snake_case__	,val[k]	,spaces + 2		)
						elif isinstance(snake_case__	,torch.Tensor		):
												print(snake_case__	,""":"""	,val.size()		)
						else:
												print(snake_case__	,""":"""	,snake_case__		)

def 	__lowerCamelCase      (		snake_case__	,snake_case__	,snake_case__	,snake_case__	,snake_case__		)							->  Union[str, Any]:






						"""simple docstring"""





						_SCREAMING_SNAKE_CASE     				=  param.size()
						if checkpoint_version == 1.0:
												# version 1.0 stores [num_heads * hidden_size * num_splits, :]
												_SCREAMING_SNAKE_CASE     				=  (num_heads, hidden_size, num_splits) + input_shape[1:]
												_SCREAMING_SNAKE_CASE     				=  param.view(*snake_case__		)
												_SCREAMING_SNAKE_CASE     				=  param.transpose(0	,2		)
												_SCREAMING_SNAKE_CASE     				=  param.transpose(1	,2		).contiguous()
						elif checkpoint_version >= 2.0:
												# other versions store [num_heads * num_splits * hidden_size, :]
												_SCREAMING_SNAKE_CASE     				=  (num_heads, num_splits, hidden_size) + input_shape[1:]
												_SCREAMING_SNAKE_CASE     				=  param.view(*snake_case__		)
												_SCREAMING_SNAKE_CASE     				=  param.transpose(0	,1		).contiguous()
						_SCREAMING_SNAKE_CASE     				=  param.view(*snake_case__		)
						return param

def 	__lowerCamelCase      (		snake_case__	,snake_case__	,snake_case__		)							->  Optional[int]:






						"""simple docstring"""





						_SCREAMING_SNAKE_CASE     				=  {}

						# old versions did not store training args
						_SCREAMING_SNAKE_CASE     				=  input_state_dict.get("""args"""	,snake_case__		)
						if ds_args is not None:
												# do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
												# from pprint import pprint
												# pprint(vars(ds_args))

												_SCREAMING_SNAKE_CASE     				=  ds_args.padded_vocab_size
												_SCREAMING_SNAKE_CASE     				=  ds_args.max_position_embeddings
												_SCREAMING_SNAKE_CASE     				=  ds_args.hidden_size
												_SCREAMING_SNAKE_CASE     				=  ds_args.num_layers
												_SCREAMING_SNAKE_CASE     				=  ds_args.num_attention_heads
												_SCREAMING_SNAKE_CASE     				=  ds_args.ffn_hidden_size
												# pprint(config)

						# The number of heads.
						_SCREAMING_SNAKE_CASE     				=  config.n_head
						# The hidden_size per head.
						_SCREAMING_SNAKE_CASE     				=  config.n_embd // config.n_head
						# Megatron-LM checkpoint version
						if "checkpoint_version" in input_state_dict.keys():
												_SCREAMING_SNAKE_CASE     				=  input_state_dict["""checkpoint_version"""]
						else:
												_SCREAMING_SNAKE_CASE     				=  0.0

						# The model.
						_SCREAMING_SNAKE_CASE     				=  input_state_dict["""model"""]
						# The language model.
						_SCREAMING_SNAKE_CASE     				=  model["""language_model"""]
						# The embeddings.
						_SCREAMING_SNAKE_CASE     				=  lm["""embedding"""]

						# The word embeddings.
						_SCREAMING_SNAKE_CASE     				=  embeddings["""word_embeddings"""]["""weight"""]
						# Truncate the embedding table to vocab_size rows.
						_SCREAMING_SNAKE_CASE     				=  word_embeddings[: config.vocab_size, :]
						_SCREAMING_SNAKE_CASE     				=  word_embeddings

						# The position embeddings.
						_SCREAMING_SNAKE_CASE     				=  embeddings["""position_embeddings"""]["""weight"""]
						# Read the causal mask dimension (seqlen). [max_sequence_length, hidden_size]
						_SCREAMING_SNAKE_CASE     				=  pos_embeddings.size(0		)
						if n_positions != config.n_positions:
												raise ValueError(
												    F'pos_embeddings.max_sequence_length={n_positions} and config.n_positions={config.n_positions} don\'t match'		)
						# Store the position embeddings.
						_SCREAMING_SNAKE_CASE     				=  pos_embeddings

						# The transformer.
						_SCREAMING_SNAKE_CASE     				=  lm["""transformer"""] if """transformer""" in lm.keys() else lm["""encoder"""]

						# The regex to extract layer names.
						_SCREAMING_SNAKE_CASE     				=  re.compile(r"""layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)"""		)

						# The simple map of names for "automated" rules.
						_SCREAMING_SNAKE_CASE     				=  {
						    """attention.dense""": """.attn.c_proj.""",
						    """self_attention.dense""": """.attn.c_proj.""",
						    """mlp.dense_h_to_4h""": """.mlp.c_fc.""",
						    """mlp.dense_4h_to_h""": """.mlp.c_proj.""",
						}

						# Extract the layers.
						for key, val in transformer.items():
												# Match the name.
												_SCREAMING_SNAKE_CASE     				=  layer_re.match(snake_case__		)

												# Stop if that's not a layer
												if m is None:
																		break

												# The index of the layer.
												_SCREAMING_SNAKE_CASE     				=  int(m.group(1		)		)
												# The name of the operation.
												_SCREAMING_SNAKE_CASE     				=  m.group(2		)
												# Is it a weight or a bias?
												_SCREAMING_SNAKE_CASE     				=  m.group(3		)

												# The name of the layer.
												_SCREAMING_SNAKE_CASE     				=  F'transformer.h.{layer_idx}'

												# For layernorm(s), simply store the layer norm.
												if op_name.endswith("""layernorm"""		):
																		_SCREAMING_SNAKE_CASE     				=  """ln_1""" if op_name.startswith("""input"""		) else """ln_2"""
																		_SCREAMING_SNAKE_CASE     				=  val

												# Transpose the QKV matrix.
												elif (
												    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
												) and weight_or_bias == "weight":
																		# Insert a tensor of 1x1xDxD bias.
																		_SCREAMING_SNAKE_CASE     				=  torch.tril(torch.ones((n_positions, n_positions)	,dtype=torch.floataa		)		).view(
																		    1	,1	,snake_case__	,snake_case__		)
																		_SCREAMING_SNAKE_CASE     				=  causal_mask

																		# Insert a "dummy" tensor for masked_bias.
																		_SCREAMING_SNAKE_CASE     				=  torch.tensor(-1e4	,dtype=torch.floataa		)
																		_SCREAMING_SNAKE_CASE     				=  masked_bias

																		_SCREAMING_SNAKE_CASE     				=  fix_query_key_value_ordering(snake_case__	,snake_case__	,3	,snake_case__	,snake_case__		)
																		# Megatron stores (3*D) x D but transformers-GPT2 expects D x 3*D.
																		_SCREAMING_SNAKE_CASE     				=  out_val.transpose(0	,1		).contiguous()
																		# Store.
																		_SCREAMING_SNAKE_CASE     				=  out_val

												# Transpose the bias.
												elif (
												    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
												) and weight_or_bias == "bias":
																		_SCREAMING_SNAKE_CASE     				=  fix_query_key_value_ordering(snake_case__	,snake_case__	,3	,snake_case__	,snake_case__		)
																		# Store. No change of shape.
																		_SCREAMING_SNAKE_CASE     				=  out_val

												# Transpose the weights.
												elif weight_or_bias == "weight":
																		_SCREAMING_SNAKE_CASE     				=  megatron_to_transformers[op_name]
																		_SCREAMING_SNAKE_CASE     				=  val.transpose(0	,1		)

												# Copy the bias.
												elif weight_or_bias == "bias":
																		_SCREAMING_SNAKE_CASE     				=  megatron_to_transformers[op_name]
																		_SCREAMING_SNAKE_CASE     				=  val

    # DEBUG.
						assert config.n_layer == layer_idx + 1

						# The final layernorm.
						_SCREAMING_SNAKE_CASE     				=  transformer["""final_layernorm.weight"""]
						_SCREAMING_SNAKE_CASE     				=  transformer["""final_layernorm.bias"""]

						# For LM head, transformers' wants the matrix to weight embeddings.
						_SCREAMING_SNAKE_CASE     				=  word_embeddings

						# It should be done!
						return output_state_dict

def 	__lowerCamelCase      (		)							->  str:






						"""simple docstring"""





						_SCREAMING_SNAKE_CASE     				=  argparse.ArgumentParser()
						parser.add_argument("""--print-checkpoint-structure"""	,action="""store_true"""		)
						parser.add_argument(
						    """path_to_checkpoint"""	,type=snake_case__	,help="""Path to the checkpoint file (.zip archive or direct .pt file)"""	,)
						parser.add_argument(
						    """--config_file"""	,default=""""""	,type=snake_case__	,help="""An optional config json file describing the pre-trained model."""	,)
						_SCREAMING_SNAKE_CASE     				=  parser.parse_args()

						# Extract the basename.
						_SCREAMING_SNAKE_CASE     				=  os.path.dirname(args.path_to_checkpoint		)

						# Load the model.
						# the .zip is very optional, let's keep it for backward compatibility
						print(F'Extracting PyTorch state dictionary from {args.path_to_checkpoint}'		)
						if args.path_to_checkpoint.endswith(""".zip"""		):
												with zipfile.ZipFile(args.path_to_checkpoint	,"""r"""		) as checkpoint:
																		with checkpoint.open("""release/mp_rank_00/model_optim_rng.pt"""		) as pytorch_dict:
																								_SCREAMING_SNAKE_CASE     				=  torch.load(snake_case__	,map_location="""cpu"""		)
						else:
												_SCREAMING_SNAKE_CASE     				=  torch.load(args.path_to_checkpoint	,map_location="""cpu"""		)

						_SCREAMING_SNAKE_CASE     				=  input_state_dict.get("""args"""	,snake_case__		)

						# Read the config, or default to the model released by NVIDIA.
						if args.config_file == "":
												if ds_args is not None:
																		if ds_args.bias_gelu_fusion:
																								_SCREAMING_SNAKE_CASE     				=  """gelu_fast"""
																		elif ds_args.openai_gelu:
																								_SCREAMING_SNAKE_CASE     				=  """gelu_new"""
																		else:
																								_SCREAMING_SNAKE_CASE     				=  """gelu"""
												else:
																		# in the very early days this used to be "gelu_new"
																		_SCREAMING_SNAKE_CASE     				=  """gelu_new"""

												# Spell out all parameters in case the defaults change.
												_SCREAMING_SNAKE_CASE     				=  GPTaConfig(
												    vocab_size=5_02_57	,n_positions=10_24	,n_embd=10_24	,n_layer=24	,n_head=16	,n_inner=40_96	,activation_function=snake_case__	,resid_pdrop=0.1	,embd_pdrop=0.1	,attn_pdrop=0.1	,layer_norm_epsilon=1e-5	,initializer_range=0.02	,summary_type="""cls_index"""	,summary_use_proj=snake_case__	,summary_activation=snake_case__	,summary_proj_to_labels=snake_case__	,summary_first_dropout=0.1	,scale_attn_weights=snake_case__	,use_cache=snake_case__	,bos_token_id=5_02_56	,eos_token_id=5_02_56	,)
						else:
												_SCREAMING_SNAKE_CASE     				=  GPTaConfig.from_json_file(args.config_file		)

						_SCREAMING_SNAKE_CASE     				=  ["""GPT2LMHeadModel"""]

						# Convert.
						print("""Converting"""		)
						_SCREAMING_SNAKE_CASE     				=  convert_megatron_checkpoint(snake_case__	,snake_case__	,snake_case__		)

						# Print the structure of converted state dict.
						if args.print_checkpoint_structure:
												recursive_print(snake_case__	,snake_case__		)

						# Add tokenizer class info to config
						# see https://github.com/huggingface/transformers/issues/13906)
						if ds_args is not None:
												_SCREAMING_SNAKE_CASE     				=  ds_args.tokenizer_type
												if tokenizer_type == "GPT2BPETokenizer":
																		_SCREAMING_SNAKE_CASE     				=  """gpt2"""
												elif tokenizer_type == "PretrainedFromHF":
																		_SCREAMING_SNAKE_CASE     				=  ds_args.tokenizer_name_or_path
												else:
																		raise ValueError(F'Unrecognized tokenizer_type {tokenizer_type}'		)
						else:
												_SCREAMING_SNAKE_CASE     				=  """gpt2"""

						_SCREAMING_SNAKE_CASE     				=  AutoTokenizer.from_pretrained(snake_case__		)
						_SCREAMING_SNAKE_CASE     				=  type(snake_case__		).__name__
						_SCREAMING_SNAKE_CASE     				=  tokenizer_class

						# Store the config to file.
						print("""Saving config"""		)
						config.save_pretrained(snake_case__		)

						# Save tokenizer based on args
						print(F'Adding {tokenizer_class} tokenizer files'		)
						tokenizer.save_pretrained(snake_case__		)

						# Store the state_dict to file.
						_SCREAMING_SNAKE_CASE     				=  os.path.join(snake_case__	,"""pytorch_model.bin"""		)
						print(F'Saving checkpoint to "{output_checkpoint_file}"'		)
						torch.save(snake_case__	,snake_case__		)


####################################################################################################

if __name__ == "__main__":
		main()

####################################################################################################









from maths.is_square_free import is_square_free
from maths.prime_factors import prime_factors

def 	__lowerCamelCase      (		snake_case__		)							->  int:






						"""simple docstring"""





						_SCREAMING_SNAKE_CASE     				=  prime_factors(snake_case__		)
						if is_square_free(snake_case__		):
												return -1 if len(snake_case__		) % 2 else 1
						return 0


if __name__ == "__main__":
		import doctest

		doctest.testmod()

