




'''simple docstring'''





import argparse
import os
import re
import zipfile

import torch

from transformers import AutoTokenizer, GPTaConfig


def 					UpperCamelCase_(     snake_case							:	Union[str, Any]							,    snake_case							:	str							,    snake_case							:	Union[str, Any]=0					):

							'''simple docstring'''






							if name is None:
														snake_case_      =				None
							else:
														snake_case_      =				"." * max(0							,    spaces - 2					) + "# {:" + str(5_0 - spaces					) + "s}"
														snake_case_      =				fmt.format(snake_case					)

							# Print and recurse (if needed).
							if isinstance(snake_case							,    snake_case					):
														if msg is not None:
																					print(snake_case					)
														for k in val.keys():
																					recursive_print(snake_case							,    val[k]							,    spaces + 2					)
							elif isinstance(snake_case							,    torch.Tensor					):
														print(snake_case							,    ":"							,    val.size()					)
							else:
														print(snake_case							,    ":"							,    snake_case					)


def 					UpperCamelCase_(     snake_case							:	str							,    snake_case							:	Any							,    snake_case							:	List[Any]							,    snake_case							:	Dict							,    snake_case							:	int					):

							'''simple docstring'''






							snake_case_      =				param.size()
							if checkpoint_version == 1.0:
														# version 1.0 stores [num_heads * hidden_size * num_splits, :]
														snake_case_      =				(num_heads, hidden_size, num_splits) + input_shape[1:]
														snake_case_      =				param.view(*snake_case					)
														snake_case_      =				param.transpose(0							,    2					)
														snake_case_      =				param.transpose(1							,    2					).contiguous()
							elif checkpoint_version >= 2.0:
														# other versions store [num_heads * num_splits * hidden_size, :]
														snake_case_      =				(num_heads, num_splits, hidden_size) + input_shape[1:]
														snake_case_      =				param.view(*snake_case					)
														snake_case_      =				param.transpose(0							,    1					).contiguous()
							snake_case_      =				param.view(*snake_case					)
							return param


def 					UpperCamelCase_(     snake_case							:	str							,    snake_case							:	str							,    snake_case							:	Tuple					):

							'''simple docstring'''






							snake_case_      =				{}

							# old versions did not store training args
							snake_case_      =				input_state_dict.get("args"							,    snake_case					)
							if ds_args is not None:
														# do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
														# from pprint import pprint
														# pprint(vars(ds_args))

														snake_case_      =				ds_args.padded_vocab_size
														snake_case_      =				ds_args.max_position_embeddings
														snake_case_      =				ds_args.hidden_size
														snake_case_      =				ds_args.num_layers
														snake_case_      =				ds_args.num_attention_heads
														snake_case_      =				ds_args.ffn_hidden_size
														# pprint(config)

							# The number of heads.
							snake_case_      =				config.n_head
							# The hidden_size per head.
							snake_case_      =				config.n_embd // config.n_head
							# Megatron-LM checkpoint version
							if "checkpoint_version" in input_state_dict.keys():
														snake_case_      =				input_state_dict["checkpoint_version"]
							else:
														snake_case_      =				0.0

							# The model.
							snake_case_      =				input_state_dict["model"]
							# The language model.
							snake_case_      =				model["language_model"]
							# The embeddings.
							snake_case_      =				lm["embedding"]

							# The word embeddings.
							snake_case_      =				embeddings["word_embeddings"]["weight"]
							# Truncate the embedding table to vocab_size rows.
							snake_case_      =				word_embeddings[: config.vocab_size, :]
							snake_case_      =				word_embeddings

							# The position embeddings.
							snake_case_      =				embeddings["position_embeddings"]["weight"]
							# Read the causal mask dimension (seqlen). [max_sequence_length, hidden_size]
							snake_case_      =				pos_embeddings.size(0					)
							if n_positions != config.n_positions:
														raise ValueError(
														    f'pos_embeddings.max_sequence_length={n_positions} and config.n_positions={config.n_positions} don\'t match'					)
							# Store the position embeddings.
							snake_case_      =				pos_embeddings

							# The transformer.
							snake_case_      =				lm["transformer"] if "transformer" in lm.keys() else lm["encoder"]

							# The regex to extract layer names.
							snake_case_      =				re.compile(r"layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)"					)

							# The simple map of names for "automated" rules.
							snake_case_      =				{
							    "attention.dense": ".attn.c_proj.",
							    "self_attention.dense": ".attn.c_proj.",
							    "mlp.dense_h_to_4h": ".mlp.c_fc.",
							    "mlp.dense_4h_to_h": ".mlp.c_proj.",
							}

							# Extract the layers.
							for key, val in transformer.items():
														# Match the name.
														snake_case_      =				layer_re.match(snake_case					)

														# Stop if that's not a layer
														if m is None:
																					break

														# The index of the layer.
														snake_case_      =				int(m.group(1					)					)
														# The name of the operation.
														snake_case_      =				m.group(2					)
														# Is it a weight or a bias?
														snake_case_      =				m.group(3					)

														# The name of the layer.
														snake_case_      =				f'transformer.h.{layer_idx}'

														# For layernorm(s), simply store the layer norm.
														if op_name.endswith("layernorm"					):
																					snake_case_      =				"ln_1" if op_name.startswith("input"					) else "ln_2"
																					snake_case_      =				val

														# Transpose the QKV matrix.
														elif (
														    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
														) and weight_or_bias == "weight":
																					# Insert a tensor of 1x1xDxD bias.
																					snake_case_      =				torch.tril(torch.ones((n_positions, n_positions)							,    dtype=torch.floataa					)					).view(
																					    1							,    1							,    snake_case							,    snake_case					)
																					snake_case_      =				causal_mask

																					# Insert a "dummy" tensor for masked_bias.
																					snake_case_      =				torch.tensor(-1e4							,    dtype=torch.floataa					)
																					snake_case_      =				masked_bias

																					snake_case_      =				fix_query_key_value_ordering(snake_case							,    snake_case							,    3							,    snake_case							,    snake_case					)
																					# Megatron stores (3*D) x D but transformers-GPT2 expects D x 3*D.
																					snake_case_      =				out_val.transpose(0							,    1					).contiguous()
																					# Store.
																					snake_case_      =				out_val

														# Transpose the bias.
														elif (
														    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
														) and weight_or_bias == "bias":
																					snake_case_      =				fix_query_key_value_ordering(snake_case							,    snake_case							,    3							,    snake_case							,    snake_case					)
																					# Store. No change of shape.
																					snake_case_      =				out_val

														# Transpose the weights.
														elif weight_or_bias == "weight":
																					snake_case_      =				megatron_to_transformers[op_name]
																					snake_case_      =				val.transpose(0							,    1					)

														# Copy the bias.
														elif weight_or_bias == "bias":
																					snake_case_      =				megatron_to_transformers[op_name]
																					snake_case_      =				val

    # DEBUG.
							assert config.n_layer == layer_idx + 1

							# The final layernorm.
							snake_case_      =				transformer["final_layernorm.weight"]
							snake_case_      =				transformer["final_layernorm.bias"]

							# For LM head, transformers' wants the matrix to weight embeddings.
							snake_case_      =				word_embeddings

							# It should be done!
							return output_state_dict


def 					UpperCamelCase_(     ):

							'''simple docstring'''






							snake_case_      =				argparse.ArgumentParser()
							parser.add_argument("--print-checkpoint-structure"							,    action="store_true"					)
							parser.add_argument(
							    "path_to_checkpoint"							,    type=snake_case							,    help="Path to the checkpoint file (.zip archive or direct .pt file)"							,    )
							parser.add_argument(
							    "--config_file"							,    default=""							,    type=snake_case							,    help="An optional config json file describing the pre-trained model."							,    )
							snake_case_      =				parser.parse_args()

							# Extract the basename.
							snake_case_      =				os.path.dirname(args.path_to_checkpoint					)

							# Load the model.
							# the .zip is very optional, let's keep it for backward compatibility
							print(f'Extracting PyTorch state dictionary from {args.path_to_checkpoint}'					)
							if args.path_to_checkpoint.endswith(".zip"					):
														with zipfile.ZipFile(args.path_to_checkpoint							,    "r"					) as checkpoint:
																					with checkpoint.open("release/mp_rank_00/model_optim_rng.pt"					) as pytorch_dict:
																												snake_case_      =				torch.load(snake_case							,    map_location="cpu"					)
							else:
														snake_case_      =				torch.load(args.path_to_checkpoint							,    map_location="cpu"					)

							snake_case_      =				input_state_dict.get("args"							,    snake_case					)

							# Read the config, or default to the model released by NVIDIA.
							if args.config_file == "":
														if ds_args is not None:
																					if ds_args.bias_gelu_fusion:
																												snake_case_      =				"gelu_fast"
																					elif ds_args.openai_gelu:
																												snake_case_      =				"gelu_new"
																					else:
																												snake_case_      =				"gelu"
														else:
																					# in the very early days this used to be "gelu_new"
																					snake_case_      =				"gelu_new"

														# Spell out all parameters in case the defaults change.
														snake_case_      =				GPTaConfig(
														    vocab_size=5_0_2_5_7							,    n_positions=1_0_2_4							,    n_embd=1_0_2_4							,    n_layer=2_4							,    n_head=1_6							,    n_inner=4_0_9_6							,    activation_function=snake_case							,    resid_pdrop=0.1							,    embd_pdrop=0.1							,    attn_pdrop=0.1							,    layer_norm_epsilon=1e-5							,    initializer_range=0.02							,    summary_type="cls_index"							,    summary_use_proj=snake_case							,    summary_activation=snake_case							,    summary_proj_to_labels=snake_case							,    summary_first_dropout=0.1							,    scale_attn_weights=snake_case							,    use_cache=snake_case							,    bos_token_id=5_0_2_5_6							,    eos_token_id=5_0_2_5_6							,    )
							else:
														snake_case_      =				GPTaConfig.from_json_file(args.config_file					)

							snake_case_      =				["GPT2LMHeadModel"]

							# Convert.
							print("Converting"					)
							snake_case_      =				convert_megatron_checkpoint(snake_case							,    snake_case							,    snake_case					)

							# Print the structure of converted state dict.
							if args.print_checkpoint_structure:
														recursive_print(snake_case							,    snake_case					)

							# Add tokenizer class info to config
							# see https://github.com/huggingface/transformers/issues/13906)
							if ds_args is not None:
														snake_case_      =				ds_args.tokenizer_type
														if tokenizer_type == "GPT2BPETokenizer":
																					snake_case_      =				"gpt2"
														elif tokenizer_type == "PretrainedFromHF":
																					snake_case_      =				ds_args.tokenizer_name_or_path
														else:
																					raise ValueError(f'Unrecognized tokenizer_type {tokenizer_type}'					)
							else:
														snake_case_      =				"gpt2"

							snake_case_      =				AutoTokenizer.from_pretrained(snake_case					)
							snake_case_      =				type(snake_case					).__name__
							snake_case_      =				tokenizer_class

							# Store the config to file.
							print("Saving config"					)
							config.save_pretrained(snake_case					)

							# Save tokenizer based on args
							print(f'Adding {tokenizer_class} tokenizer files'					)
							tokenizer.save_pretrained(snake_case					)

							# Store the state_dict to file.
							snake_case_      =				os.path.join(snake_case							,    "pytorch_model.bin"					)
							print(f'Saving checkpoint to "{output_checkpoint_file}"'					)
							torch.save(snake_case							,    snake_case					)


####################################################################################################

if __name__ == "__main__":
				main()

####################################################################################################










'''simple docstring'''





from sympy import diff, lambdify, symbols
from sympy.functions import *  # noqa: F403


def 					UpperCamelCase_(     snake_case							:	str							,    snake_case							:	complex							,    snake_case							:	str = "x"							,    snake_case							:	float = 1_0**-1_0							,    snake_case							:	int = 1							,    ):

							'''simple docstring'''






							snake_case_      =				symbols(snake_case					)
							snake_case_      =				lambdify(snake_case							,    snake_case					)
							snake_case_      =				lambdify(snake_case							,    diff(snake_case							,    snake_case					)					)

							snake_case_      =				starting_point

							while True:
														if diff_function(snake_case					) != 0:
																					snake_case_      =				prev_guess - multiplicity * func(snake_case					) / diff_function(
																					    snake_case					)
														else:
																					raise ZeroDivisionError("Could not find root"					) from None

														# Precision is checked by comparing the difference of consecutive guesses
														if abs(next_guess - prev_guess					) < precision:
																					return next_guess

														snake_case_      =				next_guess


# Let's Execute
if __name__ == "__main__":
		# Find root of trigonometric function
		# Find value of pi
		print(F"The root of sin(x) = 0 is {newton_raphson('sin(x)', 2)}")

		# Find root of polynomial
		# Find fourth Root of 5
		print(F"The root of x**4 - 5 = 0 is {newton_raphson('x**4 -5', 0.4 +5j)}")

		# Find value of e
		print(
		    "The root of log(y) - 1 = 0 is ",
		    F"{newton_raphson('log(y) - 1', 2, variable='y')}",
		)

		# Exponential Roots
		print(
		    "The root of exp(x) - 1 = 0 is",
		    F"{newton_raphson('exp(x) - 1', 10, precision=0.0_0_5)}",
		)

		# Find root of cos(x)
		print(F"The root of cos(x) = 0 is {newton_raphson('cos(x)', 0)}")




