




import argparse
import json
import os

import torch
from torch import nn

from transformers import NllbMoeConfig, NllbMoeModel
from transformers.modeling_utils import dtype_byte_size
from transformers.utils import WEIGHTS_INDEX_NAME, WEIGHTS_NAME







def      _a				(						SCREAMING_SNAKE_CASE_							:    Any					):
					__lowerCAmelCase           =	[
					    "encoder.version",
					    "decoder.version",
					    "model.encoder.version",
					    "model.decoder.version",
					    "decoder.output_projection.weight",
					    "_float_tensor",
					    "encoder.embed_positions._float_tensor",
					    "decoder.embed_positions._float_tensor",
					]
					for k in ignore_keys:
										state_dict.pop(SCREAMING_SNAKE_CASE_				,				SCREAMING_SNAKE_CASE_					)







def      _a				(						SCREAMING_SNAKE_CASE_							:    Tuple					):
					__lowerCAmelCase       ,							__lowerCAmelCase           =	emb.weight.shape
					__lowerCAmelCase           =	nn.Linear(SCREAMING_SNAKE_CASE_				,				SCREAMING_SNAKE_CASE_				,				bias=SCREAMING_SNAKE_CASE_					)
					__lowerCAmelCase           =	emb.weight.data
					return lin_layer







def      _a				(						SCREAMING_SNAKE_CASE_							:    Optional[Any]				,				SCREAMING_SNAKE_CASE_							:    int=None					):
					__lowerCAmelCase           =	{}
					for old_key in state_dict.keys():
										__lowerCAmelCase           =	old_key
										if "moe_layer.experts." in key:
															if expert_idx is not None:
																				__lowerCAmelCase           =	key.replace("moe_layer.experts.0"				,				F"""ffn.experts.expert_{expert_idx}"""					)
															else:
																				__lowerCAmelCase           =	key.replace("moe_layer.experts."				,				"ffn.experts.expert_"					)
										if "gate" in key:
															__lowerCAmelCase           =	key.replace(".moe_layer.gate.wg"				,				".ffn.router.classifier"					)
										if "fc2" and "experts" not in key:
															__lowerCAmelCase           =	key.replace(".fc2."				,				".ffn.fc2."					)
										if "fc1" and "experts" not in key:
															__lowerCAmelCase           =	key.replace(".fc1."				,				".ffn.fc1."					)
										if ".encoder_attn." in key:
															__lowerCAmelCase           =	key.replace(".encoder_attn."				,				".cross_attention."					)
										if "encoder_attn_layer_norm" in key:
															__lowerCAmelCase           =	key.replace("encoder_attn_layer_norm"				,				"cross_attention_layer_norm"					)
										if "final_layer_norm" in key:
															__lowerCAmelCase           =	key.replace("final_layer_norm"				,				"ff_layer_norm"					)
										__lowerCAmelCase           =	state_dict[old_key]
					return new_dict







def      _a				(						SCREAMING_SNAKE_CASE_							:    Any				,				SCREAMING_SNAKE_CASE_							:    int				,				SCREAMING_SNAKE_CASE_							:    Optional[Any]				,				SCREAMING_SNAKE_CASE_							:    str				,				SCREAMING_SNAKE_CASE_							:    str = WEIGHTS_NAME					):
					__lowerCAmelCase           =	[]
					__lowerCAmelCase           =	0
					os.makedirs(SCREAMING_SNAKE_CASE_				,				exist_ok=SCREAMING_SNAKE_CASE_					)

					for expert in range(SCREAMING_SNAKE_CASE_					):
										__lowerCAmelCase           =	switch_checkpoint_path + F"""-rank-{expert}.pt"""
										if os.path.isfile(SCREAMING_SNAKE_CASE_					):
															__lowerCAmelCase           =	torch.load(SCREAMING_SNAKE_CASE_					)["model"]
															remove_ignore_keys_(SCREAMING_SNAKE_CASE_					)
															__lowerCAmelCase           =	rename_fairseq_keys(SCREAMING_SNAKE_CASE_				,				SCREAMING_SNAKE_CASE_					)
															__lowerCAmelCase           =	os.path.join(
															    SCREAMING_SNAKE_CASE_				,				weights_name.replace(".bin"				,				F"""-{len(SCREAMING_SNAKE_CASE_					)+1:05d}-of-???.bin"""					)					)
															torch.save(SCREAMING_SNAKE_CASE_				,				SCREAMING_SNAKE_CASE_					)
															sharded_state_dicts.append(expert_state.keys()					)
															total_size += sum([value.numel() for key, value in expert_state.items()]					) * dtype_byte_size(
															    expert_state[list(SCREAMING_SNAKE_CASE_					)[0]].dtype					)

    # Add the last block
					__lowerCAmelCase           =	os.path.join(SCREAMING_SNAKE_CASE_				,				weights_name.replace(".bin"				,				F"""-{len(SCREAMING_SNAKE_CASE_					)+1:05d}-of-???.bin"""					)					)
					__lowerCAmelCase           =	torch.load(switch_checkpoint_path + "-shared.pt"					)["model"]
					remove_ignore_keys_(SCREAMING_SNAKE_CASE_					)
					__lowerCAmelCase           =	rename_fairseq_keys(SCREAMING_SNAKE_CASE_				,				SCREAMING_SNAKE_CASE_					)
					__lowerCAmelCase           =	shared_weights["decoder.embed_tokens.weight"]
					sharded_state_dicts.append(shared_weights.keys()					)

					# If we only have the shared weights (dummy model/experts saved on the same file)
					if len(SCREAMING_SNAKE_CASE_					) == 1:
										__lowerCAmelCase           =	os.path.join(SCREAMING_SNAKE_CASE_				,				SCREAMING_SNAKE_CASE_					)
										torch.save(SCREAMING_SNAKE_CASE_				,				SCREAMING_SNAKE_CASE_					)
										return {weights_name: sharded_state_dicts[0]}, None
					else:
										torch.save(SCREAMING_SNAKE_CASE_				,				SCREAMING_SNAKE_CASE_					)
					# Otherwise, let's build the index
					__lowerCAmelCase           =	{}
					for idx, shard in enumerate(SCREAMING_SNAKE_CASE_					):
										__lowerCAmelCase           =	weights_name.replace(".bin"				,				F"""-{idx+1:05d}-of-{len(SCREAMING_SNAKE_CASE_					):05d}.bin"""					)
										__lowerCAmelCase           =	os.path.join(SCREAMING_SNAKE_CASE_				,				weights_name.replace(".bin"				,				F"""-{idx+1:05d}-of-???.bin"""					)					)
										os.rename(SCREAMING_SNAKE_CASE_				,				os.path.join(SCREAMING_SNAKE_CASE_				,				SCREAMING_SNAKE_CASE_					)					)
										for key in shard:
															__lowerCAmelCase           =	shard_file

    # Add the metadata
					__lowerCAmelCase           =	{"total_size": total_size}
					__lowerCAmelCase           =	{"metadata": metadata, "weight_map": weight_map}

					with open(os.path.join(SCREAMING_SNAKE_CASE_				,				SCREAMING_SNAKE_CASE_					)				,				"w"				,				encoding="utf-8"					) as f:
										__lowerCAmelCase           =	json.dumps(SCREAMING_SNAKE_CASE_				,				indent=2				,				sort_keys=SCREAMING_SNAKE_CASE_					) + "\n"
										f.write(SCREAMING_SNAKE_CASE_					)

					return metadata, index


if __name__ == "__main__":
					UpperCamelCase__         =      argparse.ArgumentParser()
					# Required parameters
					parser.add_argument(
					    """--nllb_moe_checkpoint_path""",
					    default="""/home/arthur_huggingface_co/fairseq/weights/checkpoints/model_moe_54b/checkpoint_2_300000""",
					    type=str,
					    required=False,
					    help="""Path to a directory containing a folder per layer. Follows the original Google format.""",
					)
					parser.add_argument("""--dtype""", default="""float32""", type=str, required=False, help="""dtype of the saved model""")
					parser.add_argument(
					    """--pytorch_dump_folder_path""",
					    default="""/home/arthur_huggingface_co/fairseq/weights/checkpoints/hf-converted-moe-54b""",
					    type=str,
					    required=False,
					    help="""Path to the output pytorch model.""",
					)
					UpperCamelCase__         =      parser.parse_args()
					UpperCamelCase__      ,     UpperCamelCase__         =      shard_on_the_fly(
					    args.nllb_moe_checkpoint_path,
					    args.pytorch_dump_folder_path,
					    128,
					    args.dtype,
					)

					UpperCamelCase__         =      NllbMoeConfig.from_pretrained(
					    """facebook/nllb-200-3.3B""", encoder_sparse_step=4, decoder_sparse_step=4, num_experts=128
					)
					config.save_pretrained(args.pytorch_dump_folder_path)
					UpperCamelCase__         =      NllbMoeModel.from_pretrained(args.pytorch_dump_folder_path)
					print("""Done""")
					model.save_pretrained(args.pytorch_dump_folder_path)










from __future__ import annotations

import copy
import tempfile
import unittest

from transformers import CONFIG_MAPPING, AutoConfig, BertConfig, GPTaConfig, TaConfig, TapasConfig, is_tf_available
from transformers.testing_utils import (
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tensorflow_probability,
    require_tf,
    slow,
)

from ..bert.test_modeling_bert import BertModelTester


if is_tf_available():
					from transformers import (
					    TFAutoModel,
					    TFAutoModelForCausalLM,
					    TFAutoModelForMaskedLM,
					    TFAutoModelForPreTraining,
					    TFAutoModelForQuestionAnswering,
					    TFAutoModelForSeqaSeqLM,
					    TFAutoModelForSequenceClassification,
					    TFAutoModelForTableQuestionAnswering,
					    TFAutoModelForTokenClassification,
					    TFAutoModelWithLMHead,
					    TFBertForMaskedLM,
					    TFBertForPreTraining,
					    TFBertForQuestionAnswering,
					    TFBertForSequenceClassification,
					    TFBertModel,
					    TFFunnelBaseModel,
					    TFFunnelModel,
					    TFGPTaLMHeadModel,
					    TFRobertaForMaskedLM,
					    TFTaForConditionalGeneration,
					    TFTapasForQuestionAnswering,
					)
					from transformers.models.auto.modeling_tf_auto import (
					    TF_MODEL_FOR_CAUSAL_LM_MAPPING,
					    TF_MODEL_FOR_MASKED_LM_MAPPING,
					    TF_MODEL_FOR_PRETRAINING_MAPPING,
					    TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
					    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
					    TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
					    TF_MODEL_MAPPING,
					)
					from transformers.models.bert.modeling_tf_bert import TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST
					from transformers.models.gpta.modeling_tf_gpta import TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST
					from transformers.models.ta.modeling_tf_ta import TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST
					from transformers.models.tapas.modeling_tf_tapas import TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST
class       a__   ( snake_case__				):
						_a							:   Optional[int] =   """new-model"""


if is_tf_available():
					class       a__   ( snake_case__				):
											_a							:   Dict =   NewModelConfig
@require_tf
class       a__   ( unittest.TestCase				):






						@slow
						def     __SCREAMING_SNAKE_CASE(       self      ):




											"""simple docstring"""



											__lowerCAmelCase           =	"bert-base-cased"
											__lowerCAmelCase           =	AutoConfig.from_pretrained(_A      )
											self.assertIsNotNone(_A      )
											self.assertIsInstance(_A		,      _A      )

											__lowerCAmelCase           =	TFAutoModel.from_pretrained(_A      )
											self.assertIsNotNone(_A      )
											self.assertIsInstance(_A		,      _A      )






						@slow
						def     __SCREAMING_SNAKE_CASE(       self      ):




											"""simple docstring"""



											__lowerCAmelCase           =	"bert-base-cased"
											__lowerCAmelCase           =	AutoConfig.from_pretrained(_A      )
											self.assertIsNotNone(_A      )
											self.assertIsInstance(_A		,      _A      )

											__lowerCAmelCase           =	TFAutoModelForPreTraining.from_pretrained(_A      )
											self.assertIsNotNone(_A      )
											self.assertIsInstance(_A		,      _A      )






						@slow
						def     __SCREAMING_SNAKE_CASE(       self      ):




											"""simple docstring"""



											for model_name in TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																__lowerCAmelCase           =	AutoConfig.from_pretrained(_A      )
																self.assertIsNotNone(_A      )
																self.assertIsInstance(_A		,      _A      )

																__lowerCAmelCase           =	TFAutoModelForCausalLM.from_pretrained(_A      )
																__lowerCAmelCase       ,							__lowerCAmelCase           =	TFAutoModelForCausalLM.from_pretrained(_A		,      output_loading_info=_A      )
																self.assertIsNotNone(_A      )
																self.assertIsInstance(_A		,      _A      )






						@slow
						def     __SCREAMING_SNAKE_CASE(       self      ):




											"""simple docstring"""



											for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																__lowerCAmelCase           =	AutoConfig.from_pretrained(_A      )
																self.assertIsNotNone(_A      )
																self.assertIsInstance(_A		,      _A      )

																__lowerCAmelCase           =	TFAutoModelWithLMHead.from_pretrained(_A      )
																self.assertIsNotNone(_A      )
																self.assertIsInstance(_A		,      _A      )






						@slow
						def     __SCREAMING_SNAKE_CASE(       self      ):




											"""simple docstring"""



											for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																__lowerCAmelCase           =	AutoConfig.from_pretrained(_A      )
																self.assertIsNotNone(_A      )
																self.assertIsInstance(_A		,      _A      )

																__lowerCAmelCase           =	TFAutoModelForMaskedLM.from_pretrained(_A      )
																__lowerCAmelCase       ,							__lowerCAmelCase           =	TFAutoModelForMaskedLM.from_pretrained(_A		,      output_loading_info=_A      )
																self.assertIsNotNone(_A      )
																self.assertIsInstance(_A		,      _A      )






						@slow
						def     __SCREAMING_SNAKE_CASE(       self      ):




											"""simple docstring"""



											for model_name in TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																__lowerCAmelCase           =	AutoConfig.from_pretrained(_A      )
																self.assertIsNotNone(_A      )
																self.assertIsInstance(_A		,      _A      )

																__lowerCAmelCase           =	TFAutoModelForSeqaSeqLM.from_pretrained(_A      )
																__lowerCAmelCase       ,							__lowerCAmelCase           =	TFAutoModelForSeqaSeqLM.from_pretrained(_A		,      output_loading_info=_A      )
																self.assertIsNotNone(_A      )
																self.assertIsInstance(_A		,      _A      )






						@slow
						def     __SCREAMING_SNAKE_CASE(       self      ):




											"""simple docstring"""



											for model_name in ["bert-base-uncased"]:
																__lowerCAmelCase           =	AutoConfig.from_pretrained(_A      )
																self.assertIsNotNone(_A      )
																self.assertIsInstance(_A		,      _A      )

																__lowerCAmelCase           =	TFAutoModelForSequenceClassification.from_pretrained(_A      )
																self.assertIsNotNone(_A      )
																self.assertIsInstance(_A		,      _A      )






						@slow
						def     __SCREAMING_SNAKE_CASE(       self      ):




											"""simple docstring"""



											for model_name in ["bert-base-uncased"]:
																__lowerCAmelCase           =	AutoConfig.from_pretrained(_A      )
																self.assertIsNotNone(_A      )
																self.assertIsInstance(_A		,      _A      )

																__lowerCAmelCase           =	TFAutoModelForQuestionAnswering.from_pretrained(_A      )
																self.assertIsNotNone(_A      )
																self.assertIsInstance(_A		,      _A      )






						@slow
						@require_tensorflow_probability
						def     __SCREAMING_SNAKE_CASE(       self      ):




											"""simple docstring"""



											for model_name in TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST[5:6]:
																__lowerCAmelCase           =	AutoConfig.from_pretrained(_A      )
																self.assertIsNotNone(_A      )
																self.assertIsInstance(_A		,      _A      )

																__lowerCAmelCase           =	TFAutoModelForTableQuestionAnswering.from_pretrained(_A      )
																__lowerCAmelCase       ,							__lowerCAmelCase           =	TFAutoModelForTableQuestionAnswering.from_pretrained(
																    _A		,      output_loading_info=_A      )
																self.assertIsNotNone(_A      )
																self.assertIsInstance(_A		,      _A      )






						def     __SCREAMING_SNAKE_CASE(       self      ):




											"""simple docstring"""



											__lowerCAmelCase           =	TFAutoModelWithLMHead.from_pretrained(_A      )
											self.assertIsInstance(_A		,      _A      )
											self.assertEqual(model.num_parameters()		,      1_4_4_1_0      )
											self.assertEqual(model.num_parameters(only_trainable=_A      )		,      1_4_4_1_0      )






						def     __SCREAMING_SNAKE_CASE(       self      ):




											"""simple docstring"""



											__lowerCAmelCase           =	TFAutoModelWithLMHead.from_pretrained(_A      )
											self.assertIsInstance(_A		,      _A      )
											self.assertEqual(model.num_parameters()		,      1_4_4_1_0      )
											self.assertEqual(model.num_parameters(only_trainable=_A      )		,      1_4_4_1_0      )






						def     __SCREAMING_SNAKE_CASE(       self      ):




											"""simple docstring"""



											__lowerCAmelCase           =	TFAutoModel.from_pretrained("sgugger/funnel-random-tiny"      )
											self.assertIsInstance(_A		,      _A      )

											__lowerCAmelCase           =	copy.deepcopy(model.config      )
											__lowerCAmelCase           =	["FunnelBaseModel"]
											__lowerCAmelCase           =	TFAutoModel.from_config(_A      )
											self.assertIsInstance(_A		,      _A      )

											with tempfile.TemporaryDirectory() as tmp_dir:
																model.save_pretrained(_A      )
																__lowerCAmelCase           =	TFAutoModel.from_pretrained(_A      )
																self.assertIsInstance(_A		,      _A      )






						def     __SCREAMING_SNAKE_CASE(       self      ):




											"""simple docstring"""



											try:
																AutoConfig.register("new-model"		,      _A      )

																__lowerCAmelCase           =	[
																    TFAutoModel,
																    TFAutoModelForCausalLM,
																    TFAutoModelForMaskedLM,
																    TFAutoModelForPreTraining,
																    TFAutoModelForQuestionAnswering,
																    TFAutoModelForSequenceClassification,
																    TFAutoModelForTokenClassification,
																]

																for auto_class in auto_classes:
																					with self.subTest(auto_class.__name__      ):
																										# Wrong config class will raise an error
																										with self.assertRaises(_A      ):
																															auto_class.register(_A		,      _A      )
																										auto_class.register(_A		,      _A      )
																										# Trying to register something existing in the Transformers library will raise an error
																										with self.assertRaises(_A      ):
																															auto_class.register(_A		,      _A      )

																										# Now that the config is registered, it can be used as any other config with the auto-API
																										__lowerCAmelCase           =	BertModelTester(self      ).get_config()
																										__lowerCAmelCase           =	NewModelConfig(**tiny_config.to_dict()      )
																										__lowerCAmelCase           =	auto_class.from_config(_A      )
																										self.assertIsInstance(_A		,      _A      )

																										with tempfile.TemporaryDirectory() as tmp_dir:
																															model.save_pretrained(_A      )
																															__lowerCAmelCase           =	auto_class.from_pretrained(_A      )
																															self.assertIsInstance(_A		,      _A      )

											finally:
																if "new-model" in CONFIG_MAPPING._extra_content:
																					del CONFIG_MAPPING._extra_content["new-model"]
																for mapping in (
																    TF_MODEL_MAPPING,
																    TF_MODEL_FOR_PRETRAINING_MAPPING,
																    TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
																    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
																    TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
																    TF_MODEL_FOR_CAUSAL_LM_MAPPING,
																    TF_MODEL_FOR_MASKED_LM_MAPPING,
																):
																					if NewModelConfig in mapping._extra_content:
																										del mapping._extra_content[NewModelConfig]






						def     __SCREAMING_SNAKE_CASE(       self      ):




											"""simple docstring"""



											with self.assertRaisesRegex(
											    _A		,      "bert-base is not a local folder and is not a valid model identifier"      ):
																__lowerCAmelCase           =	TFAutoModel.from_pretrained("bert-base"      )






						def     __SCREAMING_SNAKE_CASE(       self      ):




											"""simple docstring"""



											with self.assertRaisesRegex(
											    _A		,      R"aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"      ):
																__lowerCAmelCase           =	TFAutoModel.from_pretrained(_A		,      revision="aaaaaa"      )






						def     __SCREAMING_SNAKE_CASE(       self      ):




											"""simple docstring"""



											with self.assertRaisesRegex(
											    _A		,      "hf-internal-testing/config-no-model does not appear to have a file named pytorch_model.bin"		,      ):
																__lowerCAmelCase           =	TFAutoModel.from_pretrained("hf-internal-testing/config-no-model"      )






						def     __SCREAMING_SNAKE_CASE(       self      ):




											"""simple docstring"""



											with self.assertRaisesRegex(_A		,      "Use `from_pt=True` to load this model"      ):
																__lowerCAmelCase           =	TFAutoModel.from_pretrained("hf-internal-testing/tiny-bert-pt-only"      )






						def     __SCREAMING_SNAKE_CASE(       self      ):




											"""simple docstring"""



											__lowerCAmelCase           =	TFAutoModel.from_pretrained("hf-internal-testing/tiny-random-bert"      )
											with RequestCounter() as counter:
																__lowerCAmelCase           =	TFAutoModel.from_pretrained("hf-internal-testing/tiny-random-bert"      )
																self.assertEqual(counter.get_request_count		,      0      )
																self.assertEqual(counter.head_request_count		,      1      )
																self.assertEqual(counter.other_request_count		,      0      )

											# With a sharded checkpoint
											__lowerCAmelCase           =	TFAutoModel.from_pretrained("ArthurZ/tiny-random-bert-sharded"      )
											with RequestCounter() as counter:
																__lowerCAmelCase           =	TFAutoModel.from_pretrained("ArthurZ/tiny-random-bert-sharded"      )
																self.assertEqual(counter.get_request_count		,      0      )
																self.assertEqual(counter.head_request_count		,      1      )
																self.assertEqual(counter.other_request_count		,      0      )




