





import argparse
import json
import os

import torch
from torch import nn

from transformers import NllbMoeConfig, NllbMoeModel
from transformers.modeling_utils import dtype_byte_size
from transformers.utils import WEIGHTS_INDEX_NAME, WEIGHTS_NAME
def 					_A	(		SCREAMING_SNAKE_CASE					:       Optional[int]						):





  """simple docstring"""



  a__     : Tuple  =[
      "encoder.version",
      "decoder.version",
      "model.encoder.version",
      "model.decoder.version",
      "decoder.output_projection.weight",
      "_float_tensor",
      "encoder.embed_positions._float_tensor",
      "decoder.embed_positions._float_tensor",
  ]
  for k in ignore_keys:
    state_dict.pop(SCREAMING_SNAKE_CASE	,	SCREAMING_SNAKE_CASE						)
def 					_A	(		SCREAMING_SNAKE_CASE					:       Optional[Any]						):





  """simple docstring"""



  a__ ,							a__     : Any  =emb.weight.shape
  a__     : Optional[Any]  =nn.Linear(SCREAMING_SNAKE_CASE	,	SCREAMING_SNAKE_CASE	,	bias=SCREAMING_SNAKE_CASE						)
  a__     : Any  =emb.weight.data
  return lin_layer
def 					_A	(		SCREAMING_SNAKE_CASE					:       Tuple	,	SCREAMING_SNAKE_CASE					:       Union[str, Any]=None						):





  """simple docstring"""



  a__     : Tuple  ={}
  for old_key in state_dict.keys():
    a__     : int  =old_key
    if "moe_layer.experts." in key:
      if expert_idx is not None:
        a__     : int  =key.replace("moe_layer.experts.0"	,	f'''ffn.experts.expert_{expert_idx}'''						)
      else:
        a__     : int  =key.replace("moe_layer.experts."	,	"ffn.experts.expert_"						)
    if "gate" in key:
      a__     : List[Any]  =key.replace(".moe_layer.gate.wg"	,	".ffn.router.classifier"						)
    if "fc2" and "experts" not in key:
      a__     : Union[str, Any]  =key.replace(".fc2."	,	".ffn.fc2."						)
    if "fc1" and "experts" not in key:
      a__     : Optional[Any]  =key.replace(".fc1."	,	".ffn.fc1."						)
    if ".encoder_attn." in key:
      a__     : int  =key.replace(".encoder_attn."	,	".cross_attention."						)
    if "encoder_attn_layer_norm" in key:
      a__     : Union[str, Any]  =key.replace("encoder_attn_layer_norm"	,	"cross_attention_layer_norm"						)
    if "final_layer_norm" in key:
      a__     : Tuple  =key.replace("final_layer_norm"	,	"ff_layer_norm"						)
    a__     : Optional[int]  =state_dict[old_key]
  return new_dict
def 					_A	(		SCREAMING_SNAKE_CASE					:       Any	,	SCREAMING_SNAKE_CASE					:       Optional[Any]	,	SCREAMING_SNAKE_CASE					:       Union[str, Any]	,	SCREAMING_SNAKE_CASE					:       Optional[int]	,	SCREAMING_SNAKE_CASE					:       str = WEIGHTS_NAME						):





  """simple docstring"""



  a__     : Any  =[]
  a__     : List[str]  =0
  os.makedirs(SCREAMING_SNAKE_CASE	,	exist_ok=SCREAMING_SNAKE_CASE						)

  for expert in range(SCREAMING_SNAKE_CASE						):
    a__     : Optional[Any]  =switch_checkpoint_path + f'''-rank-{expert}.pt'''
    if os.path.isfile(SCREAMING_SNAKE_CASE						):
      a__     : Union[str, Any]  =torch.load(SCREAMING_SNAKE_CASE						)["model"]
      remove_ignore_keys_(SCREAMING_SNAKE_CASE						)
      a__     : Tuple  =rename_fairseq_keys(SCREAMING_SNAKE_CASE	,	SCREAMING_SNAKE_CASE						)
      a__     : str  =os.path.join(
          SCREAMING_SNAKE_CASE	,	weights_name.replace(".bin"	,	f'''-{len(SCREAMING_SNAKE_CASE						)+1:05d}-of-???.bin'''						)						)
      torch.save(SCREAMING_SNAKE_CASE	,	SCREAMING_SNAKE_CASE						)
      sharded_state_dicts.append(expert_state.keys()						)
      total_size += sum([value.numel() for key, value in expert_state.items()]						) * dtype_byte_size(
          expert_state[list(SCREAMING_SNAKE_CASE						)[0]].dtype						)

    # Add the last block
  a__     : str  =os.path.join(SCREAMING_SNAKE_CASE	,	weights_name.replace(".bin"	,	f'''-{len(SCREAMING_SNAKE_CASE						)+1:05d}-of-???.bin'''						)						)
  a__     : List[str]  =torch.load(switch_checkpoint_path + "-shared.pt"						)["model"]
  remove_ignore_keys_(SCREAMING_SNAKE_CASE						)
  a__     : Optional[int]  =rename_fairseq_keys(SCREAMING_SNAKE_CASE	,	SCREAMING_SNAKE_CASE						)
  a__     : List[Any]  =shared_weights["decoder.embed_tokens.weight"]
  sharded_state_dicts.append(shared_weights.keys()						)

  # If we only have the shared weights (dummy model/experts saved on the same file)
  if len(SCREAMING_SNAKE_CASE						) == 1:
    a__     : Dict  =os.path.join(SCREAMING_SNAKE_CASE	,	SCREAMING_SNAKE_CASE						)
    torch.save(SCREAMING_SNAKE_CASE	,	SCREAMING_SNAKE_CASE						)
    return {weights_name: sharded_state_dicts[0]}, None
  else:
    torch.save(SCREAMING_SNAKE_CASE	,	SCREAMING_SNAKE_CASE						)
  # Otherwise, let's build the index
  a__     : List[Any]  ={}
  for idx, shard in enumerate(SCREAMING_SNAKE_CASE						):
    a__     : Union[str, Any]  =weights_name.replace(".bin"	,	f'''-{idx+1:05d}-of-{len(SCREAMING_SNAKE_CASE						):05d}.bin'''						)
    a__     : Dict  =os.path.join(SCREAMING_SNAKE_CASE	,	weights_name.replace(".bin"	,	f'''-{idx+1:05d}-of-???.bin'''						)						)
    os.rename(SCREAMING_SNAKE_CASE	,	os.path.join(SCREAMING_SNAKE_CASE	,	SCREAMING_SNAKE_CASE						)						)
    for key in shard:
      a__     : List[Any]  =shard_file

    # Add the metadata
  a__     : List[Any]  ={"total_size": total_size}
  a__     : int  ={"metadata": metadata, "weight_map": weight_map}

  with open(os.path.join(SCREAMING_SNAKE_CASE	,	SCREAMING_SNAKE_CASE						)	,	"w"	,	encoding="utf-8"						) as f:
    a__     : Tuple  =json.dumps(SCREAMING_SNAKE_CASE	,	indent=2	,	sort_keys=SCREAMING_SNAKE_CASE						) + "\n"
    f.write(SCREAMING_SNAKE_CASE						)

  return metadata, index


if __name__ == "__main__":
     UpperCAmelCase     :     str							=	argparse.ArgumentParser()
     # Required parameters
     parser.add_argument(
         """--nllb_moe_checkpoint_path""",
         default="""/home/arthur_huggingface_co/fairseq/weights/checkpoints/model_moe_54b/checkpoint_2_300000""",
         type=str,
         required=False,
         help="""Path to a directory containing a folder per layer. Follows the original Google format.""",
     )
     parser.add_argument("""--dtype""", default="""float32""", type=str, required=False, help="""dtype of the saved model""")
     parser.add_argument(
         """--pytorch_dump_folder_path""",
         default="""/home/arthur_huggingface_co/fairseq/weights/checkpoints/hf-converted-moe-54b""",
         type=str,
         required=False,
         help="""Path to the output pytorch model.""",
     )
     UpperCAmelCase     :     Tuple							=	parser.parse_args()
     UpperCAmelCase					,     UpperCAmelCase     :     Tuple							=	shard_on_the_fly(
         args.nllb_moe_checkpoint_path,
         args.pytorch_dump_folder_path,
         128,
         args.dtype,
     )

     UpperCAmelCase     :     Optional[int]							=	NllbMoeConfig.from_pretrained(
         """facebook/nllb-200-3.3B""", encoder_sparse_step=4, decoder_sparse_step=4, num_experts=128
     )
     config.save_pretrained(args.pytorch_dump_folder_path)
     UpperCAmelCase     :     Dict							=	NllbMoeModel.from_pretrained(args.pytorch_dump_folder_path)
     print("""Done""")
     model.save_pretrained(args.pytorch_dump_folder_path)













import unittest

from transformers import AutoTokenizer, NystromformerConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
     import torch

     from transformers import (
         NystromformerForMaskedLM,
         NystromformerForMultipleChoice,
         NystromformerForQuestionAnswering,
         NystromformerForSequenceClassification,
         NystromformerForTokenClassification,
         NystromformerModel,
     )
     from transformers.models.nystromformer.modeling_nystromformer import NYSTROMFORMER_PRETRAINED_MODEL_ARCHIVE_LIST







class     __lowerCAmelCase	:







      def __init__(   self		,				lowerCAmelCase__		,				lowerCAmelCase__=1_3		,				lowerCAmelCase__=7		,				lowerCAmelCase__=True		,				lowerCAmelCase__=True		,				lowerCAmelCase__=True		,				lowerCAmelCase__=True		,				lowerCAmelCase__=9_9		,				lowerCAmelCase__=3_2		,				lowerCAmelCase__=5		,				lowerCAmelCase__=4		,				lowerCAmelCase__=3_7		,				lowerCAmelCase__="gelu"		,				lowerCAmelCase__=0.1		,				lowerCAmelCase__=0.1		,				lowerCAmelCase__=5_1_2		,				lowerCAmelCase__=1_6		,				lowerCAmelCase__=2		,				lowerCAmelCase__=0.02		,				lowerCAmelCase__=3		,				lowerCAmelCase__=4		,				lowerCAmelCase__=None		,				)       ->						List[Any]:



        '''simple docstring'''




        a__     : Optional[Any]  =parent
        a__     : Tuple  =batch_size
        a__     : List[Any]  =seq_length
        a__     : Dict  =is_training
        a__     : Any  =use_input_mask
        a__     : int  =use_token_type_ids
        a__     : Optional[Any]  =use_labels
        a__     : Optional[Any]  =vocab_size
        a__     : List[str]  =hidden_size
        a__     : int  =num_hidden_layers
        a__     : Tuple  =num_attention_heads
        a__     : Union[str, Any]  =intermediate_size
        a__     : Optional[int]  =hidden_act
        a__     : int  =hidden_dropout_prob
        a__     : Union[str, Any]  =attention_probs_dropout_prob
        a__     : List[Any]  =max_position_embeddings
        a__     : str  =type_vocab_size
        a__     : Optional[Any]  =type_sequence_label_size
        a__     : Union[str, Any]  =initializer_range
        a__     : List[Any]  =num_labels
        a__     : str  =num_choices
        a__     : int  =scope







      def        _lowercase  (   self							)       ->						int:



        '''simple docstring'''




        a__     : List[Any]  =ids_tensor([self.batch_size, self.seq_length]		,				self.vocab_size							)

        a__     : str  =None
        if self.use_input_mask:
          a__     : List[Any]  =random_attention_mask([self.batch_size, self.seq_length]							)

        a__     : str  =None
        if self.use_token_type_ids:
          a__     : List[str]  =ids_tensor([self.batch_size, self.seq_length]		,				self.type_vocab_size							)

        a__     : Dict  =None
        a__     : str  =None
        a__     : str  =None
        if self.use_labels:
          a__     : List[str]  =ids_tensor([self.batch_size]		,				self.type_sequence_label_size							)
          a__     : Tuple  =ids_tensor([self.batch_size, self.seq_length]		,				self.num_labels							)
          a__     : Dict  =ids_tensor([self.batch_size]		,				self.num_choices							)

        a__     : Tuple  =self.get_config()

        return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels







      def        _lowercase  (   self							)       ->						Tuple:



        '''simple docstring'''




        return NystromformerConfig(
            vocab_size=self.vocab_size		,				hidden_size=self.hidden_size		,				num_hidden_layers=self.num_hidden_layers		,				num_attention_heads=self.num_attention_heads		,				intermediate_size=self.intermediate_size		,				hidden_act=self.hidden_act		,				hidden_dropout_prob=self.hidden_dropout_prob		,				attention_probs_dropout_prob=self.attention_probs_dropout_prob		,				max_position_embeddings=self.max_position_embeddings		,				type_vocab_size=self.type_vocab_size		,				is_decoder=lowerCAmelCase__		,				initializer_range=self.initializer_range		,				)







      def        _lowercase  (   self		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__							)       ->						List[Any]:



        '''simple docstring'''




        a__     : Tuple  =NystromformerModel(config=lowerCAmelCase__							)
        model.to(lowerCAmelCase__							)
        model.eval()
        a__     : Optional[Any]  =model(lowerCAmelCase__		,				attention_mask=lowerCAmelCase__		,				token_type_ids=lowerCAmelCase__							)
        a__     : str  =model(lowerCAmelCase__		,				token_type_ids=lowerCAmelCase__							)
        a__     : Optional[int]  =model(lowerCAmelCase__							)
        self.parent.assertEqual(result.last_hidden_state.shape		,				(self.batch_size, self.seq_length, self.hidden_size)							)







      def        _lowercase  (   self		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__							)       ->						int:



        '''simple docstring'''




        a__     : int  =NystromformerForMaskedLM(config=lowerCAmelCase__							)
        model.to(lowerCAmelCase__							)
        model.eval()
        a__     : Dict  =model(lowerCAmelCase__		,				attention_mask=lowerCAmelCase__		,				token_type_ids=lowerCAmelCase__		,				labels=lowerCAmelCase__							)
        self.parent.assertEqual(result.logits.shape		,				(self.batch_size, self.seq_length, self.vocab_size)							)







      def        _lowercase  (   self		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__							)       ->						Any:



        '''simple docstring'''




        a__     : Optional[int]  =NystromformerForQuestionAnswering(config=lowerCAmelCase__							)
        model.to(lowerCAmelCase__							)
        model.eval()
        a__     : str  =model(
            lowerCAmelCase__		,				attention_mask=lowerCAmelCase__		,				token_type_ids=lowerCAmelCase__		,				start_positions=lowerCAmelCase__		,				end_positions=lowerCAmelCase__		,				)
        self.parent.assertEqual(result.start_logits.shape		,				(self.batch_size, self.seq_length)							)
        self.parent.assertEqual(result.end_logits.shape		,				(self.batch_size, self.seq_length)							)







      def        _lowercase  (   self		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__							)       ->						Optional[int]:



        '''simple docstring'''




        a__     : Optional[Any]  =self.num_labels
        a__     : Dict  =NystromformerForSequenceClassification(lowerCAmelCase__							)
        model.to(lowerCAmelCase__							)
        model.eval()
        a__     : List[str]  =model(lowerCAmelCase__		,				attention_mask=lowerCAmelCase__		,				token_type_ids=lowerCAmelCase__		,				labels=lowerCAmelCase__							)
        self.parent.assertEqual(result.logits.shape		,				(self.batch_size, self.num_labels)							)







      def        _lowercase  (   self		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__							)       ->						List[str]:



        '''simple docstring'''




        a__     : Tuple  =self.num_labels
        a__     : List[str]  =NystromformerForTokenClassification(config=lowerCAmelCase__							)
        model.to(lowerCAmelCase__							)
        model.eval()
        a__     : List[Any]  =model(lowerCAmelCase__		,				attention_mask=lowerCAmelCase__		,				token_type_ids=lowerCAmelCase__		,				labels=lowerCAmelCase__							)
        self.parent.assertEqual(result.logits.shape		,				(self.batch_size, self.seq_length, self.num_labels)							)







      def        _lowercase  (   self		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__							)       ->						Tuple:



        '''simple docstring'''




        a__     : List[Any]  =self.num_choices
        a__     : Optional[Any]  =NystromformerForMultipleChoice(config=lowerCAmelCase__							)
        model.to(lowerCAmelCase__							)
        model.eval()
        a__     : List[str]  =input_ids.unsqueeze(1							).expand(-1		,				self.num_choices		,				-1							).contiguous()
        a__     : List[Any]  =token_type_ids.unsqueeze(1							).expand(-1		,				self.num_choices		,				-1							).contiguous()
        a__     : List[Any]  =input_mask.unsqueeze(1							).expand(-1		,				self.num_choices		,				-1							).contiguous()
        a__     : Dict  =model(
            lowerCAmelCase__		,				attention_mask=lowerCAmelCase__		,				token_type_ids=lowerCAmelCase__		,				labels=lowerCAmelCase__		,				)
        self.parent.assertEqual(result.logits.shape		,				(self.batch_size, self.num_choices)							)







      def        _lowercase  (   self							)       ->						Tuple:



        '''simple docstring'''




        a__     : Optional[Any]  =self.prepare_config_and_inputs()
        (
            (
            a__
        ) ,							(
            a__
        ) ,							(
            a__
        ) ,							(
            a__
        ) ,							(
            a__
        ) ,							(
            a__
        ) ,							(
            a__
        ) ,							
        )     : List[str]  =config_and_inputs
        a__     : str  ={"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
        return config, inputs_dict







@require_torch
class     __lowerCAmelCase	(   UpperCamelCase__						,    UpperCamelCase__						,    unittest.TestCase):
      _lowercase    : int            =      (
          (
              NystromformerModel,
              NystromformerForMaskedLM,
              NystromformerForMultipleChoice,
              NystromformerForQuestionAnswering,
              NystromformerForSequenceClassification,
              NystromformerForTokenClassification,
          )
          if is_torch_available()
          else ()
      )
      _lowercase    : Union[str, Any]            =      (
          {
              """feature-extraction""": NystromformerModel,
              """fill-mask""": NystromformerForMaskedLM,
              """question-answering""": NystromformerForQuestionAnswering,
              """text-classification""": NystromformerForSequenceClassification,
              """token-classification""": NystromformerForTokenClassification,
              """zero-shot""": NystromformerForSequenceClassification,
          }
          if is_torch_available()
          else {}
      )
      _lowercase    : Union[str, Any]            =      False
      _lowercase    : Union[str, Any]            =      False







      def        _lowercase  (   self							)       ->						List[str]:



        '''simple docstring'''




        a__     : Optional[int]  =NystromformerModelTester(self							)
        a__     : Optional[int]  =ConfigTester(self		,				config_class=lowerCAmelCase__		,				hidden_size=3_7							)







      def        _lowercase  (   self							)       ->						int:



        '''simple docstring'''




        self.config_tester.run_common_tests()







      def        _lowercase  (   self							)       ->						Dict:



        '''simple docstring'''




        a__     : List[str]  =self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*lowerCAmelCase__							)







      def        _lowercase  (   self							)       ->						Optional[Any]:



        '''simple docstring'''




        a__     : Tuple  =self.model_tester.prepare_config_and_inputs()
        for type in ["absolute", "relative_key", "relative_key_query"]:
          a__     : int  =type
          self.model_tester.create_and_check_model(*lowerCAmelCase__							)







      def        _lowercase  (   self							)       ->						Optional[Any]:



        '''simple docstring'''




        a__     : Optional[Any]  =self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_masked_lm(*lowerCAmelCase__							)







      def        _lowercase  (   self							)       ->						Dict:



        '''simple docstring'''




        a__     : str  =self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_multiple_choice(*lowerCAmelCase__							)







      def        _lowercase  (   self							)       ->						List[str]:



        '''simple docstring'''




        a__     : Tuple  =self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_question_answering(*lowerCAmelCase__							)







      def        _lowercase  (   self							)       ->						Union[str, Any]:



        '''simple docstring'''




        a__     : Tuple  =self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_sequence_classification(*lowerCAmelCase__							)







      def        _lowercase  (   self							)       ->						Optional[Any]:



        '''simple docstring'''




        a__     : int  =self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_token_classification(*lowerCAmelCase__							)







      @slow
      def        _lowercase  (   self							)       ->						str:



        '''simple docstring'''




        for model_name in NYSTROMFORMER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
          a__     : int  =NystromformerModel.from_pretrained(lowerCAmelCase__							)
          self.assertIsNotNone(lowerCAmelCase__							)







@require_torch
class     __lowerCAmelCase	(   unittest.TestCase):







      @slow
      def        _lowercase  (   self							)       ->						str:



        '''simple docstring'''




        a__     : str  =NystromformerModel.from_pretrained("uw-madison/nystromformer-512"							)
        a__     : int  =torch.tensor([[0, 1, 2, 3, 4, 5]]							)

        with torch.no_grad():
          a__     : Tuple  =model(lowerCAmelCase__							)[0]

        a__     : List[str]  =torch.Size((1, 6, 7_6_8)							)
        self.assertEqual(output.shape		,				lowerCAmelCase__							)

        a__     : int  =torch.tensor(
            [[[-0.45_32, -0.09_36, 0.51_37], [-0.26_76, 0.06_28, 0.61_86], [-0.36_29, -0.17_26, 0.47_16]]]							)

        self.assertTrue(torch.allclose(output[:, :3, :3]		,				lowerCAmelCase__		,				atol=1E-4							)							)







      @slow
      def        _lowercase  (   self							)       ->						Optional[Any]:



        '''simple docstring'''




        a__     : Union[str, Any]  ="the [MASK] of Belgium is Brussels"

        a__     : str  =AutoTokenizer.from_pretrained("uw-madison/nystromformer-512"							)
        a__     : int  =NystromformerForMaskedLM.from_pretrained("uw-madison/nystromformer-512"							)

        a__     : List[Any]  =tokenizer(lowerCAmelCase__		,				return_tensors="pt"							)

        with torch.no_grad():
          a__     : str  =model(encoding.input_ids							).logits

        a__     : List[str]  =token_logits[:, 2, :].argmax(-1							)[0]

        self.assertEqual(tokenizer.decode(lowerCAmelCase__							)		,				"capital"							)






