
import argparse
import json
import os
import re

import torch

from transformers import BloomConfig, BloomModel
from transformers.file_utils import CONFIG_NAME, WEIGHTS_NAME
from transformers.utils import logging


logging.set_verbosity_info()

_A   =			[
    "word_embeddings_layernorm.weight",
    "word_embeddings_layernorm.bias",
    "input_layernorm.weight",
    "input_layernorm.bias",
    "post_attention_layernorm.weight",
    "post_attention_layernorm.bias",
    "self_attention.dense.bias",
    "mlp.dense_4h_to_h.bias",
    "ln_f.weight",
    "ln_f.bias",
]

_A   =			[
    "mlp.dense_4h_to_h.weight",
    "self_attention.dense.weight",
]



def 						lowercase_      (       A__			,		A__		)							->							Dict:
    """simple docstring"""

    snake_case								= {
        "word_embeddings.weight": "word_embeddings.weight",
        "word_embeddings.norm.weight": "word_embeddings_layernorm.weight",
        "word_embeddings.norm.bias": "word_embeddings_layernorm.bias",
        "weight": "ln_f.weight",
        "bias": "ln_f.bias",
    }

    if key in layer_rename_map:
        return layer_rename_map[key]

    # Handle transformer blocks
    snake_case								= int(re.match(r".*layer_(\d*).*"			,		A__		)[1]		)
    layer_number -= 3
    return F'h.{layer_number}.' + key



def 						lowercase_      (       A__		)							->							List[str]:
    """simple docstring"""

    if dtype == torch.bool:
        return 1 / 8
    snake_case								= re.search(r"[^\d](\d+)$"			,		str(A__		)		)
    if bit_search is None:
        raise ValueError(F'`dtype` is not a valid dtype: {dtype}.'		)
    snake_case								= int(bit_search.groups()[0]		)
    return bit_size // 8



def 						lowercase_      (       A__			,		A__			,		A__			,		A__			,		A__		)							->							Optional[int]:
    """simple docstring"""

    if bloom_config_file == "":
        snake_case								= BloomConfig()
    else:
        snake_case								= BloomConfig.from_json_file(A__		)

    if shard_model:
        snake_case								= os.listdir(A__		)
        snake_case								= sorted(filter(lambda A__		: s.startswith("layer"		) and "model_00" in s			,		A__		)		)

        snake_case								= {"weight_map": {}, "metadata": {}}
        snake_case								= 0

        snake_case								= None

        snake_case								= BloomConfig()

        for j, file in enumerate(A__		):
            print("Processing file: {}".format(A__		)		)
            snake_case								= None

            for i in range(A__		):
                # load all TP files
                snake_case								= file.replace("model_00"			,		F'model_0{i}'		)
                snake_case								= torch.load(os.path.join(A__			,		A__		)			,		map_location="cpu"		)

                # Rename keys in the transformers names
                snake_case								= list(temp.keys()		)
                for key in keys:
                    snake_case								= temp.pop(A__		)

                if tensors is None:
                    snake_case								= temp
                else:
                    for key in tensors.keys():
                        if any(key.endswith(A__		) for end in WEIGHTS_TO_AVERAGE_ENDSWITH		):
                            # We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
                            tensors[key] += temp[key]
                        else:
                            # Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
                            snake_case								= 1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN		) else 0
                            # We concatenate these weights accross TP ranks
                            snake_case								= torch.cat([tensors[key], temp[key]]			,		dim=A__		)

            # Divide by the number of TP the weights we want to average
            for key in tensors.keys():
                if any(key.endswith(A__		) for end in WEIGHTS_TO_AVERAGE_ENDSWITH		):
                    snake_case								= tensors[key] / pretraining_tp
            torch.save(
                A__			,		os.path.join(
                    A__			,		"pytorch_model_{}-of-{}.bin".format(str(j + 1		).zfill(5		)			,		str(len(A__		)		).zfill(5		)		)			,		)			,		)

            for key in tensors.keys():
                snake_case								= tensors[key]
                total_size += value.numel() * get_dtype_size(value.dtype		)
                if key not in index_dict["weight_map"]:
                    snake_case								= "pytorch_model_{}-of-{}.bin".format(
                        str(j + 1		).zfill(5		)			,		str(len(A__		)		).zfill(5		)		)

        snake_case								= BloomConfig()
        snake_case								= pytorch_dump_folder_path + "/" + CONFIG_NAME
        snake_case								= total_size
        with open(A__			,		"w"			,		encoding="utf-8"		) as f:
            f.write(config.to_json_string()		)
        with open(os.path.join(A__			,		WEIGHTS_NAME + ".index.json"		)			,		"w"			,		encoding="utf-8"		) as f:
            snake_case								= json.dumps(A__			,		indent=2			,		sort_keys=A__		) + "\n"
            f.write(A__		)
    else:
        snake_case								= BloomModel(A__		)

        snake_case								= os.listdir(A__		)
        snake_case								= sorted(filter(lambda A__		: s.startswith("layer"		) and "model_00" in s			,		A__		)		)

        snake_case								= None
        for i, file in enumerate(A__		):
            snake_case								= None
            for i in range(A__		):
                # load all TP files
                snake_case								= file.replace("model_00"			,		F'model_0{i}'		)
                snake_case								= torch.load(os.path.join(A__			,		A__		)			,		map_location="cpu"		)

                # Rename keys in the transformers names
                snake_case								= list(temp.keys()		)
                for key in keys:
                    snake_case								= temp.pop(A__		)

                if tensors is None:
                    snake_case								= temp
                else:
                    for key in tensors.keys():
                        # We average (sum and then divide) some weights accross TP ranks (see https://github.com/bigscience-workshop/Megatron-DeepSpeed/blob/olruwase/sync_layer_norms/megatron/training.py#L425)
                        if any(key.endswith(A__		) for end in WEIGHTS_TO_AVERAGE_ENDSWITH		):
                            tensors[key] += temp[key]
                        else:
                            # Some weights are RowParallelLinear in Megatron-Deepspeed, others are ColumnParallel
                            snake_case								= 1 if any(text in key for text in WEIGHTS_WITH_ROW_PARALLELISM_CONTAIN		) else 0
                            # We concatenate these weights accross TP ranks
                            snake_case								= torch.cat([tensors[key], temp[key]]			,		dim=A__		)

            # Divide by the number of TP the weights we want to average
            for key in tensors.keys():
                if any(key.endswith(A__		) for end in WEIGHTS_TO_AVERAGE_ENDSWITH		):
                    snake_case								= tensors[key] / pretraining_tp

            snake_case								= model.load_state_dict(A__			,		strict=A__		)
            assert not other_keys.unexpected_keys, F'The keys {other_keys.unexpected_keys} are unexpected'
            if missing_keys is None:
                snake_case								= set(other_keys.missing_keys		)
            else:
                snake_case								= missing_keys.intersection(set(other_keys.missing_keys		)		)

        assert not missing_keys, F'The keys {missing_keys} are missing'

        # Save pytorch-model
        os.makedirs(A__			,		exist_ok=A__		)
        snake_case								= pytorch_dump_folder_path + "/" + WEIGHTS_NAME
        snake_case								= pytorch_dump_folder_path + "/" + CONFIG_NAME
        print(F'Save PyTorch model to {pytorch_weights_dump_path} with dtype {config.torch_dtype}'		)
        if config.torch_dtype is not None:
            snake_case								= model.to(config.torch_dtype		)
        torch.save(model.state_dict()			,		A__		)
        print(F'Save configuration file to {pytorch_config_dump_path}'		)
        with open(A__			,		"w"			,		encoding="utf-8"		) as f:
            f.write(config.to_json_string()		)


if __name__ == "__main__":
   _A   =			argparse.ArgumentParser()
   # Required parameters
   parser.add_argument(
       "--bloom_checkpoint_path",
       default=None,
       type=str,
       required=True,
       help="Path to the Megatron-LM checkpoint path.",
   )
   parser.add_argument(
       "--pytorch_dump_folder_path", default=None, type=str, required=True, help="Path to the output PyTorch model."
   )
   parser.add_argument(
       "--bloom_config_file",
       default="",
       type=str,
       help=(
           "An optional config json file corresponding to the pre-trained model. \n"
           "This specifies the model architecture."
       ),
   )
   parser.add_argument(
       "--shard_model",
       action="store_true",
       help="An optional setting to shard the output model \nThis enables sharding the converted checkpoint",
   )
   parser.add_argument(
       "--pretraining_tp",
       default=4,
       type=int,
       help="Pretraining TP rank that has been used when training the model in Megatron-LM \n",
   )
   _A   =			parser.parse_args()
   convert_bloom_checkpoint_to_pytorch(
       args.bloom_checkpoint_path,
       args.bloom_config_file,
       args.pytorch_dump_folder_path,
       args.shard_model,
       args.pretraining_tp,
   )



from collections import defaultdict







class 					lowerCamelCase		:





     def __init__(self				:   Tuple    ,   _A				:   Optional[int]    ,   _A				:   List[str]       )						->						Union[str, Any]:
         snake_case								= total  # total no of tasks (N)

         # DP table will have a dimension of (2^M)*N
         # initially all values are set to -1
         snake_case								= [
             [-1 for i in range(total + 1       )] for j in range(2 ** len(_A       )       )
         ]

         snake_case								= defaultdict(_A       )  # stores the list of persons for each task

         # final_mask is used to check if all persons are included by setting all bits
         # to 1
         snake_case								= (1 << len(_A       )) - 1





     def 		UpperCAmelCase(self				:   str    ,   _A				:   Optional[Any]    ,   _A				:   List[Any]       )						->						str:
         # if mask == self.finalmask all persons are distributed tasks, return 1
         if mask == self.final_mask:
             return 1

         # if not everyone gets the task and no more tasks are available, return 0
         if task_no > self.total_tasks:
             return 0

         # if case already considered
         if self.dp[mask][task_no] != -1:
             return self.dp[mask][task_no]

         # Number of ways when we don't this task in the arrangement
         snake_case								= self.count_ways_until(_A    ,   task_no + 1       )

         # now assign the tasks one by one to all possible persons and recursively
         # assign for the remaining tasks.
         if task_no in self.task:
             for p in self.task[task_no]:
                 # if p is already given a task
                 if mask & (1 << p):
                     continue

                 # assign this task to p and change the mask value. And recursively
                 # assign tasks with the new mask value.
                 total_ways_util += self.count_ways_until(mask | (1 << p)    ,   task_no + 1       )

        # save the value.
         snake_case								= total_ways_util

         return self.dp[mask][task_no]





     def 		UpperCAmelCase(self				:   Any    ,   _A				:   Dict       )						->						Optional[Any]:
         # Store the list of persons for each task
         for i in range(len(_A       )       ):
             for j in task_performed[i]:
                 self.task[j].append(_A       )

        # call the function to fill the DP table, final answer is stored in dp[0][1]
         return self.count_ways_until(0    ,   1       )


if __name__ == "__main__":
   _A   =			5  # total no of tasks (the value of N)

   # the list of tasks that can be done by M persons.
   _A   =			[[1, 3, 4], [1, 2, 5], [3, 4]]
   print(
       AssignmentUsingBitmask(task_performed, total_tasks).count_no_of_ways(
           task_performed
       )
   )

