


"""simple docstring"""



import argparse
import hashlib
import os
import urllib
import warnings

import torch
from torch import nn
from tqdm import tqdm

from transformers import WhisperConfig, WhisperForConditionalGeneration


__snake_case             =   {
    """tiny.en""": """https://openaipublic.azureedge.net/main/whisper/models/d3dd57d32accea0b295c96e26691aa14d8822fac7d9d27d5dc00b4ca2826dd03/tiny.en.pt""",
    """tiny""": """https://openaipublic.azureedge.net/main/whisper/models/65147644a518d12f04e32d6f3b26facc3f8dd46e5390956a9424a650c0ce22b9/tiny.pt""",
    """base.en""": """https://openaipublic.azureedge.net/main/whisper/models/25a8566e1d0c1e2231d1c762132cd20e0f96a85d16145c3a00adf5d1ac670ead/base.en.pt""",
    """base""": """https://openaipublic.azureedge.net/main/whisper/models/ed3a0b6b1c0edf879ad9b11b1af5a0e6ab5db9205f891f668f8b0e6c6326e34e/base.pt""",
    """small.en""": """https://openaipublic.azureedge.net/main/whisper/models/f953ad0fd29cacd07d5a9eda5624af0f6bcf2258be67c92b79389873d91e0872/small.en.pt""",
    """small""": """https://openaipublic.azureedge.net/main/whisper/models/9ecf779972d90ba49c06d968637d720dd632c55bbf19d441fb42bf17a411e794/small.pt""",
    """medium.en""": """https://openaipublic.azureedge.net/main/whisper/models/d7440d1dc186f76616474e0ff0b3b6b879abc9d1a4926b7adfa41db2d497ab4f/medium.en.pt""",
    """medium""": """https://openaipublic.azureedge.net/main/whisper/models/345ae4da62f9b3d59415adc60127b97c714f32e89e936602e85993674d08dcb1/medium.pt""",
    """large""": """https://openaipublic.azureedge.net/main/whisper/models/e4b87e7e0bf463eb8e6956e646f1e277e901512310def2c24bf0e11bd3c28e9a/large.pt""",
    """large-v2""": """https://openaipublic.azureedge.net/main/whisper/models/81f7c96c852ee8fc832187b0132e569d6c3065a3252ed18e56effd0b6a73e524/large-v2.pt""",
}







def   __lowerCAmelCase						(						lowercase  :  Tuple			)     ->						List[Any]:


    """simple docstring"""





    snake_case   :				Optional[int]				 =   ["layers", "blocks"]
    for k in ignore_keys:
        state_dict.pop(lowercase						,							lowercase			)


__snake_case             =   {
    """blocks""": """layers""",
    """mlp.0""": """fc1""",
    """mlp.2""": """fc2""",
    """mlp_ln""": """final_layer_norm""",
    """.attn.query""": """.self_attn.q_proj""",
    """.attn.key""": """.self_attn.k_proj""",
    """.attn.value""": """.self_attn.v_proj""",
    """.attn_ln""": """.self_attn_layer_norm""",
    """.attn.out""": """.self_attn.out_proj""",
    """.cross_attn.query""": """.encoder_attn.q_proj""",
    """.cross_attn.key""": """.encoder_attn.k_proj""",
    """.cross_attn.value""": """.encoder_attn.v_proj""",
    """.cross_attn_ln""": """.encoder_attn_layer_norm""",
    """.cross_attn.out""": """.encoder_attn.out_proj""",
    """decoder.ln.""": """decoder.layer_norm.""",
    """encoder.ln.""": """encoder.layer_norm.""",
    """token_embedding""": """embed_tokens""",
    """encoder.positional_embedding""": """encoder.embed_positions.weight""",
    """decoder.positional_embedding""": """decoder.embed_positions.weight""",
    """ln_post""": """layer_norm""",
}







def   __lowerCAmelCase						(						lowercase  :  Any			)     ->						Union[str, Any]:


    """simple docstring"""





    snake_case   :				Union[str, Any]				 =   list(s_dict.keys()			)
    for key in keys:
        snake_case   :				List[str]				 =   key
        for k, v in WHISPER_MAPPING.items():
            if k in key:
                snake_case   :				int				 =   new_key.replace(lowercase						,							lowercase			)

        print(F'{key} -> {new_key}'			)

        snake_case   :				str				 =   s_dict.pop(lowercase			)
    return s_dict







def   __lowerCAmelCase						(						lowercase  :  Union[str, Any]			)     ->						Union[str, Any]:


    """simple docstring"""





    snake_case ,snake_case   :				List[str]				 =   emb.weight.shape
    snake_case   :				Any				 =   nn.Linear(lowercase						,							lowercase						,							bias=lowercase			)
    snake_case   :				Dict				 =   emb.weight.data
    return lin_layer







def   __lowerCAmelCase						(						lowercase  :  str						,							lowercase  :  str			)     ->						bytes:


    """simple docstring"""





    os.makedirs(lowercase						,							exist_ok=lowercase			)
    snake_case   :				Any				 =   os.path.basename(lowercase			)

    snake_case   :				List[Any]				 =   url.split("/"			)[-2]
    snake_case   :				Union[str, Any]				 =   os.path.join(lowercase						,							lowercase			)

    if os.path.exists(lowercase			) and not os.path.isfile(lowercase			):
        raise RuntimeError(F'{download_target} exists and is not a regular file'			)

    if os.path.isfile(lowercase			):
        snake_case   :				str				 =   open(lowercase						,							"rb"			).read()
        if hashlib.shaaaa(lowercase			).hexdigest() == expected_shaaaa:
            return model_bytes
        else:
            warnings.warn(F'{download_target} exists, but the SHA256 checksum does not match; re-downloading the file'			)

    with urllib.request.urlopen(lowercase			) as source, open(lowercase						,							"wb"			) as output:
        with tqdm(
            total=int(source.info().get("Content-Length"			)			)						,							ncols=80						,							unit="iB"						,							unit_scale=lowercase						,							unit_divisor=1024			) as loop:
            while True:
                snake_case   :				Optional[Any]				 =   source.read(8192			)
                if not buffer:
                    break

                output.write(lowercase			)
                loop.update(len(lowercase			)			)

    snake_case   :				Optional[int]				 =   open(lowercase						,							"rb"			).read()
    if hashlib.shaaaa(lowercase			).hexdigest() != expected_shaaaa:
        raise RuntimeError(
            "Model has been downloaded but the SHA256 checksum does not not match. Please retry loading the model."			)

    return model_bytes







def   __lowerCAmelCase						(						lowercase  :  List[str]						,							lowercase  :  Dict			)     ->						Tuple:


    """simple docstring"""





    if ".pt" not in checkpoint_path:
        snake_case   :				Dict				 =   _download(_MODELS[checkpoint_path]			)
    else:
        snake_case   :				str				 =   torch.load(lowercase						,							map_location="cpu"			)
    snake_case   :				List[str]				 =   original_checkpoint["dims"]
    snake_case   :				int				 =   original_checkpoint["model_state_dict"]
    snake_case   :				Any				 =   state_dict["decoder.token_embedding.weight"]
    remove_ignore_keys_(lowercase			)
    rename_keys(lowercase			)
    snake_case   :				Any				 =   True
    snake_case   :				int				 =   state_dict["decoder.layers.0.fc1.weight"].shape[0]

    snake_case   :				List[str]				 =   WhisperConfig(
        vocab_size=dimensions["n_vocab"]						,							encoder_ffn_dim=lowercase						,							decoder_ffn_dim=lowercase						,							num_mel_bins=dimensions["n_mels"]						,							d_model=dimensions["n_audio_state"]						,							max_target_positions=dimensions["n_text_ctx"]						,							encoder_layers=dimensions["n_audio_layer"]						,							encoder_attention_heads=dimensions["n_audio_head"]						,							decoder_layers=dimensions["n_text_layer"]						,							decoder_attention_heads=dimensions["n_text_state"]						,							max_source_positions=dimensions["n_audio_ctx"]						,							)

    snake_case   :				List[str]				 =   WhisperForConditionalGeneration(lowercase			)
    snake_case ,snake_case   :				Dict				 =   model.model.load_state_dict(lowercase						,							strict=lowercase			)
    if len(lowercase			) > 0 and not set(lowercase			) <= {
        "encoder.embed_positions.weights",
        "decoder.embed_positions.weights",
    }:
        raise ValueError(
            "Only `encoder.embed_positions.weights` and `decoder.embed_positions.weights`  are allowed to be missing,"
            F' but all the following weights are missing {missing}'			)

    if tie_embeds:
        snake_case   :				str				 =   make_linear_from_emb(model.model.decoder.embed_tokens			)
    else:
        snake_case   :				str				 =   proj_out_weights

    model.save_pretrained(lowercase			)


if __name__ == "__main__":
  __snake_case             =   argparse.ArgumentParser()
  # # Required parameters
  parser.add_argument("""--checkpoint_path""", type=str, help="""Patht to the downloaded checkpoints""")
  parser.add_argument("""--pytorch_dump_folder_path""", default=None, type=str, help="""Path to the output PyTorch model.""")
  __snake_case             =   parser.parse_args()

  convert_openai_whisper_to_tfms(args.checkpoint_path, args.pytorch_dump_folder_path)









"""simple docstring"""



import argparse
import logging
import pickle
from collections import Counter


logging.basicConfig(
    format="""%(asctime)s - %(levelname)s - %(name)s - %(message)s""", datefmt="""%m/%d/%Y %H:%M:%S""", level=logging.INFO
)
__snake_case             =   logging.getLogger(__name__)

if __name__ == "__main__":
  __snake_case             =   argparse.ArgumentParser(
      description="""Token Counts for smoothing the masking probabilities in MLM (cf XLM/word2vec)"""
  )
  parser.add_argument(
      """--data_file""", type=str, default="""data/dump.bert-base-uncased.pickle""", help="""The binarized dataset."""
  )
  parser.add_argument(
      """--token_counts_dump""", type=str, default="""data/token_counts.bert-base-uncased.pickle""", help="""The dump file."""
  )
  parser.add_argument("""--vocab_size""", default=30522, type=int)
  __snake_case             =   parser.parse_args()

  logger.info(F'''Loading data from {args.data_file}''')
  with open(args.data_file, """rb""") as fp:
    __snake_case             =   pickle.load(fp)

  logger.info("""Counting occurrences for MLM.""")
  __snake_case             =   Counter()
  for tk_ids in data:
    counter.update(tk_ids)
  __snake_case             =   [0] * args.vocab_size
  for k, v in counter.items():
    __snake_case             =   v

  logger.info(F'''Dump to {args.token_counts_dump}''')
  with open(args.token_counts_dump, """wb""") as handle:
    pickle.dump(counts, handle, protocol=pickle.HIGHEST_PROTOCOL)





