



'''simple docstring'''




import argparse
import hashlib
import os
import urllib
import warnings

import torch
from torch import nn
from tqdm import tqdm

from transformers import WhisperConfig, WhisperForConditionalGeneration


lowerCamelCase							:str       = {
    'tiny.en': 'https://openaipublic.azureedge.net/main/whisper/models/d3dd57d32accea0b295c96e26691aa14d8822fac7d9d27d5dc00b4ca2826dd03/tiny.en.pt',
    'tiny': 'https://openaipublic.azureedge.net/main/whisper/models/65147644a518d12f04e32d6f3b26facc3f8dd46e5390956a9424a650c0ce22b9/tiny.pt',
    'base.en': 'https://openaipublic.azureedge.net/main/whisper/models/25a8566e1d0c1e2231d1c762132cd20e0f96a85d16145c3a00adf5d1ac670ead/base.en.pt',
    'base': 'https://openaipublic.azureedge.net/main/whisper/models/ed3a0b6b1c0edf879ad9b11b1af5a0e6ab5db9205f891f668f8b0e6c6326e34e/base.pt',
    'small.en': 'https://openaipublic.azureedge.net/main/whisper/models/f953ad0fd29cacd07d5a9eda5624af0f6bcf2258be67c92b79389873d91e0872/small.en.pt',
    'small': 'https://openaipublic.azureedge.net/main/whisper/models/9ecf779972d90ba49c06d968637d720dd632c55bbf19d441fb42bf17a411e794/small.pt',
    'medium.en': 'https://openaipublic.azureedge.net/main/whisper/models/d7440d1dc186f76616474e0ff0b3b6b879abc9d1a4926b7adfa41db2d497ab4f/medium.en.pt',
    'medium': 'https://openaipublic.azureedge.net/main/whisper/models/345ae4da62f9b3d59415adc60127b97c714f32e89e936602e85993674d08dcb1/medium.pt',
    'large': 'https://openaipublic.azureedge.net/main/whisper/models/e4b87e7e0bf463eb8e6956e646f1e277e901512310def2c24bf0e11bd3c28e9a/large.pt',
    'large-v2': 'https://openaipublic.azureedge.net/main/whisper/models/81f7c96c852ee8fc832187b0132e569d6c3065a3252ed18e56effd0b6a73e524/large-v2.pt',
}



def      a      (			lowerCamelCase__   ):
			'''simple docstring'''





			A_     :						Dict     =					["""layers""", """blocks"""]
			for k in ignore_keys:
						state_dict.pop(lowercase__						,				lowercase__   )


lowerCamelCase							:Union[str, Any]       = {
    'blocks': 'layers',
    'mlp.0': 'fc1',
    'mlp.2': 'fc2',
    'mlp_ln': 'final_layer_norm',
    '.attn.query': '.self_attn.q_proj',
    '.attn.key': '.self_attn.k_proj',
    '.attn.value': '.self_attn.v_proj',
    '.attn_ln': '.self_attn_layer_norm',
    '.attn.out': '.self_attn.out_proj',
    '.cross_attn.query': '.encoder_attn.q_proj',
    '.cross_attn.key': '.encoder_attn.k_proj',
    '.cross_attn.value': '.encoder_attn.v_proj',
    '.cross_attn_ln': '.encoder_attn_layer_norm',
    '.cross_attn.out': '.encoder_attn.out_proj',
    'decoder.ln.': 'decoder.layer_norm.',
    'encoder.ln.': 'encoder.layer_norm.',
    'token_embedding': 'embed_tokens',
    'encoder.positional_embedding': 'encoder.embed_positions.weight',
    'decoder.positional_embedding': 'decoder.embed_positions.weight',
    'ln_post': 'layer_norm',
}



def      a      (			lowerCamelCase__   ):
			'''simple docstring'''





			A_     :						int     =					list(s_dict.keys()   )
			for key in keys:
						A_     :						Dict     =					key
						for k, v in WHISPER_MAPPING.items():
									if k in key:
												A_     :						Optional[Any]     =					new_key.replace(lowercase__						,				lowercase__   )

						print(f'{key} -> {new_key}'   )

						A_     :						List[Any]     =					s_dict.pop(lowercase__   )
			return s_dict



def      a      (			lowerCamelCase__   ):
			'''simple docstring'''





			A_,   A_     :						Optional[Any]     =					emb.weight.shape
			A_     :						List[str]     =					nn.Linear(lowercase__						,				lowercase__						,				bias=lowercase__   )
			A_     :						List[Any]     =					emb.weight.data
			return lin_layer



def      a      (			lowerCamelCase__						,				lowerCamelCase__   ):
			'''simple docstring'''





			os.makedirs(lowercase__						,				exist_ok=lowercase__   )
			A_     :						List[str]     =					os.path.basename(lowercase__   )

			A_     :						List[str]     =					url.split("""/"""   )[-2]
			A_     :						List[Any]     =					os.path.join(lowercase__						,				lowercase__   )

			if os.path.exists(lowercase__   ) and not os.path.isfile(lowercase__   ):
						raise RuntimeError(f'{download_target} exists and is not a regular file'   )

			if os.path.isfile(lowercase__   ):
						A_     :						Tuple     =					open(lowercase__						,				"""rb"""   ).read()
						if hashlib.shaaaa(lowercase__   ).hexdigest() == expected_shaaaa:
									return model_bytes
						else:
									warnings.warn(f'{download_target} exists, but the SHA256 checksum does not match; re-downloading the file'   )

			with urllib.request.urlopen(lowercase__   ) as source, open(lowercase__						,				"""wb"""   ) as output:
						with tqdm(
						    total=int(source.info().get("""Content-Length"""   )   )						,				ncols=80						,				unit="""iB"""						,				unit_scale=lowercase__						,				unit_divisor=10_24   ) as loop:
									while True:
												A_     :						List[Any]     =					source.read(81_92   )
												if not buffer:
															break

												output.write(lowercase__   )
												loop.update(len(lowercase__   )   )

			A_     :						Tuple     =					open(lowercase__						,				"""rb"""   ).read()
			if hashlib.shaaaa(lowercase__   ).hexdigest() != expected_shaaaa:
						raise RuntimeError(
						    """Model has been downloaded but the SHA256 checksum does not not match. Please retry loading the model."""   )

			return model_bytes



def      a      (			lowerCamelCase__						,				lowerCamelCase__   ):
			'''simple docstring'''





			if ".pt" not in checkpoint_path:
						A_     :						Optional[Any]     =					_download(_MODELS[checkpoint_path]   )
			else:
						A_     :						Dict     =					torch.load(lowercase__						,				map_location="""cpu"""   )
			A_     :						int     =					original_checkpoint["""dims"""]
			A_     :						List[Any]     =					original_checkpoint["""model_state_dict"""]
			A_     :						Dict     =					state_dict["""decoder.token_embedding.weight"""]
			remove_ignore_keys_(lowercase__   )
			rename_keys(lowercase__   )
			A_     :						Optional[Any]     =					True
			A_     :						List[Any]     =					state_dict["""decoder.layers.0.fc1.weight"""].shape[0]

			A_     :						List[Any]     =					WhisperConfig(
			    vocab_size=dimensions["""n_vocab"""]						,				encoder_ffn_dim=lowercase__						,				decoder_ffn_dim=lowercase__						,				num_mel_bins=dimensions["""n_mels"""]						,				d_model=dimensions["""n_audio_state"""]						,				max_target_positions=dimensions["""n_text_ctx"""]						,				encoder_layers=dimensions["""n_audio_layer"""]						,				encoder_attention_heads=dimensions["""n_audio_head"""]						,				decoder_layers=dimensions["""n_text_layer"""]						,				decoder_attention_heads=dimensions["""n_text_state"""]						,				max_source_positions=dimensions["""n_audio_ctx"""]						,				)

			A_     :						int     =					WhisperForConditionalGeneration(lowercase__   )
			A_,   A_     :						Optional[int]     =					model.model.load_state_dict(lowercase__						,				strict=lowercase__   )
			if len(lowercase__   ) > 0 and not set(lowercase__   ) <= {
			    "encoder.embed_positions.weights",
			    "decoder.embed_positions.weights",
			}:
						raise ValueError(
						    """Only `encoder.embed_positions.weights` and `decoder.embed_positions.weights`  are allowed to be missing,"""
						    f' but all the following weights are missing {missing}'   )

			if tie_embeds:
						A_     :						Dict     =					make_linear_from_emb(model.model.decoder.embed_tokens   )
			else:
						A_     :						Optional[Any]     =					proj_out_weights

			model.save_pretrained(lowercase__   )


if __name__ == "__main__":
	lowerCamelCase							:List[Any]       = argparse.ArgumentParser()
	# # Required parameters
	parser.add_argument('''--checkpoint_path''', type=str, help='''Patht to the downloaded checkpoints''')
	parser.add_argument('''--pytorch_dump_folder_path''', default=None, type=str, help='''Path to the output PyTorch model.''')
	lowerCamelCase							:Tuple       = parser.parse_args()

	convert_openai_whisper_to_tfms(args.checkpoint_path, args.pytorch_dump_folder_path)




'''simple docstring'''




import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


lowerCamelCase							:Union[str, Any]       = logging.get_logger(__name__)

lowerCamelCase							:Union[str, Any]       = {'''vocab_file''': '''spiece.model'''}

lowerCamelCase							:Tuple       = {
    '''vocab_file''': {
        '''TsinghuaAI/CPM-Generate''': '''https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/spiece.model''',
    }
}







class      _lowerCAmelCase							(      __UpperCAmelCase			):


	def __init__(self	,				lowercase	,				lowercase=False	,				lowercase=True	,				lowercase=False	,				lowercase="<s>"	,				lowercase="</s>"	,				lowercase="<unk>"	,				lowercase="<sep>"	,				lowercase="<pad>"	,				lowercase="<cls>"	,				lowercase="<mask>"	,				lowercase=["<eop>", "<eod>"]	,				lowercase = None	,				**lowercase	,				):
				A_     :						Optional[int]     =					AddedToken(lowercase	,				lstrip=lowercase	,				rstrip=lowercase							) if isinstance(lowercase	,				lowercase							) else mask_token

				A_     :						str     =					{} if sp_model_kwargs is None else sp_model_kwargs

				super().__init__(
				    do_lower_case=lowercase	,				remove_space=lowercase	,				keep_accents=lowercase	,				bos_token=lowercase	,				eos_token=lowercase	,				unk_token=lowercase	,				sep_token=lowercase	,				pad_token=lowercase	,				cls_token=lowercase	,				mask_token=lowercase	,				additional_special_tokens=lowercase	,				sp_model_kwargs=self.sp_model_kwargs	,				**lowercase	,				)

				A_     :						List[str]     =					3

				A_     :						List[str]     =					do_lower_case
				A_     :						Tuple     =					remove_space
				A_     :						Tuple     =					keep_accents
				A_     :						Tuple     =					vocab_file

				A_     :						int     =					spm.SentencePieceProcessor(**self.sp_model_kwargs							)
				self.sp_model.Load(lowercase							)

				try:
							import jieba
				except ModuleNotFoundError as error:
							raise error.__class__(
							    """You need to install jieba to use CpmTokenizer or CpmTokenizerFast. """
							    """See https://pypi.org/project/jieba/ for installation."""							)
				A_     :						Tuple     =					jieba
				A_     :						int     =					str.maketrans(""" \n"""	,				"""\u2582\u2583"""							)


	@property
	# Copied from transformers.models.xlnet.tokenization_xlnet.XLNetTokenizer.vocab_size
	def    _a     (self							):
				return len(self.sp_model							)


	def    _a     (self							):
				A_     :						str     =					{self.convert_ids_to_tokens(lowercase							): i for i in range(self.vocab_size							)}
				vocab.update(self.added_tokens_encoder							)
				return vocab


	def __getstate__(self							):
				A_     :						Optional[Any]     =					self.__dict__.copy()
				A_     :						Tuple     =					None
				return state


	def __setstate__(self	,				lowercase							):
				A_     :						Tuple     =					d

				# for backward compatibility
				if not hasattr(self	,				"""sp_model_kwargs"""							):
							A_     :						List[Any]     =					{}

				A_     :						int     =					spm.SentencePieceProcessor(**self.sp_model_kwargs							)
				self.sp_model.Load(self.vocab_file							)


	def    _a     (self	,				lowercase							):
				if self.remove_space:
							A_     :						Union[str, Any]     =					""" """.join(inputs.strip().split()							)
				else:
							A_     :						Optional[int]     =					inputs
				A_     :						List[str]     =					outputs.replace("""``"""	,				"""\""""							).replace("""''"""	,				"""\""""							)

				if not self.keep_accents:
							A_     :						str     =					unicodedata.normalize("""NFKD"""	,				lowercase							)
							A_     :						Union[str, Any]     =					"""""".join([c for c in outputs if not unicodedata.combining(lowercase							)]							)
				if self.do_lower_case:
							A_     :						int     =					outputs.lower()

				return outputs


	def    _a     (self	,				lowercase							):
				A_     :						Optional[int]     =					self.preprocess_text(lowercase							)
				A_     :						Dict     =					self.sp_model.encode(lowercase	,				out_type=lowercase							)
				A_     :						List[Any]     =					[]
				for piece in pieces:
							if len(lowercase							) > 1 and piece[-1] == str(""","""							) and piece[-2].isdigit():
										A_     :						Union[str, Any]     =					self.sp_model.EncodeAsPieces(piece[:-1].replace(lowercase	,				""""""							)							)
										if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
													if len(cur_pieces[0]							) == 1:
																A_     :						Dict     =					cur_pieces[1:]
													else:
																A_     :						str     =					cur_pieces[0][1:]
										cur_pieces.append(piece[-1]							)
										new_pieces.extend(lowercase							)
							else:
										new_pieces.append(lowercase							)

				return new_pieces


	def    _a     (self	,				lowercase							):
				return self.sp_model.PieceToId(lowercase							)


	def    _a     (self	,				lowercase							):
				return self.sp_model.IdToPiece(lowercase							)


	def    _a     (self	,				lowercase							):
				A_     :						int     =					"""""".join(lowercase							).replace(lowercase	,				""" """							).strip()
				return out_string


	def    _a     (self	,				lowercase	,				lowercase = None							):
				A_     :						List[str]     =					[self.sep_token_id]
				A_     :						Any     =					[self.cls_token_id]
				if token_ids_a is None:
							return token_ids_a + sep + cls
				return token_ids_a + sep + token_ids_a + sep + cls


	def    _a     (self	,				lowercase	,				lowercase = None	,				lowercase = False							):

				if already_has_special_tokens:
							return super().get_special_tokens_mask(
							    token_ids_a=lowercase	,				token_ids_a=lowercase	,				already_has_special_tokens=lowercase							)

				if token_ids_a is not None:
							return ([0] * len(lowercase							)) + [1] + ([0] * len(lowercase							)) + [1, 1]
				return ([0] * len(lowercase							)) + [1, 1]


	def    _a     (self	,				lowercase	,				lowercase = None							):
				A_     :						str     =					[self.sep_token_id]
				A_     :						str     =					[2]

				if token_ids_a is None:
							return len(token_ids_a + sep							) * [0] + cls_segment_id
				return len(token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1] + cls_segment_id


	def    _a     (self	,				lowercase	,				lowercase = None							):
				if not os.path.isdir(lowercase							):
							logger.error(F'Vocabulary path ({save_directory}) should be a directory'							)
							return
				A_     :						Tuple     =					os.path.join(
				    lowercase	,				(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]							)

				if os.path.abspath(self.vocab_file							) != os.path.abspath(lowercase							) and os.path.isfile(self.vocab_file							):
							copyfile(self.vocab_file	,				lowercase							)
				elif not os.path.isfile(self.vocab_file							):
							with open(lowercase	,				"""wb"""							) as fi:
										A_     :						Dict     =					self.sp_model.serialized_model_proto()
										fi.write(lowercase							)

				return (out_vocab_file,)


	def    _a     (self	,				*lowercase	,				**lowercase							):
				A_     :						Any     =					super()._decode(*lowercase	,				**lowercase							)
				A_     :						int     =					text.replace(""" """	,				""""""							).replace("""\u2582"""	,				""" """							).replace("""\u2583"""	,				"""\n"""							)
				return text