


import argparse
import re

from flax.traverse_util import flatten_dict, unflatten_dict
from tax import checkpoints

from transformers import SwitchTransformersConfig, SwitchTransformersForConditionalGeneration
from transformers.modeling_flax_pytorch_utils import load_flax_weights_in_pytorch_model
from transformers.utils import logging


logging.set_verbosity_info()


# should not include what is already done by the `from_pt` argument
snake_case							:	int					      =				{
    '''/attention/''': '''/0/SelfAttention/''',
    '''/self_attention/''': '''/0/SelfAttention/''',
    '''/encoder_decoder_attention/''': '''/1/EncDecAttention/''',
    '''value''': '''v''',
    '''query''': '''q''',
    '''key''': '''k''',
    '''out''': '''o''',
    '''pre_self_attention_layer_norm''': '''0/layer_norm''',
    '''pre_cross_attention_layer_norm''': '''1/layer_norm''',
    '''pre_attention_layer_norm''': '''0/layer_norm''',  # previously 1, but seems wrong
    '''token_embedder''': '''shared''',
    '''encoder_norm''': '''final_layer_norm''',
    '''decoder_norm''': '''final_layer_norm''',
    '''relpos_bias/rel_embedding''': '''block/0/layer/0/SelfAttention/relative_attention_bias/weight''',
    '''router/router_weights/w/''': '''router/classifier/''',
    '''roer/roer_weights/w/''': '''router/classifier/''',
    '''logits_dense''': '''lm_head''',
}






def 							__lowercase       (			__lowerCAmelCase							:    Optional[int]       ):
							# 1. in HF T5, we have block.{x}.layer.{y}. which corresponds to layer.{x} in
							# the original model
							a__					=					list(s_dict.keys()       )
							for key in keys:
														a__					=					R'.*/layers_(\d+)'
														a__					=					key
														if re.match(snake_case__ ,   snake_case__       ):
																					a__					=					re.sub(R'layers_(\d+)' ,   R'block/\1/layer' ,   snake_case__       )

														a__					=					R'(encoder|decoder)\/'

														if re.match(snake_case__ ,   snake_case__       ):
																					a__					=					re.match(snake_case__ ,   snake_case__       ).groups()
																					if groups[0] == "encoder":
																												a__					=					re.sub(R'/mlp/' ,   R'/1/mlp/' ,   snake_case__       )
																												a__					=					re.sub(R'/pre_mlp_layer_norm/' ,   R'/1/layer_norm/' ,   snake_case__       )

																					elif groups[0] == "decoder":
																												a__					=					re.sub(R'/mlp/' ,   R'/2/mlp/' ,   snake_case__       )
																												a__					=					re.sub(R'/pre_mlp_layer_norm/' ,   R'/2/layer_norm/' ,   snake_case__       )

        # 2. Convert other classic mappings
														for old_key, temp_key in MOE_LAYER_NAME_MAPPING.items():
																					if old_key in new_key:
																												a__					=					new_key.replace(snake_case__ ,   snake_case__       )

														print(F'{key} -> {new_key}'       )
														a__					=					s_dict.pop(snake_case__       )

							if "encoder/block/0/layer/0/SelfAttention/relative_attention_bias/weight" in s_dict:
														a__					=					s_dict[
														    'encoder/block/0/layer/0/SelfAttention/relative_attention_bias/weight'
														].T
							if "decoder/block/0/layer/0/SelfAttention/relative_attention_bias/weight" in s_dict:
														a__					=					s_dict[
														    'decoder/block/0/layer/0/SelfAttention/relative_attention_bias/weight'
														].T

							# 3. Take extra care of the EXPERTS layer
							for key in list(s_dict.keys()       ):
														if "expert" in key:
																					a__					=					s_dict[key].shape[0]
																					a__					=					s_dict[key]
																					for idx in range(snake_case__       ):
																												a__					=					expert_weihts[idx]
																												print(F'{key} -> {key.replace("expert/" ,   "nested fstring"       )}'       )

																					s_dict.pop(snake_case__       )

							return s_dict


snake_case							:	str					      =				{
    '''NUM_ENCODER_LAYERS''': '''num_layers''',
    '''NUM_DECODER_LAYERS''': '''num_decoder_layers''',
    '''NUM_HEADS''': '''num_heads''',
    '''HEAD_DIM''': '''d_kv''',
    '''EMBED_DIM''': '''d_model''',
    '''MLP_DIM''': '''d_ff''',
    '''NUM_SELECTED_EXPERTS''': '''num_selected_experts''',
    '''NUM_ENCODER_SPARSE_LAYERS''': '''num_sparse_encoder_layers''',
    '''NUM_DECODER_SPARSE_LAYERS''': '''num_sparse_decoder_layers''',
    '''dense.MlpBlock.activations''': '''feed_forward_proj''',
}






def 							__lowercase       (			__lowerCAmelCase							:    Dict ,   __lowerCAmelCase							:    Optional[int]       ):
							# Convert a google style config to the hugging face fromat
							import regex as re

							with open(snake_case__ ,   'r'       ) as f:
														a__					=					f.read()

							a__					=					re.findall(R'(.*) = ([0-9.]*)' ,   snake_case__       )
							a__					=					{}
							for param, value in regex_match:
														if param in GIN_TO_CONFIG_MAPPING and value != "":
																					a__					=					float(snake_case__       ) if '.' in value else int(snake_case__       )

							a__					=					re.findall(R'(.*activations) = \(\'(.*)\',\)' ,   snake_case__       )[0]
							a__					=					str(activation[1]       )

							a__					=					num_experts
							a__					=					SwitchTransformersConfig(**snake_case__       )
							return config






def 							__lowercase       (			__lowerCAmelCase							:    Dict ,   __lowerCAmelCase							:    List[str] ,   __lowerCAmelCase							:    str=None ,   __lowerCAmelCase							:    Union[str, Any]="./" ,   __lowerCAmelCase							:    Optional[Any]=8       ):
							# Initialise PyTorch model

							print(F'Loading flax weights from : {flax_checkpoint_path}'       )
							a__					=					checkpoints.load_tax_checkpoint(snake_case__       )

							if gin_file is not None:
														a__					=					convert_gin_to_config(snake_case__ ,   snake_case__       )
							else:
														a__					=					SwitchTransformersConfig.from_pretrained(snake_case__       )

							a__					=					SwitchTransformersForConditionalGeneration(snake_case__       )

							a__					=					flax_params['target']
							a__					=					flatten_dict(snake_case__ ,   sep='/'       )
							a__					=					rename_keys(snake_case__       )
							a__					=					unflatten_dict(snake_case__ ,   sep='/'       )

							# Load the flax params in the PT model
							load_flax_weights_in_pytorch_model(snake_case__ ,   snake_case__       )

							print(F'Save PyTorch model to {pytorch_dump_path}'       )
							pt_model.save_pretrained(snake_case__       )


if __name__ == "__main__":
						snake_case							:	Dict					      =				argparse.ArgumentParser()
						# Required parameters
						parser.add_argument(
						    '''--switch_t5x_checkpoint_path''',
						    default=None,
						    type=str,
						    required=True,
						    help=(
						        '''The config json file corresponding to the pre-trained SwitchTransformers model. \nThis specifies the'''
						        ''' model architecture. If not provided, a `gin_file` has to be provided.'''
						    ),
						)
						parser.add_argument(
						    '''--gin_file''',
						    default=None,
						    type=str,
						    required=False,
						    help='''Path to the gin config file. If not provided, a `config_file` has to be passed   ''',
						)
						parser.add_argument(
						    '''--config_name''', default=None, type=str, required=False, help='''Config name of SwitchTransformers model.'''
						)
						parser.add_argument(
						    '''--pytorch_dump_folder_path''', default=None, type=str, required=True, help='''Path to the output pytorch model.'''
						)
						parser.add_argument('''--num_experts''', default=8, type=int, required=False, help='''Number of experts''')
						snake_case							:	Optional[Any]					      =				parser.parse_args()
						convert_flax_checkpoint_to_pytorch(
						    args.switch_tax_checkpoint_path,
						    args.config_name,
						    args.gin_file,
						    args.pytorch_dump_folder_path,
						    args.num_experts,
						)








import inspect

import jax
import jax.lax as lax
import jax.numpy as jnp

from ..utils import add_start_docstrings
from ..utils.logging import get_logger


snake_case							:	Dict					      =				get_logger(__name__)


snake_case							:	str					      =				r'''
    Args:
        input_ids (`jnp.ndarray` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`PreTrainedTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        scores (`jnp.ndarray` of shape `(batch_size, config.vocab_size)`):
            Prediction scores of a language modeling head. These can be logits for each vocabulary when not using beam
            search or log softmax for each vocabulary token when using beam search
        kwargs (`Dict[str, Any]`, *optional*):
            Additional logits processor specific kwargs.

    Return:
        `jnp.ndarray` of shape `(batch_size, config.vocab_size)`: The processed prediction scores.

'''







class 	snake_case_       :
					@add_start_docstrings(__snake_case				)
					def __call__(				self       :Dict      ,__snake_case       :jnp.ndarray      ,__snake_case       :jnp.ndarray				) ->   jnp.ndarray:
												raise NotImplementedError(
												    F'{self.__class__} is an abstract class. Only classes inheriting this class can be called.'				)







class 	snake_case_       :
					@add_start_docstrings(__snake_case				)
					def __call__(				self       :List[str]      ,__snake_case       :jnp.ndarray      ,__snake_case       :jnp.ndarray				) ->   jnp.ndarray:
												raise NotImplementedError(
												    F'{self.__class__} is an abstract class. Only classes inheriting this class can be called.'				)







class 	snake_case_       (lowerCamelCase_				):
					@add_start_docstrings(__snake_case				)
					def __call__(				self       :Dict      ,__snake_case       :jnp.ndarray      ,__snake_case       :jnp.ndarray      ,__snake_case       :int      ,**__snake_case       :Any				) ->   jnp.ndarray:
												for processor in self:
																			a__					=					inspect.signature(processor.__call__				).parameters
																			if len(__snake_case				) > 3:
																										if not all(arg in kwargs for arg in list(function_args.keys()				)[2:]				):
																																	raise ValueError(
																																	    F'Make sure that all the required parameters: {list(function_args.keys()				)} for '
																																	    F'{processor.__class__} are passed to the logits processor.'				)
																										a__					=					processor(__snake_case      ,__snake_case      ,__snake_case      ,**__snake_case				)
																			else:
																										a__					=					processor(__snake_case      ,__snake_case      ,__snake_case				)
												return scores







class 	snake_case_       (lowerCamelCase_				):
					def __init__(				self       :str      ,__snake_case       :float				) ->   Tuple:
												if not isinstance(__snake_case      ,__snake_case				) or not (temperature > 0):
																			raise ValueError(F'`temperature` has to be a strictly positive float, but is {temperature}'				)

												a__					=					temperature



					def __call__(				self       :Optional[int]      ,__snake_case       :jnp.ndarray      ,__snake_case       :jnp.ndarray      ,__snake_case       :int				) ->   jnp.ndarray:
												a__					=					scores / self.temperature
												return scores







class 	snake_case_       (lowerCamelCase_				):
					def __init__(				self       :Any      ,__snake_case       :float      ,__snake_case       :float = -float('Inf'				)      ,__snake_case       :int = 1				) ->   Dict:
												if not isinstance(__snake_case      ,__snake_case				) or (top_p < 0 or top_p > 1.0):
																			raise ValueError(F'`top_p` has to be a float > 0 and < 1, but is {top_p}'				)
												if not isinstance(__snake_case      ,__snake_case				) or (min_tokens_to_keep < 1):
																			raise ValueError(F'`min_tokens_to_keep` has to be a positive integer, but is {min_tokens_to_keep}'				)

												a__					=					top_p
												a__					=					filter_value
												a__					=					min_tokens_to_keep



					def __call__(				self       :Optional[int]      ,__snake_case       :jnp.ndarray      ,__snake_case       :jnp.ndarray      ,__snake_case       :int				) ->   jnp.ndarray:
												a__				, a__					=					lax.top_k(__snake_case      ,scores.shape[-1]				)

												a__					=					jnp.full_like(__snake_case      ,self.filter_value				)
												a__					=					jax.nn.softmax(__snake_case      ,axis=-1				).cumsum(axis=-1				)
												a__					=					cumulative_probs < self.top_p

												# include the token that is higher than top_p as well
												a__					=					jnp.roll(__snake_case      ,1				)
												score_mask |= score_mask.at[:, 0].set(__snake_case				)

												# min tokens to keep
												a__					=					score_mask.at[:, : self.min_tokens_to_keep].set(__snake_case				)

												a__					=					jnp.where(__snake_case      ,__snake_case      ,__snake_case				)
												a__					=					jax.lax.sort_key_val(__snake_case      ,__snake_case				)[-1]

												return next_scores







class 	snake_case_       (lowerCamelCase_				):
					def __init__(				self       :List[str]      ,__snake_case       :int      ,__snake_case       :float = -float('Inf'				)      ,__snake_case       :int = 1				) ->   Any:
												if not isinstance(__snake_case      ,__snake_case				) or top_k <= 0:
																			raise ValueError(F'`top_k` has to be a strictly positive integer, but is {top_k}'				)

												a__					=					max(__snake_case      ,__snake_case				)
												a__					=					filter_value



					def __call__(				self       :int      ,__snake_case       :jnp.ndarray      ,__snake_case       :jnp.ndarray      ,__snake_case       :int				) ->   jnp.ndarray:
												a__				, a__					=					scores.shape
												a__					=					jnp.full(batch_size * vocab_size      ,self.filter_value				)

												a__					=					min(self.top_k      ,scores.shape[-1]				)  # Safety check
												a__				, a__					=					lax.top_k(__snake_case      ,__snake_case				)
												a__					=					jnp.broadcast_to((jnp.arange(__snake_case				) * vocab_size)[:, None]      ,(batch_size, topk)				).flatten()
												a__					=					topk_scores.flatten()
												a__					=					topk_indices.flatten() + shift

												a__					=					next_scores_flat.at[topk_indices_flat].set(__snake_case				)
												a__					=					next_scores_flat.reshape(__snake_case      ,__snake_case				)
												return next_scores







class 	snake_case_       (lowerCamelCase_				):
					def __init__(				self       :int      ,__snake_case       :int				) ->   str:
												a__					=					bos_token_id



					def __call__(				self       :List[Any]      ,__snake_case       :jnp.ndarray      ,__snake_case       :jnp.ndarray      ,__snake_case       :int				) ->   jnp.ndarray:
												a__					=					jnp.full(scores.shape      ,-float('inf'				)				)

												a__					=					1 - jnp.bool_(cur_len - 1				)

												a__					=					jnp.where(__snake_case      ,new_scores.at[:, self.bos_token_id].set(0				)      ,__snake_case				)

												return scores







class 	snake_case_       (lowerCamelCase_				):
					def __init__(				self       :Union[str, Any]      ,__snake_case       :int      ,__snake_case       :int				) ->   List[Any]:
												a__					=					max_length
												a__					=					eos_token_id



					def __call__(				self       :int      ,__snake_case       :jnp.ndarray      ,__snake_case       :jnp.ndarray      ,__snake_case       :int				) ->   jnp.ndarray:
												a__					=					jnp.full(scores.shape      ,-float('inf'				)				)

												a__					=					1 - jnp.bool_(cur_len - self.max_length + 1				)

												a__					=					jnp.where(__snake_case      ,new_scores.at[:, self.eos_token_id].set(0				)      ,__snake_case				)

												return scores







class 	snake_case_       (lowerCamelCase_				):
					def __init__(				self       :str      ,__snake_case       :int      ,__snake_case       :int				) ->   List[str]:
												if not isinstance(__snake_case      ,__snake_case				) or min_length < 0:
																			raise ValueError(F'`min_length` has to be a positive integer, but is {min_length}'				)

												if not isinstance(__snake_case      ,__snake_case				) or eos_token_id < 0:
																			raise ValueError(F'`eos_token_id` has to be a positive integer, but is {eos_token_id}'				)

												a__					=					min_length
												a__					=					eos_token_id



					def __call__(				self       :Any      ,__snake_case       :jnp.ndarray      ,__snake_case       :jnp.ndarray      ,__snake_case       :int				) ->   jnp.ndarray:
												# create boolean flag to decide if min length penalty should be applied
												a__					=					1 - jnp.clip(cur_len - self.min_length      ,0      ,1				)

												a__					=					jnp.where(__snake_case      ,scores.at[:, self.eos_token_id].set(-float('inf'				)				)      ,__snake_case				)

												return scores







class 	snake_case_       (lowerCamelCase_				):
					def __init__(				self       :Optional[int]      ,__snake_case       :List[str]      ,__snake_case       :Optional[int]				) ->   Tuple:
												a__					=					list(__snake_case				)
												a__					=					begin_index



					def __call__(				self       :str      ,__snake_case       :List[str]      ,__snake_case       :str      ,__snake_case       :int				) ->   str:
												a__					=					1 - jnp.bool_(cur_len - self.begin_index				)

												a__					=					jnp.where(__snake_case      ,scores.at[:, self.begin_suppress_tokens].set(-float('inf'				)				)      ,__snake_case				)

												return scores







class 	snake_case_       (lowerCamelCase_				):
					def __init__(				self       :List[str]      ,__snake_case       :list				) ->   List[Any]:
												a__					=					list(__snake_case				)



					def __call__(				self       :Dict      ,__snake_case       :jnp.ndarray      ,__snake_case       :jnp.ndarray      ,__snake_case       :int				) ->   jnp.ndarray:
												a__					=					scores.at[..., self.suppress_tokens].set(-float('inf'				)				)

												return scores







class 	snake_case_       (lowerCamelCase_				):
					def __init__(				self       :Dict      ,__snake_case       :Optional[int]				) ->   Union[str, Any]:
												a__					=					dict(__snake_case				)
												# Converts the dictionary of format {index: token} containing the tokens to be forced to an array, where the
												# index of the array corresponds to the index of the token to be forced, for XLA compatibility.
												# Indexes without forced tokens will have a negative value.
												a__					=					jnp.ones((max(force_token_map.keys()				) + 1)      ,dtype=jnp.intaa				) * -1
												for index, token in force_token_map.items():
																			if token is not None:
																										a__					=					force_token_array.at[index].set(__snake_case				)
												a__					=					jnp.intaa(__snake_case				)



					def __call__(				self       :Optional[int]      ,__snake_case       :jnp.ndarray      ,__snake_case       :jnp.ndarray      ,__snake_case       :int				) ->   jnp.ndarray:
												def _force_token(__snake_case       :Optional[Any]				):
																			a__					=					scores.shape[0]
																			a__					=					self.force_token_array[generation_idx]

																			a__					=					jnp.ones_like(__snake_case      ,dtype=scores.dtype				) * -float('inf'				)
																			a__					=					jnp.zeros((batch_size, 1)      ,dtype=scores.dtype				)
																			a__					=					lax.dynamic_update_slice(__snake_case      ,__snake_case      ,(0, current_token)				)
																			return new_scores

												a__					=					lax.cond(
												    cur_len >= self.force_token_array.shape[0]      ,lambda: scores      ,lambda: lax.cond(
												        self.force_token_array[cur_len] >= 0      ,lambda: _force_token(__snake_case				)      ,lambda: scores      ,)      ,)
												return scores







class 	snake_case_       (lowerCamelCase_				):
					def __init__(				self       :Any      ,__snake_case       :List[str]      ,__snake_case       :str      ,__snake_case       :List[Any]				) ->   Optional[int]:
												a__					=					generate_config.eos_token_id
												a__					=					generate_config.no_timestamps_token_id
												a__					=					generate_config.no_timestamps_token_id + 1

												a__					=					decoder_input_length + 1

												if generate_config.is_multilingual:
																			# room for language token and task token
																			self.begin_index += 2
												if hasattr(__snake_case      ,'max_initial_timestamp_index'				):
																			a__					=					generate_config.max_initial_timestamp_index
												else:
																			a__					=					model_config.vocab_size
												if self.max_initial_timestamp_index is None:
																			a__					=					model_config.vocab_size



					def __call__(				self       :Any      ,__snake_case       :List[Any]      ,__snake_case       :Optional[int]      ,__snake_case       :Optional[Any]				) ->   Tuple:
												# suppress <|notimestamps|> which is handled by without_timestamps
												a__					=					scores.at[:, self.no_timestamps_token_id].set(-float('inf'				)				)

												def handle_pairs(__snake_case       :List[str]      ,__snake_case       :Union[str, Any]				):
																			a__					=					jnp.where((cur_len - self.begin_index) >= 1      ,__snake_case      ,__snake_case				)
																			a__					=					jnp.where(
																			    input_ids_k[cur_len - 1] >= self.timestamp_begin      ,True and last_was_timestamp      ,__snake_case      ,)

																			a__					=					jnp.where((cur_len - self.begin_index) < 2      ,__snake_case      ,__snake_case				)
																			a__					=					jnp.where(
																			    input_ids_k[cur_len - 2] >= self.timestamp_begin      ,__snake_case      ,__snake_case      ,)

																			return jnp.where(
																			    __snake_case      ,jnp.where(
																			        penultimate_was_timestamp > 0      ,scores_k.at[self.timestamp_begin :].set(-float('inf'				)				)      ,scores_k.at[: self.eos_token_id].set(-float('inf'				)				)      ,)      ,__snake_case      ,)

												a__					=					jax.vmap(__snake_case				)(__snake_case      ,__snake_case				)

												a__					=					jnp.where(cur_len == self.begin_index      ,__snake_case      ,__snake_case				)
												a__					=					jnp.where(
												    self.max_initial_timestamp_index is not None      ,True and apply_max_initial_timestamp      ,__snake_case      ,)

												a__					=					self.timestamp_begin + self.max_initial_timestamp_index

												a__					=					jnp.where(
												    __snake_case      ,scores.at[:, last_allowed + 1 :].set(-float('inf'				)				)      ,__snake_case      ,)

												# if sum of probability over timestamps is above any other token, sample timestamp
												a__					=					jax.nn.log_softmax(__snake_case      ,axis=-1				)

												def handle_cumulative_probs(__snake_case       :Dict      ,__snake_case       :List[Any]				):
																			a__					=					jax.nn.logsumexp(logprobs_k[self.timestamp_begin :]      ,axis=-1				)
																			a__					=					jnp.max(logprobs_k[: self.timestamp_begin]				)
																			return jnp.where(
																			    timestamp_logprob > max_text_token_logprob      ,scores_k.at[: self.timestamp_begin].set(-float('inf'				)				)      ,__snake_case      ,)

												a__					=					jax.vmap(__snake_case				)(__snake_case      ,__snake_case				)

												return scores




