

from pickle import UnpicklingError

import jax
import jax.numpy as jnp
import numpy as np
from flax.serialization import from_bytes
from flax.traverse_util import flatten_dict

from ..utils import logging


_lowerCamelCase    :       Dict        =							logging.get_logger(__name__)

def       a__					(		UpperCAmelCase							:      Optional[int]    ,			UpperCAmelCase							:      List[str]      )  ->						List[Any]:
  try:
    with open(UpperCAmelCase    ,			'''rb'''      ) as flax_state_f:
      UpperCAmelCase	:							Optional[int]						=			from_bytes(UpperCAmelCase    ,			flax_state_f.read()      )
  except UnpicklingError as e:
    try:
      with open(UpperCAmelCase      ) as f:
        if f.read().startswith('''version'''      ):
          raise OSError(
              '''You seem to have cloned a repository without having git-lfs installed. Please'''
              ''' install git-lfs and run `git lfs install` followed by `git lfs pull` in the'''
              ''' folder you cloned.'''      )
        else:
          raise ValueError from e
    except (UnicodeDecodeError, ValueError):
      raise EnvironmentError(f'''Unable to convert {model_file} to Flax deserializable object. '''      )

  return load_flax_weights_in_pytorch_model(UpperCAmelCase    ,			UpperCAmelCase      )






def       a__					(		UpperCAmelCase							:      List[str]    ,			UpperCAmelCase							:      Union[str, Any]      )  ->						int:

  try:
    import torch  # noqa: F401
  except ImportError:
    logger.error(
        '''Loading Flax weights in PyTorch requires both PyTorch and Flax to be installed. Please see'''
        ''' https://pytorch.org/ and https://flax.readthedocs.io/en/latest/installation.html for installation'''
        ''' instructions.'''      )
    raise

  # check if we have bf16 weights
  UpperCAmelCase	:							Dict						=			flatten_dict(jax.tree_util.tree_map(lambda UpperCAmelCase      : x.dtype == jnp.bfloataa    ,			UpperCAmelCase      )      ).values()
  if any(UpperCAmelCase      ):
    # convert all weights to fp32 if they are bf16 since torch.from_numpy can-not handle bf16

    # and bf16 is not fully supported in PT yet.
    logger.warning(
        '''Found ``bfloat16`` weights in Flax model. Casting all ``bfloat16`` weights to ``float32`` '''
        '''before loading those in PyTorch model.'''      )
    UpperCAmelCase	:							Dict						=			jax.tree_util.tree_map(
        lambda UpperCAmelCase      : params.astype(np.floataa      ) if params.dtype == jnp.bfloataa else params    ,			UpperCAmelCase      )

  UpperCAmelCase	:							Any						=			''''''

  UpperCAmelCase	:							List[Any]						=			flatten_dict(UpperCAmelCase    ,			sep='''.'''      )
  UpperCAmelCase	:							List[Any]						=			pt_model.state_dict()

  # keep track of unexpected & missing keys
  UpperCAmelCase	:							List[str]						=			[]
  UpperCAmelCase	:							str						=			set(pt_model_dict.keys()      )

  for flax_key_tuple, flax_tensor in flax_state_dict.items():
    UpperCAmelCase	:							Tuple						=			flax_key_tuple.split('''.'''      )

    if flax_key_tuple_array[-1] == "kernel" and flax_tensor.ndim == 4:
      UpperCAmelCase	:							Dict						=			flax_key_tuple_array[:-1] + ['''weight''']
      UpperCAmelCase	:							Any						=			jnp.transpose(UpperCAmelCase    ,			(3, 2, 0, 1)      )
    elif flax_key_tuple_array[-1] == "kernel":
      UpperCAmelCase	:							Any						=			flax_key_tuple_array[:-1] + ['''weight''']
      UpperCAmelCase	:							str						=			flax_tensor.T
    elif flax_key_tuple_array[-1] == "scale":
      UpperCAmelCase	:							Union[str, Any]						=			flax_key_tuple_array[:-1] + ['''weight''']

    if "time_embedding" not in flax_key_tuple_array:
      for i, flax_key_tuple_string in enumerate(UpperCAmelCase      ):
        UpperCAmelCase	:							Any						=			(
            flax_key_tuple_string.replace('''_0'''    ,			'''.0'''      )
            .replace('''_1'''    ,			'''.1'''      )
            .replace('''_2'''    ,			'''.2'''      )
            .replace('''_3'''    ,			'''.3'''      )
            .replace('''_4'''    ,			'''.4'''      )
            .replace('''_5'''    ,			'''.5'''      )
            .replace('''_6'''    ,			'''.6'''      )
            .replace('''_7'''    ,			'''.7'''      )
            .replace('''_8'''    ,			'''.8'''      )
            .replace('''_9'''    ,			'''.9'''      )
        )

    UpperCAmelCase	:							str						=			'''.'''.join(UpperCAmelCase      )

    if flax_key in pt_model_dict:
      if flax_tensor.shape != pt_model_dict[flax_key].shape:
        raise ValueError(
            f'''Flax checkpoint seems to be incorrect. Weight {flax_key_tuple} was expected '''
            f'''to be of shape {pt_model_dict[flax_key].shape}, but is {flax_tensor.shape}.'''      )
      else:
        # add weight to pytorch dict
        UpperCAmelCase	:							int						=			np.asarray(UpperCAmelCase      ) if not isinstance(UpperCAmelCase    ,			np.ndarray      ) else flax_tensor
        UpperCAmelCase	:							Tuple						=			torch.from_numpy(UpperCAmelCase      )
        # remove from missing keys
        missing_keys.remove(UpperCAmelCase      )
    else:
      # weight is not expected by PyTorch model
      unexpected_keys.append(UpperCAmelCase      )

  pt_model.load_state_dict(UpperCAmelCase      )

  # re-transform missing_keys to list
  UpperCAmelCase	:							int						=			list(UpperCAmelCase      )

  if len(UpperCAmelCase      ) > 0:
    logger.warning(
        '''Some weights of the Flax model were not used when initializing the PyTorch model'''
        f''' {pt_model.__class__.__name__}: {unexpected_keys}\n- This IS expected if you are initializing'''
        f''' {pt_model.__class__.__name__} from a Flax model trained on another task or with another architecture'''
        ''' (e.g. initializing a BertForSequenceClassification model from a FlaxBertForPreTraining model).\n- This'''
        f''' IS NOT expected if you are initializing {pt_model.__class__.__name__} from a Flax model that you expect'''
        ''' to be exactly identical (e.g. initializing a BertForSequenceClassification model from a'''
        ''' FlaxBertForSequenceClassification model).'''      )
  if len(UpperCAmelCase      ) > 0:
    logger.warning(
        f'''Some weights of {pt_model.__class__.__name__} were not initialized from the Flax model and are newly'''
        f''' initialized: {missing_keys}\nYou should probably TRAIN this model on a down-stream task to be able to'''
        ''' use it for predictions and inference.'''      )

  return pt_model








import torch
from torch import nn





class  __UpperCAmelCase					(	nn.Module						):



   def __init__(							self							:					List[Any],		__A							:					List[Any],		__A							:					Optional[Any],		__A							:					int,		__A							:					List[Any],		__A							:					int=1,		__A							:					List[str]=False						):
     super().__init__()

     UpperCAmelCase	:							Union[str, Any]						=			n_token
     UpperCAmelCase	:							List[str]						=			d_embed
     UpperCAmelCase	:							Dict						=			d_proj

     UpperCAmelCase	:							List[Any]						=			cutoffs + [n_token]
     UpperCAmelCase	:							Dict						=			[0] + self.cutoffs
     UpperCAmelCase	:							int						=			div_val

     UpperCAmelCase	:							Union[str, Any]						=			self.cutoffs[0]
     UpperCAmelCase	:							str						=			len(self.cutoffs						) - 1
     UpperCAmelCase	:							Optional[int]						=			self.shortlist_size + self.n_clusters

     if self.n_clusters > 0:
       UpperCAmelCase	:							str						=			nn.Parameter(torch.zeros(self.n_clusters,		self.d_embed						)						)
       UpperCAmelCase	:							List[str]						=			nn.Parameter(torch.zeros(self.n_clusters						)						)

     UpperCAmelCase	:							Dict						=			nn.ModuleList()
     UpperCAmelCase	:							Optional[int]						=			nn.ParameterList()

     if div_val == 1:
       for i in range(len(self.cutoffs						)						):
         if d_proj != d_embed:
           self.out_projs.append(nn.Parameter(torch.FloatTensor(__A,		__A						)						)						)
         else:
           self.out_projs.append(__A						)

       self.out_layers.append(nn.Linear(__A,		__A						)						)
     else:
       for i in range(len(self.cutoffs						)						):
         UpperCAmelCase			,						UpperCAmelCase	:							Dict						=			self.cutoff_ends[i], self.cutoff_ends[i + 1]
         UpperCAmelCase	:							str						=			d_embed // (div_val**i)

         self.out_projs.append(nn.Parameter(torch.FloatTensor(__A,		__A						)						)						)

         self.out_layers.append(nn.Linear(__A,		r_idx - l_idx						)						)

     UpperCAmelCase	:							Optional[int]						=			keep_order



   def 	__magic_name__   (							self							:					Union[str, Any],		__A							:					List[str],		__A							:					Any,		__A							:					Dict,		__A							:					Optional[Any]						):
     if proj is None:
       UpperCAmelCase	:							List[Any]						=			nn.functional.linear(__A,		__A,		bias=__A						)
     else:
       # if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
       UpperCAmelCase	:							Union[str, Any]						=			nn.functional.linear(__A,		proj.t().contiguous()						)
       UpperCAmelCase	:							Optional[int]						=			nn.functional.linear(__A,		__A,		bias=__A						)
       # else:
       #     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
       #     if bias is not None:
       #         logit = logit + bias

     return logit



   def 	__magic_name__   (							self							:					int,		__A							:					int,		__A							:					List[Any]=None,		__A							:					Dict=False						):

     if labels is not None:
       # Shift so that tokens < n predict n
       UpperCAmelCase	:							List[Any]						=			hidden[..., :-1, :].contiguous()
       UpperCAmelCase	:							Any						=			labels[..., 1:].contiguous()
       UpperCAmelCase	:							Optional[Any]						=			hidden.view(-1,		hidden.size(-1						)						)
       UpperCAmelCase	:							Union[str, Any]						=			labels.view(-1						)
       if hidden.size(0						) != labels.size(0						):
         raise RuntimeError('''Input and labels should have the same size in the batch dimension.'''						)
     else:
       UpperCAmelCase	:							str						=			hidden.view(-1,		hidden.size(-1						)						)

     if self.n_clusters == 0:
       UpperCAmelCase	:							List[str]						=			self._compute_logit(__A,		self.out_layers[0].weight,		self.out_layers[0].bias,		self.out_projs[0]						)
       if labels is not None:
         UpperCAmelCase	:							Optional[int]						=			labels != -1_0_0
         UpperCAmelCase	:							Dict						=			torch.zeros_like(__A,		dtype=hidden.dtype,		device=hidden.device						)
         UpperCAmelCase	:							Any						=			(
             -nn.functional.log_softmax(__A,		dim=-1						)[mask].gather(1,		labels[mask].unsqueeze(1						)						).squeeze(1						)
         )
       else:
         UpperCAmelCase	:							Any						=			nn.functional.log_softmax(__A,		dim=-1						)
     else:
       # construct weights and biases
       UpperCAmelCase			,						UpperCAmelCase	:							Union[str, Any]						=			[], []
       for i in range(len(self.cutoffs						)						):
         if self.div_val == 1:
           UpperCAmelCase			,						UpperCAmelCase	:							List[Any]						=			self.cutoff_ends[i], self.cutoff_ends[i + 1]
           UpperCAmelCase	:							List[str]						=			self.out_layers[0].weight[l_idx:r_idx]
           UpperCAmelCase	:							Optional[int]						=			self.out_layers[0].bias[l_idx:r_idx]
         else:
           UpperCAmelCase	:							List[Any]						=			self.out_layers[i].weight
           UpperCAmelCase	:							Dict						=			self.out_layers[i].bias

         if i == 0:
           UpperCAmelCase	:							List[str]						=			torch.cat([weight_i, self.cluster_weight],		dim=0						)
           UpperCAmelCase	:							List[Any]						=			torch.cat([bias_i, self.cluster_bias],		dim=0						)

         weights.append(__A						)
         biases.append(__A						)

       UpperCAmelCase			,						UpperCAmelCase			,						UpperCAmelCase	:							str						=			weights[0], biases[0], self.out_projs[0]

       UpperCAmelCase	:							Dict						=			self._compute_logit(__A,		__A,		__A,		__A						)
       UpperCAmelCase	:							int						=			nn.functional.log_softmax(__A,		dim=1						)

       if labels is None:
         UpperCAmelCase	:							Optional[int]						=			hidden.new_empty((head_logit.size(0						), self.n_token)						)
       else:
         UpperCAmelCase	:							Union[str, Any]						=			torch.zeros_like(__A,		dtype=hidden.dtype,		device=hidden.device						)

       UpperCAmelCase	:							Union[str, Any]						=			0
       UpperCAmelCase	:							Any						=			[0] + self.cutoffs
       for i in range(len(__A						) - 1						):
         UpperCAmelCase			,						UpperCAmelCase	:							Optional[int]						=			cutoff_values[i], cutoff_values[i + 1]

         if labels is not None:
           UpperCAmelCase	:							List[str]						=			(labels >= l_idx) & (labels < r_idx)
           UpperCAmelCase	:							Tuple						=			mask_i.nonzero().squeeze()

           if indices_i.numel() == 0:
             continue

           UpperCAmelCase	:							Any						=			labels.index_select(0,		__A						) - l_idx
           UpperCAmelCase	:							Dict						=			head_logprob.index_select(0,		__A						)
           UpperCAmelCase	:							List[str]						=			hidden.index_select(0,		__A						)
         else:
           UpperCAmelCase	:							Tuple						=			hidden

         if i == 0:
           if labels is not None:
             UpperCAmelCase	:							Union[str, Any]						=			head_logprob_i.gather(1,		target_i[:, None]						).squeeze(1						)
           else:
             UpperCAmelCase	:							Optional[int]						=			head_logprob[:, : self.cutoffs[0]]
         else:
           UpperCAmelCase			,						UpperCAmelCase			,						UpperCAmelCase	:							List[str]						=			weights[i], biases[i], self.out_projs[i]

           UpperCAmelCase	:							List[str]						=			self._compute_logit(__A,		__A,		__A,		__A						)
           UpperCAmelCase	:							Dict						=			nn.functional.log_softmax(__A,		dim=1						)
           UpperCAmelCase	:							int						=			self.cutoffs[0] + i - 1  # No probability for the head cluster
           if labels is not None:
             UpperCAmelCase	:							Union[str, Any]						=			head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
                 1,		target_i[:, None]						).squeeze(1						)
           else:
             UpperCAmelCase	:							int						=			head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
             UpperCAmelCase	:							Optional[Any]						=			logprob_i

         if labels is not None:
           if (hasattr(self,		'''keep_order'''						) and self.keep_order) or keep_order:
             out.index_copy_(0,		__A,		-logprob_i						)
           else:
             out[offset : offset + logprob_i.size(0						)].copy_(-logprob_i						)
           offset += logprob_i.size(0						)

     return out





   def 	__magic_name__   (							self							:					Tuple,		__A							:					List[str]						):
     if self.n_clusters == 0:
       UpperCAmelCase	:							int						=			self._compute_logit(__A,		self.out_layers[0].weight,		self.out_layers[0].bias,		self.out_projs[0]						)
       return nn.functional.log_softmax(__A,		dim=-1						)
     else:
       # construct weights and biases
       UpperCAmelCase			,						UpperCAmelCase	:							Any						=			[], []
       for i in range(len(self.cutoffs						)						):
         if self.div_val == 1:
           UpperCAmelCase			,						UpperCAmelCase	:							Any						=			self.cutoff_ends[i], self.cutoff_ends[i + 1]
           UpperCAmelCase	:							List[Any]						=			self.out_layers[0].weight[l_idx:r_idx]
           UpperCAmelCase	:							List[str]						=			self.out_layers[0].bias[l_idx:r_idx]
         else:
           UpperCAmelCase	:							List[str]						=			self.out_layers[i].weight
           UpperCAmelCase	:							Dict						=			self.out_layers[i].bias

         if i == 0:
           UpperCAmelCase	:							Dict						=			torch.cat([weight_i, self.cluster_weight],		dim=0						)
           UpperCAmelCase	:							str						=			torch.cat([bias_i, self.cluster_bias],		dim=0						)

         weights.append(__A						)
         biases.append(__A						)

       UpperCAmelCase			,						UpperCAmelCase			,						UpperCAmelCase	:							Tuple						=			weights[0], biases[0], self.out_projs[0]
       UpperCAmelCase	:							int						=			self._compute_logit(__A,		__A,		__A,		__A						)

       UpperCAmelCase	:							Optional[int]						=			hidden.new_empty((head_logit.size(0						), self.n_token)						)
       UpperCAmelCase	:							Dict						=			nn.functional.log_softmax(__A,		dim=1						)

       UpperCAmelCase	:							List[str]						=			[0] + self.cutoffs
       for i in range(len(__A						) - 1						):
         UpperCAmelCase			,						UpperCAmelCase	:							Optional[Any]						=			cutoff_values[i], cutoff_values[i + 1]

         if i == 0:
           UpperCAmelCase	:							Any						=			head_logprob[:, : self.cutoffs[0]]
         else:
           UpperCAmelCase			,						UpperCAmelCase			,						UpperCAmelCase	:							int						=			weights[i], biases[i], self.out_projs[i]

           UpperCAmelCase	:							Tuple						=			self._compute_logit(__A,		__A,		__A,		__A						)
           UpperCAmelCase	:							List[Any]						=			nn.functional.log_softmax(__A,		dim=1						)

           UpperCAmelCase	:							Optional[int]						=			head_logprob[:, -i] + tail_logprob_i
           UpperCAmelCase	:							Optional[Any]						=			logprob_i

       return out





