




'''simple docstring'''






import torch
from torch import nn






class        __snake_case	(       nn.Module):







    """simple docstring"""



    def __init__(       self   :  List[Any]   ,						lowerCamelCase   :  List[Any]   ,						lowerCamelCase   :  Tuple   ,						lowerCamelCase   :  int   ,						lowerCamelCase   :  Union[str, Any]   ,						lowerCamelCase   :  List[Any]=1   ,						lowerCamelCase   :  Tuple=False		)       ->  List[Any]:
      super().__init__()

      lowerCAmelCase_      :				Optional[Any]      =       n_token
      lowerCAmelCase_      :				int      =       d_embed
      lowerCAmelCase_      :				Any      =       d_proj

      lowerCAmelCase_      :				str      =       cutoffs + [n_token]
      lowerCAmelCase_      :				Optional[Any]      =       [0] + self.cutoffs
      lowerCAmelCase_      :				Union[str, Any]      =       div_val

      lowerCAmelCase_      :				Optional[Any]      =       self.cutoffs[0]
      lowerCAmelCase_      :				List[Any]      =       len(self.cutoffs		) - 1
      lowerCAmelCase_      :				List[Any]      =       self.shortlist_size + self.n_clusters

      if self.n_clusters > 0:
        lowerCAmelCase_      :				Optional[Any]      =       nn.Parameter(torch.zeros(self.n_clusters   ,						self.d_embed		)		)
        lowerCAmelCase_      :				List[Any]      =       nn.Parameter(torch.zeros(self.n_clusters		)		)

      lowerCAmelCase_      :				Tuple      =       nn.ModuleList()
      lowerCAmelCase_      :				Any      =       nn.ParameterList()

      if div_val == 1:
        for i in range(len(self.cutoffs		)		):
          if d_proj != d_embed:
            self.out_projs.append(nn.Parameter(torch.FloatTensor(lowerCamelCase   ,						lowerCamelCase		)		)		)
          else:
            self.out_projs.append(lowerCamelCase		)

        self.out_layers.append(nn.Linear(lowerCamelCase   ,						lowerCamelCase		)		)
      else:
        for i in range(len(self.cutoffs		)		):
          lowerCAmelCase_      :				str      =       self.cutoff_ends[i], self.cutoff_ends[i + 1]
          lowerCAmelCase_      :				List[str]      =       d_embed // (div_val**i)

          self.out_projs.append(nn.Parameter(torch.FloatTensor(lowerCamelCase   ,						lowerCamelCase		)		)		)

          self.out_layers.append(nn.Linear(lowerCamelCase   ,						r_idx - l_idx		)		)

      lowerCAmelCase_      :				Optional[int]      =       keep_order

    def __lowercase       (       self   :  Tuple   ,						lowerCamelCase   :  Optional[Any]   ,						lowerCamelCase   :  Optional[int]   ,						lowerCamelCase   :  Any   ,						lowerCamelCase   :  Any		)       ->  Dict:
      if proj is None:
        lowerCAmelCase_      :				List[Any]      =       nn.functional.linear(lowerCamelCase   ,						lowerCamelCase   ,						bias=lowerCamelCase		)
      else:
        # if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
        lowerCAmelCase_      :				int      =       nn.functional.linear(lowerCamelCase   ,						proj.t().contiguous()		)
        lowerCAmelCase_      :				Tuple      =       nn.functional.linear(lowerCamelCase   ,						lowerCamelCase   ,						bias=lowerCamelCase		)
        # else:
        #     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
        #     if bias is not None:
        #         logit = logit + bias

      return logit

    def __lowercase       (       self   :  Tuple   ,						lowerCamelCase   :  Optional[int]   ,						lowerCamelCase   :  Optional[int]=None   ,						lowerCamelCase   :  List[str]=False		)       ->  Union[str, Any]:

      if labels is not None:
        # Shift so that tokens < n predict n
        lowerCAmelCase_      :				int      =       hidden[..., :-1, :].contiguous()
        lowerCAmelCase_      :				int      =       labels[..., 1:].contiguous()
        lowerCAmelCase_      :				Optional[int]      =       hidden.view(-1   ,						hidden.size(-1		)		)
        lowerCAmelCase_      :				Union[str, Any]      =       labels.view(-1		)
        if hidden.size(0		) != labels.size(0		):
          raise RuntimeError("""Input and labels should have the same size in the batch dimension."""		)
      else:
        lowerCAmelCase_      :				int      =       hidden.view(-1   ,						hidden.size(-1		)		)

      if self.n_clusters == 0:
        lowerCAmelCase_      :				Union[str, Any]      =       self._compute_logit(lowerCamelCase   ,						self.out_layers[0].weight   ,						self.out_layers[0].bias   ,						self.out_projs[0]		)
        if labels is not None:
          lowerCAmelCase_      :				Union[str, Any]      =       labels != -1_00
          lowerCAmelCase_      :				Optional[Any]      =       torch.zeros_like(lowerCamelCase   ,						dtype=hidden.dtype   ,						device=hidden.device		)
          lowerCAmelCase_      :				Dict      =       (
              -nn.functional.log_softmax(lowerCamelCase   ,						dim=-1		)[mask].gather(1   ,						labels[mask].unsqueeze(1		)		).squeeze(1		)
          )
        else:
          lowerCAmelCase_      :				int      =       nn.functional.log_softmax(lowerCamelCase   ,						dim=-1		)
      else:
        # construct weights and biases
        lowerCAmelCase_      :				int      =       [], []
        for i in range(len(self.cutoffs		)		):
          if self.div_val == 1:
            lowerCAmelCase_      :				Union[str, Any]      =       self.cutoff_ends[i], self.cutoff_ends[i + 1]
            lowerCAmelCase_      :				Optional[Any]      =       self.out_layers[0].weight[l_idx:r_idx]
            lowerCAmelCase_      :				Tuple      =       self.out_layers[0].bias[l_idx:r_idx]
          else:
            lowerCAmelCase_      :				List[str]      =       self.out_layers[i].weight
            lowerCAmelCase_      :				Tuple      =       self.out_layers[i].bias

          if i == 0:
            lowerCAmelCase_      :				Tuple      =       torch.cat([weight_i, self.cluster_weight]   ,						dim=0		)
            lowerCAmelCase_      :				str      =       torch.cat([bias_i, self.cluster_bias]   ,						dim=0		)

          weights.append(lowerCamelCase		)
          biases.append(lowerCamelCase		)

        lowerCAmelCase_      :				Any      =       weights[0], biases[0], self.out_projs[0]

        lowerCAmelCase_      :				Optional[Any]      =       self._compute_logit(lowerCamelCase   ,						lowerCamelCase   ,						lowerCamelCase   ,						lowerCamelCase		)
        lowerCAmelCase_      :				Dict      =       nn.functional.log_softmax(lowerCamelCase   ,						dim=1		)

        if labels is None:
          lowerCAmelCase_      :				Any      =       hidden.new_empty((head_logit.size(0		), self.n_token)		)
        else:
          lowerCAmelCase_      :				Dict      =       torch.zeros_like(lowerCamelCase   ,						dtype=hidden.dtype   ,						device=hidden.device		)

        lowerCAmelCase_      :				Optional[Any]      =       0
        lowerCAmelCase_      :				Tuple      =       [0] + self.cutoffs
        for i in range(len(lowerCamelCase		) - 1		):
          lowerCAmelCase_      :				Tuple      =       cutoff_values[i], cutoff_values[i + 1]

          if labels is not None:
            lowerCAmelCase_      :				Union[str, Any]      =       (labels >= l_idx) & (labels < r_idx)
            lowerCAmelCase_      :				int      =       mask_i.nonzero().squeeze()

            if indices_i.numel() == 0:
              continue

            lowerCAmelCase_      :				List[str]      =       labels.index_select(0   ,						lowerCamelCase		) - l_idx
            lowerCAmelCase_      :				Optional[Any]      =       head_logprob.index_select(0   ,						lowerCamelCase		)
            lowerCAmelCase_      :				Dict      =       hidden.index_select(0   ,						lowerCamelCase		)
          else:
            lowerCAmelCase_      :				Optional[Any]      =       hidden

          if i == 0:
            if labels is not None:
              lowerCAmelCase_      :				List[str]      =       head_logprob_i.gather(1   ,						target_i[:, None]		).squeeze(1		)
            else:
              lowerCAmelCase_      :				str      =       head_logprob[:, : self.cutoffs[0]]
          else:
            lowerCAmelCase_      :				Optional[Any]      =       weights[i], biases[i], self.out_projs[i]

            lowerCAmelCase_      :				int      =       self._compute_logit(lowerCamelCase   ,						lowerCamelCase   ,						lowerCamelCase   ,						lowerCamelCase		)
            lowerCAmelCase_      :				Optional[Any]      =       nn.functional.log_softmax(lowerCamelCase   ,						dim=1		)
            lowerCAmelCase_      :				Union[str, Any]      =       self.cutoffs[0] + i - 1  # No probability for the head cluster
            if labels is not None:
              lowerCAmelCase_      :				List[str]      =       head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
                  1   ,						target_i[:, None]		).squeeze(1		)
            else:
              lowerCAmelCase_      :				Optional[Any]      =       head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
              lowerCAmelCase_      :				Optional[Any]      =       logprob_i

          if labels is not None:
            if (hasattr(self   ,						"""keep_order"""		) and self.keep_order) or keep_order:
              out.index_copy_(0   ,						lowerCamelCase   ,						-logprob_i		)
            else:
              out[offset : offset + logprob_i.size(0		)].copy_(-logprob_i		)
            offset += logprob_i.size(0		)

      return out





    def __lowercase       (       self   :  Optional[int]   ,						lowerCamelCase   :  List[Any]		)       ->  Dict:
      if self.n_clusters == 0:
        lowerCAmelCase_      :				Optional[int]      =       self._compute_logit(lowerCamelCase   ,						self.out_layers[0].weight   ,						self.out_layers[0].bias   ,						self.out_projs[0]		)
        return nn.functional.log_softmax(lowerCamelCase   ,						dim=-1		)
      else:
        # construct weights and biases
        lowerCAmelCase_      :				int      =       [], []
        for i in range(len(self.cutoffs		)		):
          if self.div_val == 1:
            lowerCAmelCase_      :				Optional[Any]      =       self.cutoff_ends[i], self.cutoff_ends[i + 1]
            lowerCAmelCase_      :				List[Any]      =       self.out_layers[0].weight[l_idx:r_idx]
            lowerCAmelCase_      :				Optional[int]      =       self.out_layers[0].bias[l_idx:r_idx]
          else:
            lowerCAmelCase_      :				Union[str, Any]      =       self.out_layers[i].weight
            lowerCAmelCase_      :				Dict      =       self.out_layers[i].bias

          if i == 0:
            lowerCAmelCase_      :				Tuple      =       torch.cat([weight_i, self.cluster_weight]   ,						dim=0		)
            lowerCAmelCase_      :				Dict      =       torch.cat([bias_i, self.cluster_bias]   ,						dim=0		)

          weights.append(lowerCamelCase		)
          biases.append(lowerCamelCase		)

        lowerCAmelCase_      :				Optional[int]      =       weights[0], biases[0], self.out_projs[0]
        lowerCAmelCase_      :				Optional[Any]      =       self._compute_logit(lowerCamelCase   ,						lowerCamelCase   ,						lowerCamelCase   ,						lowerCamelCase		)

        lowerCAmelCase_      :				Any      =       hidden.new_empty((head_logit.size(0		), self.n_token)		)
        lowerCAmelCase_      :				Dict      =       nn.functional.log_softmax(lowerCamelCase   ,						dim=1		)

        lowerCAmelCase_      :				Dict      =       [0] + self.cutoffs
        for i in range(len(lowerCamelCase		) - 1		):
          lowerCAmelCase_      :				Tuple      =       cutoff_values[i], cutoff_values[i + 1]

          if i == 0:
            lowerCAmelCase_      :				str      =       head_logprob[:, : self.cutoffs[0]]
          else:
            lowerCAmelCase_      :				Dict      =       weights[i], biases[i], self.out_projs[i]

            lowerCAmelCase_      :				Optional[int]      =       self._compute_logit(lowerCamelCase   ,						lowerCamelCase   ,						lowerCamelCase   ,						lowerCamelCase		)
            lowerCAmelCase_      :				Optional[int]      =       nn.functional.log_softmax(lowerCamelCase   ,						dim=1		)

            lowerCAmelCase_      :				Optional[int]      =       head_logprob[:, -i] + tail_logprob_i
            lowerCAmelCase_      :				List[str]      =       logprob_i

        return out







'''simple docstring'''






from ..utils import DummyObject, requires_backends






class        __snake_case	(       metaclass=_SCREAMING_SNAKE_CASE):







    """simple docstring"""



    lowercase       			= ['note_seq']

    def __init__(       self   :  Tuple   ,						*lowerCamelCase   :  Optional[int]   ,						**lowerCamelCase   :  Tuple		)       ->  Any:
      requires_backends(self   ,						["""note_seq"""]		)

    @classmethod
    def __lowercase       (       cls   :  Optional[Any]   ,						*lowerCamelCase   :  List[Any]   ,						**lowerCamelCase   :  Dict		)       ->  List[Any]:
      requires_backends(cls   ,						["""note_seq"""]		)





    @classmethod
    def __lowercase       (       cls   :  Union[str, Any]   ,						*lowerCamelCase   :  Union[str, Any]   ,						**lowerCamelCase   :  List[Any]		)       ->  Optional[int]:
      requires_backends(cls   ,						["""note_seq"""]		)



