
"""simple docstring"""





import math

import torch
from torch import nn

from ..configuration_utils import ConfigMixin, register_to_config
from .attention_processor import Attention
from .embeddings import get_timestep_embedding
from .modeling_utils import ModelMixin






class    _UpperCAmelCase(   lowerCamelCase				,					lowerCamelCase     ):



		@register_to_config
		def __init__( self    ,		__a = 1_28    ,		__a = 2_56    ,		__a = 2000.0    ,		__a = 7_68    ,		__a = 12    ,		__a = 12    ,		__a = 64    ,		__a = 20_48    ,		__a = 0.1    ,		)	->	Tuple:






							'''simple docstring'''
							super().__init__()

							_UpperCamelCase										=  nn.Sequential(
							    nn.Linear(__a    ,		d_model * 4    ,		bias=__a)    ,		nn.SiLU()    ,		nn.Linear(d_model * 4    ,		d_model * 4    ,		bias=__a)    ,		nn.SiLU()    ,		)

							_UpperCamelCase										=  nn.Embedding(__a    ,		__a)
							_UpperCamelCase										=  False

							_UpperCamelCase										=  nn.Linear(__a    ,		__a    ,		bias=__a)

							_UpperCamelCase										=  nn.Dropout(p=__a)

							_UpperCamelCase										=  nn.ModuleList()
							for lyr_num in range(__a):
												# FiLM conditional T5 decoder
												_UpperCamelCase										=  DecoderLayer(d_model=__a    ,		d_kv=__a    ,		num_heads=__a    ,		d_ff=__a    ,		dropout_rate=__a)
												self.decoders.append(__a)

							_UpperCamelCase										=  TaLayerNorm(__a)

							_UpperCamelCase										=  nn.Dropout(p=__a)
							_UpperCamelCase										=  nn.Linear(__a    ,		__a    ,		bias=__a)



		def 						UpperCAmelCase							( self    ,		__a    ,		__a)	->	str:






							'''simple docstring'''
							_UpperCamelCase										=  torch.mul(query_input.unsqueeze(-1)    ,		key_input.unsqueeze(-2))
							return mask.unsqueeze(-3)




		def 						UpperCAmelCase							( self    ,		__a    ,		__a    ,		__a)	->	List[str]:






							'''simple docstring'''
							_UpperCamelCase			,  _UpperCamelCase			,  _UpperCamelCase										=  decoder_input_tokens.shape
							assert decoder_noise_time.shape == (batch,)

							# decoder_noise_time is in [0, 1), so rescale to expected timing range.
							_UpperCamelCase										=  get_timestep_embedding(
							    decoder_noise_time * self.config.max_decoder_noise_time    ,		embedding_dim=self.config.d_model    ,		max_period=self.config.max_decoder_noise_time    ,		).to(dtype=self.dtype)

							_UpperCamelCase										=  self.conditioning_emb(__a).unsqueeze(1)

							assert conditioning_emb.shape == (batch, 1, self.config.d_model * 4)

							_UpperCamelCase										=  decoder_input_tokens.shape[1]

							# If we want to use relative positions for audio context, we can just offset
							# this sequence by the length of encodings_and_masks.
							_UpperCamelCase										=  torch.broadcast_to(
							    torch.arange(__a    ,		device=decoder_input_tokens.device)    ,		(batch, seq_length)    ,		)

							_UpperCamelCase										=  self.position_encoding(__a)

							_UpperCamelCase										=  self.continuous_inputs_projection(__a)
							inputs += position_encodings
							_UpperCamelCase										=  self.dropout(__a)

							# decoder: No padding present.
							_UpperCamelCase										=  torch.ones(
							    decoder_input_tokens.shape[:2]    ,		device=decoder_input_tokens.device    ,		dtype=inputs.dtype)

							# Translate encoding masks to encoder-decoder masks.
							_UpperCamelCase										=  [(x, self.encoder_decoder_mask(__a    ,		__a)) for x, y in encodings_and_masks]

							# cross attend style: concat encodings
							_UpperCamelCase										=  torch.cat([x[0] for x in encodings_and_encdec_masks]    ,		dim=1)
							_UpperCamelCase										=  torch.cat([x[1] for x in encodings_and_encdec_masks]    ,		dim=-1)

							for lyr in self.decoders:
												_UpperCamelCase										=  lyr(
												    __a    ,		conditioning_emb=__a    ,		encoder_hidden_states=__a    ,		encoder_attention_mask=__a    ,		)[0]

							_UpperCamelCase										=  self.decoder_norm(__a)
							_UpperCamelCase										=  self.post_dropout(__a)

							_UpperCamelCase										=  self.spec_out(__a)
							return spec_out






class    _UpperCAmelCase(   nn.Module     ):



		def __init__( self    ,		__a    ,		__a    ,		__a    ,		__a    ,		__a    ,		__a=1e-6)	->	List[str]:






							'''simple docstring'''
							super().__init__()
							_UpperCamelCase										=  nn.ModuleList()

							# cond self attention: layer 0
							self.layer.append(
							    TaLayerSelfAttentionCond(d_model=__a    ,		d_kv=__a    ,		num_heads=__a    ,		dropout_rate=__a))

							# cross attention: layer 1
							self.layer.append(
							    TaLayerCrossAttention(
							        d_model=__a    ,		d_kv=__a    ,		num_heads=__a    ,		dropout_rate=__a    ,		layer_norm_epsilon=__a    ,		))

							# Film Cond MLP + dropout: last layer
							self.layer.append(
							    TaLayerFFCond(d_model=__a    ,		d_ff=__a    ,		dropout_rate=__a    ,		layer_norm_epsilon=__a))




		def 						UpperCAmelCase							( self    ,		__a    ,		__a=None    ,		__a=None    ,		__a=None    ,		__a=None    ,		__a=None    ,		)	->	str:






							'''simple docstring'''
							_UpperCamelCase										=  self.layer[0](
							    __a    ,		conditioning_emb=__a    ,		attention_mask=__a    ,		)

							if encoder_hidden_states is not None:
												_UpperCamelCase										=  torch.where(encoder_attention_mask > 0    ,		0    ,		-1e10).to(
												    encoder_hidden_states.dtype)

												_UpperCamelCase										=  self.layer[1](
												    __a    ,		key_value_states=__a    ,		attention_mask=__a    ,		)

							# Apply Film Conditional Feed Forward layer
							_UpperCamelCase										=  self.layer[-1](__a    ,		__a)

							return (hidden_states,)






class    _UpperCAmelCase(   nn.Module     ):



		def __init__( self    ,		__a    ,		__a    ,		__a    ,		__a)	->	Tuple:






							'''simple docstring'''
							super().__init__()
							_UpperCamelCase										=  TaLayerNorm(__a)
							_UpperCamelCase										=  TaFiLMLayer(in_features=d_model * 4    ,		out_features=__a)
							_UpperCamelCase										=  Attention(query_dim=__a    ,		heads=__a    ,		dim_head=__a    ,		out_bias=__a    ,		scale_qk=__a)
							_UpperCamelCase										=  nn.Dropout(__a)




		def 						UpperCAmelCase							( self    ,		__a    ,		__a=None    ,		__a=None    ,		)	->	int:






							'''simple docstring'''
							# pre_self_attention_layer_norm
							_UpperCamelCase										=  self.layer_norm(__a)

							if conditioning_emb is not None:
												_UpperCamelCase										=  self.FiLMLayer(__a    ,		__a)

							# Self-attention block
							_UpperCamelCase										=  self.attention(__a)

							_UpperCamelCase										=  hidden_states + self.dropout(__a)

							return hidden_states






class    _UpperCAmelCase(   nn.Module     ):



		def __init__( self    ,		__a    ,		__a    ,		__a    ,		__a    ,		__a)	->	Optional[Any]:






							'''simple docstring'''
							super().__init__()
							_UpperCamelCase										=  Attention(query_dim=__a    ,		heads=__a    ,		dim_head=__a    ,		out_bias=__a    ,		scale_qk=__a)
							_UpperCamelCase										=  TaLayerNorm(__a    ,		eps=__a)
							_UpperCamelCase										=  nn.Dropout(__a)




		def 						UpperCAmelCase							( self    ,		__a    ,		__a=None    ,		__a=None    ,		)	->	Optional[Any]:






							'''simple docstring'''
							_UpperCamelCase										=  self.layer_norm(__a)
							_UpperCamelCase										=  self.attention(
							    __a    ,		encoder_hidden_states=__a    ,		attention_mask=attention_mask.squeeze(1)    ,		)
							_UpperCamelCase										=  hidden_states + self.dropout(__a)
							return layer_output






class    _UpperCAmelCase(   nn.Module     ):



		def __init__( self    ,		__a    ,		__a    ,		__a    ,		__a)	->	Optional[Any]:






							'''simple docstring'''
							super().__init__()
							_UpperCamelCase										=  TaDenseGatedActDense(d_model=__a    ,		d_ff=__a    ,		dropout_rate=__a)
							_UpperCamelCase										=  TaFiLMLayer(in_features=d_model * 4    ,		out_features=__a)
							_UpperCamelCase										=  TaLayerNorm(__a    ,		eps=__a)
							_UpperCamelCase										=  nn.Dropout(__a)




		def 						UpperCAmelCase							( self    ,		__a    ,		__a=None)	->	Optional[int]:






							'''simple docstring'''
							_UpperCamelCase										=  self.layer_norm(__a)
							if conditioning_emb is not None:
												_UpperCamelCase										=  self.film(__a    ,		__a)

							_UpperCamelCase										=  self.DenseReluDense(__a)
							_UpperCamelCase										=  hidden_states + self.dropout(__a)
							return hidden_states






class    _UpperCAmelCase(   nn.Module     ):



		def __init__( self    ,		__a    ,		__a    ,		__a)	->	List[Any]:






							'''simple docstring'''
							super().__init__()
							_UpperCamelCase										=  nn.Linear(__a    ,		__a    ,		bias=__a)
							_UpperCamelCase										=  nn.Linear(__a    ,		__a    ,		bias=__a)
							_UpperCamelCase										=  nn.Linear(__a    ,		__a    ,		bias=__a)
							_UpperCamelCase										=  nn.Dropout(__a)
							_UpperCamelCase										=  NewGELUActivation()




		def 						UpperCAmelCase							( self    ,		__a)	->	Union[str, Any]:






							'''simple docstring'''
							_UpperCamelCase										=  self.act(self.wi_a(__a))
							_UpperCamelCase										=  self.wi_a(__a)
							_UpperCamelCase										=  hidden_gelu * hidden_linear
							_UpperCamelCase										=  self.dropout(__a)

							_UpperCamelCase										=  self.wo(__a)
							return hidden_states






class    _UpperCAmelCase(   nn.Module     ):



		def __init__( self    ,		__a    ,		__a=1e-6)	->	Optional[int]:






							'''simple docstring'''
							super().__init__()
							_UpperCamelCase										=  nn.Parameter(torch.ones(__a))
							_UpperCamelCase										=  eps




		def 						UpperCAmelCase							( self    ,		__a)	->	Union[str, Any]:






							'''simple docstring'''
							# T5 uses a layer_norm which only scales and doesn't shift, which is also known as Root Mean
							# Square Layer Normalization https://arxiv.org/abs/1910.07467 thus variance is calculated
							# w/o mean and there is no bias. Additionally we want to make sure that the accumulation for
							# half-precision inputs is done in fp32

							_UpperCamelCase										=  hidden_states.to(torch.floataa).pow(2).mean(-1    ,		keepdim=__a)
							_UpperCamelCase										=  hidden_states * torch.rsqrt(variance + self.variance_epsilon)

							# convert into half-precision if necessary
							if self.weight.dtype in [torch.floataa, torch.bfloataa]:
												_UpperCamelCase										=  hidden_states.to(self.weight.dtype)

							return self.weight * hidden_states






class    _UpperCAmelCase(   nn.Module     ):



		def 						UpperCAmelCase							( self    ,		__a)	->	torch.Tensor:






							'''simple docstring'''
							return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi) * (input + 0.04_4715 * torch.pow(__a    ,		3.0))))






class    _UpperCAmelCase(   nn.Module     ):



		def __init__( self    ,		__a    ,		__a)	->	Optional[int]:






							'''simple docstring'''
							super().__init__()
							_UpperCamelCase										=  nn.Linear(__a    ,		out_features * 2    ,		bias=__a)




		def 						UpperCAmelCase							( self    ,		__a    ,		__a)	->	Tuple:






							'''simple docstring'''
							_UpperCamelCase										=  self.scale_bias(__a)
							_UpperCamelCase			,  _UpperCamelCase										=  torch.chunk(__a    ,		2    ,		-1)
							_UpperCamelCase										=  x * (1 + scale) + shift
							return x








"""simple docstring"""





from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


_a														=		logging.get_logger(__name__)

_a														=		{
    """bert-base-uncased""": """https://huggingface.co/bert-base-uncased/resolve/main/config.json""",
    """bert-large-uncased""": """https://huggingface.co/bert-large-uncased/resolve/main/config.json""",
    """bert-base-cased""": """https://huggingface.co/bert-base-cased/resolve/main/config.json""",
    """bert-large-cased""": """https://huggingface.co/bert-large-cased/resolve/main/config.json""",
    """bert-base-multilingual-uncased""": """https://huggingface.co/bert-base-multilingual-uncased/resolve/main/config.json""",
    """bert-base-multilingual-cased""": """https://huggingface.co/bert-base-multilingual-cased/resolve/main/config.json""",
    """bert-base-chinese""": """https://huggingface.co/bert-base-chinese/resolve/main/config.json""",
    """bert-base-german-cased""": """https://huggingface.co/bert-base-german-cased/resolve/main/config.json""",
    """bert-large-uncased-whole-word-masking""": (
        """https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/config.json"""
    ),
    """bert-large-cased-whole-word-masking""": (
        """https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/config.json"""
    ),
    """bert-large-uncased-whole-word-masking-finetuned-squad""": (
        """https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/config.json"""
    ),
    """bert-large-cased-whole-word-masking-finetuned-squad""": (
        """https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/config.json"""
    ),
    """bert-base-cased-finetuned-mrpc""": """https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/config.json""",
    """bert-base-german-dbmdz-cased""": """https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/config.json""",
    """bert-base-german-dbmdz-uncased""": """https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/config.json""",
    """cl-tohoku/bert-base-japanese""": """https://huggingface.co/cl-tohoku/bert-base-japanese/resolve/main/config.json""",
    """cl-tohoku/bert-base-japanese-whole-word-masking""": (
        """https://huggingface.co/cl-tohoku/bert-base-japanese-whole-word-masking/resolve/main/config.json"""
    ),
    """cl-tohoku/bert-base-japanese-char""": (
        """https://huggingface.co/cl-tohoku/bert-base-japanese-char/resolve/main/config.json"""
    ),
    """cl-tohoku/bert-base-japanese-char-whole-word-masking""": (
        """https://huggingface.co/cl-tohoku/bert-base-japanese-char-whole-word-masking/resolve/main/config.json"""
    ),
    """TurkuNLP/bert-base-finnish-cased-v1""": (
        """https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/config.json"""
    ),
    """TurkuNLP/bert-base-finnish-uncased-v1""": (
        """https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/config.json"""
    ),
    """wietsedv/bert-base-dutch-cased""": """https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/config.json""",
    # See all BERT models at https://huggingface.co/models?filter=bert
}






class    _UpperCAmelCase(   lowerCamelCase     ):
		lowercase__      =				'bert'



		def __init__( self    ,		__a=3_05_22    ,		__a=7_68    ,		__a=12    ,		__a=12    ,		__a=30_72    ,		__a="gelu"    ,		__a=0.1    ,		__a=0.1    ,		__a=5_12    ,		__a=2    ,		__a=0.02    ,		__a=1e-12    ,		__a=0    ,		__a="absolute"    ,		__a=True    ,		__a=None    ,		**__a    ,		)	->	Optional[Any]:






							'''simple docstring'''
							super().__init__(pad_token_id=__a    ,		**__a)

							_UpperCamelCase										=  vocab_size
							_UpperCamelCase										=  hidden_size
							_UpperCamelCase										=  num_hidden_layers
							_UpperCamelCase										=  num_attention_heads
							_UpperCamelCase										=  hidden_act
							_UpperCamelCase										=  intermediate_size
							_UpperCamelCase										=  hidden_dropout_prob
							_UpperCamelCase										=  attention_probs_dropout_prob
							_UpperCamelCase										=  max_position_embeddings
							_UpperCamelCase										=  type_vocab_size
							_UpperCamelCase										=  initializer_range
							_UpperCamelCase										=  layer_norm_eps
							_UpperCamelCase										=  position_embedding_type
							_UpperCamelCase										=  use_cache
							_UpperCamelCase										=  classifier_dropout






class    _UpperCAmelCase(   lowerCamelCase     ):



		@property
		def 						UpperCAmelCase							( self)	->	Mapping[str, Mapping[int, str]]:






							'''simple docstring'''
							if self.task == "multiple-choice":
												_UpperCamelCase										=  {0: '''batch''', 1: '''choice''', 2: '''sequence'''}
							else:
												_UpperCamelCase										=  {0: '''batch''', 1: '''sequence'''}
							return OrderedDict(
							    [
							        ('''input_ids''', dynamic_axis),
							        ('''attention_mask''', dynamic_axis),
							        ('''token_type_ids''', dynamic_axis),
							    ])






