
from typing import Optional

from torch import nn

from .transformer_ad import TransformeraDModel, TransformeraDModelOutput







class 					lowerCamelCase		(  nn.Module  ):





     def __init__(self				:   List[str]    ,   _A				:   int = 1_6    ,   _A				:   int = 8_8    ,   _A				:   Optional[int] = None    ,   _A				:   int = 1    ,   _A				:   float = 0.0    ,   _A				:   int = 3_2    ,   _A				:   Optional[int] = None    ,   _A				:   bool = False    ,   _A				:   Optional[int] = None    ,   _A				:   Optional[int] = None    ,   _A				:   str = "geglu"    ,   _A				:   Optional[int] = None    ,   )						->						List[Any]:
         super().__init__()
         snake_case								= nn.ModuleList(
             [
                 TransformeraDModel(
                     num_attention_heads=_A    ,   attention_head_dim=_A    ,   in_channels=_A    ,   num_layers=_A    ,   dropout=_A    ,   norm_num_groups=_A    ,   cross_attention_dim=_A    ,   attention_bias=_A    ,   sample_size=_A    ,   num_vector_embeds=_A    ,   activation_fn=_A    ,   num_embeds_ada_norm=_A    ,   )
                 for _ in range(2       )
             ]       )

         # Variables that can be set by a pipeline:

         # The ratio of transformer1 to transformer2's output states to be combined during inference
         snake_case								= 0.5

         # The shape of `encoder_hidden_states` is expected to be
         # `(batch_size, condition_lengths[0]+condition_lengths[1], num_features)`
         snake_case								= [7_7, 2_5_7]

         # Which transformer to use to encode which condition.
         # E.g. `(1, 0)` means that we'll use `transformers[1](conditions[0])` and `transformers[0](conditions[1])`
         snake_case								= [1, 0]





     def 		UpperCAmelCase(self				:   Optional[int]    ,   _A				:   Tuple    ,   _A				:   Any    ,   _A				:   List[Any]=None    ,   _A				:   Optional[int]=None    ,   _A				:   Tuple=None    ,   _A				:   bool = True    ,   )						->						Optional[Any]:
         snake_case								= hidden_states

         snake_case								= []
         snake_case								= 0
         # attention_mask is not used yet
         for i in range(2       ):
             # for each of the two transformers, pass the corresponding condition tokens
             snake_case								= encoder_hidden_states[:, tokens_start : tokens_start + self.condition_lengths[i]]
             snake_case								= self.transformer_index_for_condition[i]
             snake_case								= self.transformers[transformer_index](
                 _A    ,   encoder_hidden_states=_A    ,   timestep=_A    ,   cross_attention_kwargs=_A    ,   return_dict=_A    ,   )[0]
             encoded_states.append(encoded_state - input_states       )
             tokens_start += self.condition_lengths[i]

         snake_case								= encoded_states[0] * self.mix_ratio + encoded_states[1] * (1 - self.mix_ratio)
         snake_case								= output_states + input_states

         if not return_dict:
             return (output_states,)

         return TransformeraDModelOutput(sample=_A       )



from ...configuration_utils import PretrainedConfig
from ...utils import logging


_A   =			logging.get_logger(__name__)

_A   =			{
    "google/realm-cc-news-pretrained-embedder": (
        "https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/config.json"
    ),
    "google/realm-cc-news-pretrained-encoder": (
        "https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/config.json"
    ),
    "google/realm-cc-news-pretrained-scorer": (
        "https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/config.json"
    ),
    "google/realm-cc-news-pretrained-openqa": (
        "https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/config.json"
    ),
    "google/realm-orqa-nq-openqa": "https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/config.json",
    "google/realm-orqa-nq-reader": "https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/config.json",
    "google/realm-orqa-wq-openqa": "https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/config.json",
    "google/realm-orqa-wq-reader": "https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/config.json",
    # See all REALM models at https://huggingface.co/models?filter=realm
}







class 					lowerCamelCase		(  A_  ):
     UpperCAmelCase__    :							str									=		"realm"





     def __init__(self				:   Optional[int]    ,   _A				:   Optional[Any]=3_0_5_2_2    ,   _A				:   Tuple=7_6_8    ,   _A				:   List[str]=1_2_8    ,   _A				:   Optional[Any]=1_2    ,   _A				:   Dict=1_2    ,   _A				:   Tuple=8    ,   _A				:   Dict=3_0_7_2    ,   _A				:   Union[str, Any]="gelu_new"    ,   _A				:   Any=0.1    ,   _A				:   int=0.1    ,   _A				:   Union[str, Any]=5_1_2    ,   _A				:   List[str]=2    ,   _A				:   Any=0.02    ,   _A				:   int=1E-12    ,   _A				:   Tuple=2_5_6    ,   _A				:   Optional[Any]=1_0    ,   _A				:   Any=1E-3    ,   _A				:   int=5    ,   _A				:   int=3_2_0    ,   _A				:   Dict=1_3_3_5_3_7_1_8    ,   _A				:   Any=5_0_0_0    ,   _A				:   Union[str, Any]=1    ,   _A				:   Dict=0    ,   _A				:   int=2    ,   **_A				:   Union[str, Any]    ,   )						->						Optional[Any]:
         super().__init__(pad_token_id=_A    ,   bos_token_id=_A    ,   eos_token_id=_A    ,   **_A       )

         # Common config
         snake_case								= vocab_size
         snake_case								= max_position_embeddings
         snake_case								= hidden_size
         snake_case								= retriever_proj_size
         snake_case								= num_hidden_layers
         snake_case								= num_attention_heads
         snake_case								= num_candidates
         snake_case								= intermediate_size
         snake_case								= hidden_act
         snake_case								= hidden_dropout_prob
         snake_case								= attention_probs_dropout_prob
         snake_case								= initializer_range
         snake_case								= type_vocab_size
         snake_case								= layer_norm_eps

         # Reader config
         snake_case								= span_hidden_size
         snake_case								= max_span_width
         snake_case								= reader_layer_norm_eps
         snake_case								= reader_beam_size
         snake_case								= reader_seq_len

         # Retrieval config
         snake_case								= num_block_records
         snake_case								= searcher_beam_size

