"""simple docstring"""
import copy
import os
from typing import Union

from ...configuration_utils import PretrainedConfig
from ...models.auto.modeling_auto import MODEL_FOR_CAUSAL_LM_MAPPING_NAMES
from ...utils import logging
from ..auto import CONFIG_MAPPING


SCREAMING_SNAKE_CASE__   =					logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__   =					{
    "Salesforce/instruct-blip-flan-t5": "https://huggingface.co/Salesforce/instruct-blip-flan-t5/resolve/main/config.json",
}



class 					lowerCAmelCase_      (     lowerCAmelCase   ):



				"""simple docstring"""
				_lowerCAmelCase       :      str     					=  """instructblip_vision_model"""

				def __init__(			self						,      lowerCAmelCase=14_08						,      lowerCAmelCase=61_44						,      lowerCAmelCase=39						,      lowerCAmelCase=16						,      lowerCAmelCase=2_24						,      lowerCAmelCase=14						,      lowerCAmelCase="gelu"						,      lowerCAmelCase=1E-6						,      lowerCAmelCase=0.0						,      lowerCAmelCase=1E-10						,      lowerCAmelCase=True						,      **lowerCAmelCase						,      ):






										"""simple docstring"""

										super().__init__(**lowerCAmelCase      )

										snake_case        =     hidden_size
										snake_case        =     intermediate_size
										snake_case        =     num_hidden_layers
										snake_case        =     num_attention_heads
										snake_case        =     patch_size
										snake_case        =     image_size
										snake_case        =     initializer_range
										snake_case        =     attention_dropout
										snake_case        =     layer_norm_eps
										snake_case        =     hidden_act
										snake_case        =     qkv_bias

				@classmethod
				def 						snake_case	(			cls						,      lowerCAmelCase						,      **lowerCAmelCase      ):






										"""simple docstring"""

										cls._set_token_in_kwargs(lowerCAmelCase      )

										snake_case   ,snake_case        =     cls.get_config_dict(lowerCAmelCase						,      **lowerCAmelCase      )

										# get the vision config dict if we are loading from InstructBlipConfig
										if config_dict.get('model_type'      ) == "instructblip":
																snake_case        =     config_dict['vision_config']

										if "model_type" in config_dict and hasattr(cls						,      'model_type'      ) and config_dict["model_type"] != cls.model_type:
																logger.warning(
																    F"""You are using a model of type {config_dict["model_type"]} to instantiate a model of type """
																    F"""{cls.model_type}. This is not supported for all configurations of models and can yield errors."""      )

										return cls.from_dict(lowerCAmelCase						,      **lowerCAmelCase      )





class 					lowerCAmelCase_      (     lowerCAmelCase   ):



				"""simple docstring"""
				_lowerCAmelCase       :      str     					=  """instructblip_qformer"""

				def __init__(			self						,      lowerCAmelCase=3_05_22						,      lowerCAmelCase=7_68						,      lowerCAmelCase=12						,      lowerCAmelCase=12						,      lowerCAmelCase=30_72						,      lowerCAmelCase="gelu"						,      lowerCAmelCase=0.1						,      lowerCAmelCase=0.1						,      lowerCAmelCase=5_12						,      lowerCAmelCase=0.02						,      lowerCAmelCase=1E-12						,      lowerCAmelCase=0						,      lowerCAmelCase="absolute"						,      lowerCAmelCase=2						,      lowerCAmelCase=14_08						,      **lowerCAmelCase						,      ):






										"""simple docstring"""

										super().__init__(pad_token_id=lowerCAmelCase						,      **lowerCAmelCase      )

										snake_case        =     vocab_size
										snake_case        =     hidden_size
										snake_case        =     num_hidden_layers
										snake_case        =     num_attention_heads
										snake_case        =     hidden_act
										snake_case        =     intermediate_size
										snake_case        =     hidden_dropout_prob
										snake_case        =     attention_probs_dropout_prob
										snake_case        =     max_position_embeddings
										snake_case        =     initializer_range
										snake_case        =     layer_norm_eps
										snake_case        =     position_embedding_type
										snake_case        =     cross_attention_frequency
										snake_case        =     encoder_hidden_size

				@classmethod
				def 						snake_case	(			cls						,      lowerCAmelCase						,      **lowerCAmelCase      ):






										"""simple docstring"""

										cls._set_token_in_kwargs(lowerCAmelCase      )

										snake_case   ,snake_case        =     cls.get_config_dict(lowerCAmelCase						,      **lowerCAmelCase      )

										# get the qformer config dict if we are loading from InstructBlipConfig
										if config_dict.get('model_type'      ) == "instructblip":
																snake_case        =     config_dict['qformer_config']

										if "model_type" in config_dict and hasattr(cls						,      'model_type'      ) and config_dict["model_type"] != cls.model_type:
																logger.warning(
																    F"""You are using a model of type {config_dict["model_type"]} to instantiate a model of type """
																    F"""{cls.model_type}. This is not supported for all configurations of models and can yield errors."""      )

										return cls.from_dict(lowerCAmelCase						,      **lowerCAmelCase      )





class 					lowerCAmelCase_      (     lowerCAmelCase   ):



				"""simple docstring"""
				_lowerCAmelCase       :      Union[str, Any]     					=  """instructblip"""
				_lowerCAmelCase       :      Tuple     					=  True

				def __init__(			self						,      lowerCAmelCase=None						,      lowerCAmelCase=None						,      lowerCAmelCase=None						,      lowerCAmelCase=32						,      **lowerCAmelCase      ):






										"""simple docstring"""

										super().__init__(**lowerCAmelCase      )

										if vision_config is None:
																snake_case        =     {}
																logger.info('vision_config is None. initializing the InstructBlipVisionConfig with default values.'      )

										if qformer_config is None:
																snake_case        =     {}
																logger.info('qformer_config is None. Initializing the InstructBlipQFormerConfig with default values.'      )

										if text_config is None:
																snake_case        =     {}
																logger.info('text_config is None. Initializing the text config with default values (`OPTConfig`).'      )

										snake_case        =     InstructBlipVisionConfig(**lowerCAmelCase      )
										snake_case        =     InstructBlipQFormerConfig(**lowerCAmelCase      )
										snake_case        =     text_config['model_type'] if 'model_type' in text_config else 'opt'
										snake_case        =     CONFIG_MAPPING[text_model_type](**lowerCAmelCase      )

										snake_case        =     self.text_config.tie_word_embeddings
										snake_case        =     self.text_config.is_encoder_decoder

										snake_case        =     num_query_tokens
										snake_case        =     self.vision_config.hidden_size
										snake_case        =     self.text_config.model_type in MODEL_FOR_CAUSAL_LM_MAPPING_NAMES
										snake_case        =     1.0
										snake_case        =     0.02

				@classmethod
				def 						snake_case	(			cls						,      lowerCAmelCase						,      lowerCAmelCase						,      lowerCAmelCase						,      **lowerCAmelCase						,      ):






										"""simple docstring"""

										return cls(
										    vision_config=vision_config.to_dict()						,      qformer_config=qformer_config.to_dict()						,      text_config=text_config.to_dict()						,      **lowerCAmelCase						,      )

				def 						snake_case	(			self      ):






										"""simple docstring"""

										snake_case        =     copy.deepcopy(self.__dict__      )
										snake_case        =     self.vision_config.to_dict()
										snake_case        =     self.qformer_config.to_dict()
										snake_case        =     self.text_config.to_dict()
										snake_case        =     self.__class__.model_type
										return output



"""simple docstring"""
from typing import Optional

import numpy as np
import torch
from torch import nn
from transformers import GPTaConfig, GPTaLMHeadModel
from transformers.modeling_utils import ModuleUtilsMixin

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin



class 					lowerCAmelCase_      (     lowerCAmelCase       ,      lowerCAmelCase       ,      lowerCAmelCase   ):



				"""simple docstring"""
				_lowerCAmelCase       :      int     					=  [r"""h\.\d+\.attn\.bias""", r"""h\.\d+\.attn\.masked_bias"""]

				@register_to_config
				def __init__(			self						,      lowerCAmelCase						,      lowerCAmelCase						,      lowerCAmelCase = None						,      lowerCAmelCase = 5_02_57						,      lowerCAmelCase = 10_24						,      lowerCAmelCase = 7_68						,      lowerCAmelCase = 12						,      lowerCAmelCase = 12						,      lowerCAmelCase = None						,      lowerCAmelCase = "gelu_new"						,      lowerCAmelCase = 0.1						,      lowerCAmelCase = 0.1						,      lowerCAmelCase = 0.1						,      lowerCAmelCase = 1E-5						,      lowerCAmelCase = 0.02						,      lowerCAmelCase = True						,      lowerCAmelCase = True						,      lowerCAmelCase = False						,      lowerCAmelCase = False						,      ):






										"""simple docstring"""

										super().__init__()

										snake_case        =     prefix_length

										if prefix_inner_dim != n_embd and prefix_hidden_dim is None:
																raise ValueError(
																    F"""`prefix_hidden_dim` cannot be `None` when `prefix_inner_dim`: {prefix_hidden_dim} and"""
																    F""" `n_embd`: {n_embd} are not equal."""      )

										snake_case        =     prefix_inner_dim
										snake_case        =     prefix_hidden_dim

										snake_case        =     (
										    nn.Linear(self.prefix_inner_dim						,      self.prefix_hidden_dim      )
										    if self.prefix_hidden_dim is not None
										    else nn.Identity()
										)
										snake_case        =     (
										    nn.Linear(self.prefix_hidden_dim						,      lowerCAmelCase      ) if self.prefix_hidden_dim is not None else nn.Identity()
										)

										snake_case        =     GPTaConfig(
										    vocab_size=lowerCAmelCase						,      n_positions=lowerCAmelCase						,      n_embd=lowerCAmelCase						,      n_layer=lowerCAmelCase						,      n_head=lowerCAmelCase						,      n_inner=lowerCAmelCase						,      activation_function=lowerCAmelCase						,      resid_pdrop=lowerCAmelCase						,      embd_pdrop=lowerCAmelCase						,      attn_pdrop=lowerCAmelCase						,      layer_norm_epsilon=lowerCAmelCase						,      initializer_range=lowerCAmelCase						,      scale_attn_weights=lowerCAmelCase						,      use_cache=lowerCAmelCase						,      scale_attn_by_inverse_layer_idx=lowerCAmelCase						,      reorder_and_upcast_attn=lowerCAmelCase						,      )
										snake_case        =     GPTaLMHeadModel(lowerCAmelCase      )

				def 						snake_case	(			self						,      lowerCAmelCase						,      lowerCAmelCase						,      lowerCAmelCase = None						,      lowerCAmelCase = None						,      ):






										"""simple docstring"""

										snake_case        =     self.transformer.transformer.wte(lowerCAmelCase      )
										snake_case        =     self.encode_prefix(lowerCAmelCase      )
										snake_case        =     self.decode_prefix(lowerCAmelCase      )
										snake_case        =     torch.cat((prefix_embeds, embedding_text)						,      dim=1      )

										if labels is not None:
																snake_case        =     self.get_dummy_token(input_ids.shape[0]						,      input_ids.device      )
																snake_case        =     torch.cat((dummy_token, input_ids)						,      dim=1      )
										snake_case        =     self.transformer(inputs_embeds=lowerCAmelCase						,      labels=lowerCAmelCase						,      attention_mask=lowerCAmelCase      )
										if self.prefix_hidden_dim is not None:
																return out, hidden
										else:
																return out

				def 						snake_case	(			self						,      lowerCAmelCase						,      lowerCAmelCase      ):






										"""simple docstring"""

										return torch.zeros(lowerCAmelCase						,      self.prefix_length						,      dtype=torch.intaa						,      device=lowerCAmelCase      )

				def 						snake_case	(			self						,      lowerCAmelCase      ):






										"""simple docstring"""

										return self.encode_prefix(lowerCAmelCase      )

				@torch.no_grad()
				def 						snake_case	(			self						,      lowerCAmelCase						,      lowerCAmelCase						,      lowerCAmelCase      ):






										"""simple docstring"""

										snake_case        =     torch.split(lowerCAmelCase						,      1						,      dim=0      )
										snake_case        =     []
										snake_case        =     []
										for feature in features:
																snake_case        =     self.decode_prefix(feature.to(lowerCAmelCase      )      )  # back to the clip feature
																# Only support beam search for now
																snake_case   ,snake_case        =     self.generate_beam(
																    input_embeds=lowerCAmelCase						,      device=lowerCAmelCase						,      eos_token_id=lowerCAmelCase      )
																generated_tokens.append(output_tokens[0]      )
																generated_seq_lengths.append(seq_lengths[0]      )
										snake_case        =     torch.stack(lowerCAmelCase      )
										snake_case        =     torch.stack(lowerCAmelCase      )
										return generated_tokens, generated_seq_lengths

				@torch.no_grad()
				def 						snake_case	(			self						,      lowerCAmelCase=None						,      lowerCAmelCase=None						,      lowerCAmelCase=None						,      lowerCAmelCase = 5						,      lowerCAmelCase = 67						,      lowerCAmelCase = 1.0						,      lowerCAmelCase = None						,      ):






										"""simple docstring"""

										snake_case        =     eos_token_id
										snake_case        =     None
										snake_case        =     None
										snake_case        =     torch.ones(lowerCAmelCase						,      device=lowerCAmelCase						,      dtype=torch.int      )
										snake_case        =     torch.zeros(lowerCAmelCase						,      device=lowerCAmelCase						,      dtype=torch.bool      )

										if input_embeds is not None:
																snake_case        =     input_embeds
										else:
																snake_case        =     self.transformer.transformer.wte(lowerCAmelCase      )

										for i in range(lowerCAmelCase      ):
																snake_case        =     self.transformer(inputs_embeds=lowerCAmelCase      )
																snake_case        =     outputs.logits
																snake_case        =     logits[:, -1, :] / (temperature if temperature > 0 else 1.0)
																snake_case        =     logits.softmax(-1      ).log()

																if scores is None:
																						snake_case   ,snake_case        =     logits.topk(lowerCAmelCase						,      -1      )
																						snake_case        =     generated.expand(lowerCAmelCase						,      *generated.shape[1:]      )
																						snake_case   ,snake_case        =     next_tokens.permute(1						,      0      ), scores.squeeze(0      )
																						if tokens is None:
																												snake_case        =     next_tokens
																						else:
																												snake_case        =     tokens.expand(lowerCAmelCase						,      *tokens.shape[1:]      )
																												snake_case        =     torch.cat((tokens, next_tokens)						,      dim=1      )
																else:
																						snake_case        =     -float(np.inf      )
																						snake_case        =     0
																						snake_case        =     scores[:, None] + logits
																						seq_lengths[~is_stopped] += 1
																						snake_case        =     scores_sum / seq_lengths[:, None]
																						snake_case   ,snake_case        =     scores_sum_average.view(-1      ).topk(lowerCAmelCase						,      -1      )
																						snake_case        =     next_tokens // scores_sum.shape[1]
																						snake_case        =     seq_lengths[next_tokens_source]
																						snake_case        =     next_tokens % scores_sum.shape[1]
																						snake_case        =     next_tokens.unsqueeze(1      )
																						snake_case        =     tokens[next_tokens_source]
																						snake_case        =     torch.cat((tokens, next_tokens)						,      dim=1      )
																						snake_case        =     generated[next_tokens_source]
																						snake_case        =     scores_sum_average * seq_lengths
																						snake_case        =     is_stopped[next_tokens_source]

																snake_case        =     self.transformer.transformer.wte(next_tokens.squeeze()      ).view(generated.shape[0]						,      1						,      -1      )
																snake_case        =     torch.cat((generated, next_token_embed)						,      dim=1      )
																snake_case        =     is_stopped + next_tokens.eq(lowerCAmelCase      ).squeeze()
																if is_stopped.all():
																						break

										snake_case        =     scores / seq_lengths
										snake_case        =     scores.argsort(descending=lowerCAmelCase      )
										# tokens tensors are already padded to max_seq_length
										snake_case        =     [tokens[i] for i in order]
										snake_case        =     torch.stack(lowerCAmelCase						,      dim=0      )
										snake_case        =     torch.tensor([seq_lengths[i] for i in order]						,      dtype=seq_lengths.dtype      )
										return output_texts, seq_lengths


