





"""simple docstring"""




import itertools
import os
import random
import tempfile
import unittest

import numpy as np
from datasets import load_dataset

from transformers import is_speech_available
from transformers.testing_utils import check_json_file_has_correct_format, require_torch, require_torchaudio
from transformers.utils.import_utils import is_torch_available

from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin


if is_speech_available():
     from transformers import WhisperFeatureExtractor

if is_torch_available():
     import torch

UpperCAmelCase     :     Any							=	random.Random()
def 					_A	(		SCREAMING_SNAKE_CASE					:       Union[str, Any]	,	SCREAMING_SNAKE_CASE					:       Union[str, Any]=1.0	,	SCREAMING_SNAKE_CASE					:       List[Any]=None	,	SCREAMING_SNAKE_CASE					:       Optional[int]=None						):





  """simple docstring"""



  if rng is None:
    a__     : Union[str, Any]  =global_rng

  a__     : Tuple  =[]
  for batch_idx in range(shape[0]						):
    values.append([]						)
    for _ in range(shape[1]						):
      values[-1].append(rng.random() * scale						)

  return values







@require_torch
@require_torchaudio
class     __lowerCAmelCase	(   unittest.TestCase):







      def __init__(   self		,				lowerCAmelCase__		,				lowerCAmelCase__=7		,				lowerCAmelCase__=4_0_0		,				lowerCAmelCase__=2_0_0_0		,				lowerCAmelCase__=1_0		,				lowerCAmelCase__=1_6_0		,				lowerCAmelCase__=8		,				lowerCAmelCase__=0.0		,				lowerCAmelCase__=4_0_0_0		,				lowerCAmelCase__=False		,				lowerCAmelCase__=True		,				)       ->						Any:



        '''simple docstring'''




        a__     : str  =parent
        a__     : List[str]  =batch_size
        a__     : Any  =min_seq_length
        a__     : Tuple  =max_seq_length
        a__     : Dict  =(self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
        a__     : Any  =padding_value
        a__     : Optional[Any]  =sampling_rate
        a__     : Optional[int]  =return_attention_mask
        a__     : Optional[Any]  =do_normalize
        a__     : Optional[int]  =feature_size
        a__     : Tuple  =chunk_length
        a__     : Optional[int]  =hop_length







      def        _lowercase  (   self							)       ->						int:



        '''simple docstring'''




        return {
            "feature_size": self.feature_size,
            "hop_length": self.hop_length,
            "chunk_length": self.chunk_length,
            "padding_value": self.padding_value,
            "sampling_rate": self.sampling_rate,
            "return_attention_mask": self.return_attention_mask,
            "do_normalize": self.do_normalize,
        }







      def        _lowercase  (   self		,				lowerCAmelCase__=False		,				lowerCAmelCase__=False							)       ->						List[Any]:



        '''simple docstring'''




        def _flatten(lowerCAmelCase__							):
          return list(itertools.chain(*lowerCAmelCase__							)							)

        if equal_length:
          a__     : Optional[int]  =[floats_list((self.max_seq_length, self.feature_size)							) for _ in range(self.batch_size							)]
        else:
          # make sure that inputs increase in size
          a__     : List[Any]  =[
              floats_list((x, self.feature_size)							)
              for x in range(self.min_seq_length		,				self.max_seq_length		,				self.seq_length_diff							)
          ]
        if numpify:
          a__     : Tuple  =[np.asarray(lowerCAmelCase__							) for x in speech_inputs]
        return speech_inputs







@require_torch
@require_torchaudio
class     __lowerCAmelCase	(   __lowercase						,    unittest.TestCase):
      _lowercase    : Dict            =      WhisperFeatureExtractor if is_speech_available() else None







      def        _lowercase  (   self							)       ->						str:



        '''simple docstring'''




        a__     : Optional[Any]  =WhisperFeatureExtractionTester(self							)







      def        _lowercase  (   self							)       ->						str:



        '''simple docstring'''




        a__     : str  =self.feature_extraction_class(**self.feat_extract_dict							)

        with tempfile.TemporaryDirectory() as tmpdirname:
          a__     : Optional[int]  =feat_extract_first.save_pretrained(lowerCAmelCase__							)[0]
          check_json_file_has_correct_format(lowerCAmelCase__							)
          a__     : List[Any]  =self.feature_extraction_class.from_pretrained(lowerCAmelCase__							)

        a__     : int  =feat_extract_first.to_dict()
        a__     : str  =feat_extract_second.to_dict()
        a__     : str  =feat_extract_first.mel_filters
        a__     : Union[str, Any]  =feat_extract_second.mel_filters
        self.assertTrue(np.allclose(lowerCAmelCase__		,				lowerCAmelCase__							)							)
        self.assertEqual(lowerCAmelCase__		,				lowerCAmelCase__							)







      def        _lowercase  (   self							)       ->						Optional[int]:



        '''simple docstring'''




        a__     : List[Any]  =self.feature_extraction_class(**self.feat_extract_dict							)

        with tempfile.TemporaryDirectory() as tmpdirname:
          a__     : List[str]  =os.path.join(lowerCAmelCase__		,				"feat_extract.json"							)
          feat_extract_first.to_json_file(lowerCAmelCase__							)
          a__     : Union[str, Any]  =self.feature_extraction_class.from_json_file(lowerCAmelCase__							)

        a__     : Dict  =feat_extract_first.to_dict()
        a__     : List[str]  =feat_extract_second.to_dict()
        a__     : int  =feat_extract_first.mel_filters
        a__     : List[str]  =feat_extract_second.mel_filters
        self.assertTrue(np.allclose(lowerCAmelCase__		,				lowerCAmelCase__							)							)
        self.assertEqual(lowerCAmelCase__		,				lowerCAmelCase__							)







      def        _lowercase  (   self							)       ->						Dict:



        '''simple docstring'''




        a__     : Tuple  =self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()							)
        # create three inputs of length 800, 1000, and 1200
        a__     : str  =[floats_list((1, x)							)[0] for x in range(8_0_0		,				1_4_0_0		,				2_0_0							)]
        a__     : Union[str, Any]  =[np.asarray(lowerCAmelCase__							) for speech_input in speech_inputs]

        # Test feature size
        a__     : Tuple  =feature_extractor(lowerCAmelCase__		,				padding="max_length"		,				return_tensors="np"							).input_features
        self.assertTrue(input_features.ndim == 3							)
        self.assertTrue(input_features.shape[-1] == feature_extractor.nb_max_frames							)
        self.assertTrue(input_features.shape[-2] == feature_extractor.feature_size							)

        # Test not batched input
        a__     : List[str]  =feature_extractor(speech_inputs[0]		,				return_tensors="np"							).input_features
        a__     : str  =feature_extractor(np_speech_inputs[0]		,				return_tensors="np"							).input_features
        self.assertTrue(np.allclose(lowerCAmelCase__		,				lowerCAmelCase__		,				atol=1E-3							)							)

        # Test batched
        a__     : Tuple  =feature_extractor(lowerCAmelCase__		,				return_tensors="np"							).input_features
        a__     : int  =feature_extractor(lowerCAmelCase__		,				return_tensors="np"							).input_features
        for enc_seq_a, enc_seq_a in zip(lowerCAmelCase__		,				lowerCAmelCase__							):
          self.assertTrue(np.allclose(lowerCAmelCase__		,				lowerCAmelCase__		,				atol=1E-3							)							)

        # Test 2-D numpy arrays are batched.
        a__     : Optional[Any]  =[floats_list((1, x)							)[0] for x in (8_0_0, 8_0_0, 8_0_0)]
        a__     : int  =np.asarray(lowerCAmelCase__							)
        a__     : Union[str, Any]  =feature_extractor(lowerCAmelCase__		,				return_tensors="np"							).input_features
        a__     : Tuple  =feature_extractor(lowerCAmelCase__		,				return_tensors="np"							).input_features
        for enc_seq_a, enc_seq_a in zip(lowerCAmelCase__		,				lowerCAmelCase__							):
          self.assertTrue(np.allclose(lowerCAmelCase__		,				lowerCAmelCase__		,				atol=1E-3							)							)

        # Test truncation required
        a__     : Optional[int]  =[floats_list((1, x)							)[0] for x in range(2_0_0		,				(feature_extractor.n_samples + 5_0_0)		,				2_0_0							)]
        a__     : List[str]  =[np.asarray(lowerCAmelCase__							) for speech_input in speech_inputs]

        a__     : str  =[x[: feature_extractor.n_samples] for x in speech_inputs]
        a__     : List[str]  =[np.asarray(lowerCAmelCase__							) for speech_input in speech_inputs_truncated]

        a__     : Optional[Any]  =feature_extractor(lowerCAmelCase__		,				return_tensors="np"							).input_features
        a__     : Any  =feature_extractor(lowerCAmelCase__		,				return_tensors="np"							).input_features
        for enc_seq_a, enc_seq_a in zip(lowerCAmelCase__		,				lowerCAmelCase__							):
          self.assertTrue(np.allclose(lowerCAmelCase__		,				lowerCAmelCase__		,				atol=1E-3							)							)







      def        _lowercase  (   self							)       ->						Union[str, Any]:



        '''simple docstring'''




        import torch

        a__     : List[Any]  =self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()							)
        a__     : Optional[Any]  =np.random.rand(1_0_0		,				3_2							).astype(np.floataa							)
        a__     : Optional[Any]  =np_speech_inputs.tolist()

        for inputs in [py_speech_inputs, np_speech_inputs]:
          a__     : List[Any]  =feature_extractor.pad([{"input_features": inputs}]		,				return_tensors="np"							)
          self.assertTrue(np_processed.input_features.dtype == np.floataa							)
          a__     : List[Any]  =feature_extractor.pad([{"input_features": inputs}]		,				return_tensors="pt"							)
          self.assertTrue(pt_processed.input_features.dtype == torch.floataa							)







      def        _lowercase  (   self		,				lowerCAmelCase__							)       ->						List[str]:



        '''simple docstring'''




        a__     : Any  =load_dataset("hf-internal-testing/librispeech_asr_dummy"		,				"clean"		,				split="validation"							)
        # automatic decoding with librispeech
        a__     : str  =ds.sort("id"							).select(range(lowerCAmelCase__							)							)[:num_samples]["""audio"""]

        return [x["array"] for x in speech_samples]







      def        _lowercase  (   self							)       ->						List[Any]:



        '''simple docstring'''




        a__     : Union[str, Any]  =torch.tensor(
            [
                0.11_93, -0.09_46, -0.10_98, -0.01_96, 0.02_25, -0.06_90, -0.17_36, 0.09_51,
                0.09_71, -0.08_17, -0.07_02, 0.01_62, 0.02_60, 0.00_17, -0.01_92, -0.16_78,
                0.07_09, -0.18_67, -0.06_55, -0.02_74, -0.02_34, -0.18_84, -0.05_16, -0.05_54,
                -0.02_74, -0.14_25, -0.14_23, 0.08_37, 0.03_77, -0.08_54
            ]							)
        # fmt: on

        a__     : List[str]  =self._load_datasamples(1							)
        a__     : List[str]  =WhisperFeatureExtractor()
        a__     : int  =feature_extractor(lowerCAmelCase__		,				return_tensors="pt"							).input_features
        self.assertEqual(input_features.shape		,				(1, 8_0, 3_0_0_0)							)
        self.assertTrue(torch.allclose(input_features[0, 0, :3_0]		,				lowerCAmelCase__		,				atol=1E-4							)							)







      def        _lowercase  (   self							)       ->						Dict:



        '''simple docstring'''




        a__     : Union[str, Any]  =self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()							)
        a__     : str  =self._load_datasamples(1							)[0]
        a__     : int  =((audio - audio.min()) / (audio.max() - audio.min())) * 6_5_5_3_5  # Rescale to [0, 65535] to show issue
        a__     : List[str]  =feat_extract.zero_mean_unit_var_norm([audio]		,				attention_mask=lowerCAmelCase__							)[0]

        self.assertTrue(np.all(np.mean(lowerCAmelCase__							) < 1E-3							)							)
        self.assertTrue(np.all(np.abs(np.var(lowerCAmelCase__							) - 1							) < 1E-3							)							)













import copy
import os
from typing import Union

from ...configuration_utils import PretrainedConfig
from ...models.auto.modeling_auto import MODEL_FOR_CAUSAL_LM_MAPPING_NAMES
from ...utils import logging
from ..auto import CONFIG_MAPPING


UpperCAmelCase     :     List[Any]							=	logging.get_logger(__name__)

UpperCAmelCase     :     Tuple							=	{
    """Salesforce/instruct-blip-flan-t5""": """https://huggingface.co/Salesforce/instruct-blip-flan-t5/resolve/main/config.json""",
}







class     __lowerCAmelCase	(   UpperCamelCase__):
      _lowercase    : Dict            =      """instructblip_vision_model"""







      def __init__(   self		,				lowerCAmelCase__=1_4_0_8		,				lowerCAmelCase__=6_1_4_4		,				lowerCAmelCase__=3_9		,				lowerCAmelCase__=1_6		,				lowerCAmelCase__=2_2_4		,				lowerCAmelCase__=1_4		,				lowerCAmelCase__="gelu"		,				lowerCAmelCase__=1E-6		,				lowerCAmelCase__=0.0		,				lowerCAmelCase__=1E-10		,				lowerCAmelCase__=True		,				**lowerCAmelCase__		,				)       ->						Optional[Any]:



        '''simple docstring'''




        super().__init__(**lowerCAmelCase__							)

        a__     : Tuple  =hidden_size
        a__     : Any  =intermediate_size
        a__     : Union[str, Any]  =num_hidden_layers
        a__     : Optional[Any]  =num_attention_heads
        a__     : List[str]  =patch_size
        a__     : int  =image_size
        a__     : Tuple  =initializer_range
        a__     : Any  =attention_dropout
        a__     : List[Any]  =layer_norm_eps
        a__     : Optional[Any]  =hidden_act
        a__     : Optional[Any]  =qkv_bias







      @classmethod
      def        _lowercase  (   cls		,				lowerCAmelCase__		,				**lowerCAmelCase__							)       ->						"PretrainedConfig":



        '''simple docstring'''




        cls._set_token_in_kwargs(lowerCAmelCase__							)

        a__ ,							a__     : Optional[Any]  =cls.get_config_dict(lowerCAmelCase__		,				**lowerCAmelCase__							)

        # get the vision config dict if we are loading from InstructBlipConfig
        if config_dict.get("model_type"							) == "instructblip":
          a__     : Any  =config_dict["vision_config"]

        if "model_type" in config_dict and hasattr(cls		,				"model_type"							) and config_dict["model_type"] != cls.model_type:
          logger.warning(
              F'''You are using a model of type {config_dict["model_type"]} to instantiate a model of type '''
              F'''{cls.model_type}. This is not supported for all configurations of models and can yield errors.'''							)

        return cls.from_dict(lowerCAmelCase__		,				**lowerCAmelCase__							)







class     __lowerCAmelCase	(   UpperCamelCase__):
      _lowercase    : Dict            =      """instructblip_qformer"""







      def __init__(   self		,				lowerCAmelCase__=3_0_5_2_2		,				lowerCAmelCase__=7_6_8		,				lowerCAmelCase__=1_2		,				lowerCAmelCase__=1_2		,				lowerCAmelCase__=3_0_7_2		,				lowerCAmelCase__="gelu"		,				lowerCAmelCase__=0.1		,				lowerCAmelCase__=0.1		,				lowerCAmelCase__=5_1_2		,				lowerCAmelCase__=0.02		,				lowerCAmelCase__=1E-12		,				lowerCAmelCase__=0		,				lowerCAmelCase__="absolute"		,				lowerCAmelCase__=2		,				lowerCAmelCase__=1_4_0_8		,				**lowerCAmelCase__		,				)       ->						Dict:



        '''simple docstring'''




        super().__init__(pad_token_id=lowerCAmelCase__		,				**lowerCAmelCase__							)

        a__     : Optional[int]  =vocab_size
        a__     : Optional[Any]  =hidden_size
        a__     : str  =num_hidden_layers
        a__     : Optional[int]  =num_attention_heads
        a__     : Dict  =hidden_act
        a__     : Optional[int]  =intermediate_size
        a__     : Union[str, Any]  =hidden_dropout_prob
        a__     : Optional[int]  =attention_probs_dropout_prob
        a__     : List[Any]  =max_position_embeddings
        a__     : Union[str, Any]  =initializer_range
        a__     : Optional[int]  =layer_norm_eps
        a__     : int  =position_embedding_type
        a__     : int  =cross_attention_frequency
        a__     : Tuple  =encoder_hidden_size







      @classmethod
      def        _lowercase  (   cls		,				lowerCAmelCase__		,				**lowerCAmelCase__							)       ->						"PretrainedConfig":



        '''simple docstring'''




        cls._set_token_in_kwargs(lowerCAmelCase__							)

        a__ ,							a__     : str  =cls.get_config_dict(lowerCAmelCase__		,				**lowerCAmelCase__							)

        # get the qformer config dict if we are loading from InstructBlipConfig
        if config_dict.get("model_type"							) == "instructblip":
          a__     : Optional[int]  =config_dict["qformer_config"]

        if "model_type" in config_dict and hasattr(cls		,				"model_type"							) and config_dict["model_type"] != cls.model_type:
          logger.warning(
              F'''You are using a model of type {config_dict["model_type"]} to instantiate a model of type '''
              F'''{cls.model_type}. This is not supported for all configurations of models and can yield errors.'''							)

        return cls.from_dict(lowerCAmelCase__		,				**lowerCAmelCase__							)







class     __lowerCAmelCase	(   UpperCamelCase__):
      _lowercase    : Dict            =      """instructblip"""
      _lowercase    : List[Any]            =      True







      def __init__(   self		,				lowerCAmelCase__=None		,				lowerCAmelCase__=None		,				lowerCAmelCase__=None		,				lowerCAmelCase__=3_2		,				**lowerCAmelCase__							)       ->						str:



        '''simple docstring'''




        super().__init__(**lowerCAmelCase__							)

        if vision_config is None:
          a__     : List[Any]  ={}
          logger.info("vision_config is None. initializing the InstructBlipVisionConfig with default values."							)

        if qformer_config is None:
          a__     : Tuple  ={}
          logger.info("qformer_config is None. Initializing the InstructBlipQFormerConfig with default values."							)

        if text_config is None:
          a__     : Dict  ={}
          logger.info("text_config is None. Initializing the text config with default values (`OPTConfig`)."							)

        a__     : Dict  =InstructBlipVisionConfig(**lowerCAmelCase__							)
        a__     : Union[str, Any]  =InstructBlipQFormerConfig(**lowerCAmelCase__							)
        a__     : Tuple  =text_config["model_type"] if "model_type" in text_config else "opt"
        a__     : List[str]  =CONFIG_MAPPING[text_model_type](**lowerCAmelCase__							)

        a__     : Union[str, Any]  =self.text_config.tie_word_embeddings
        a__     : Optional[Any]  =self.text_config.is_encoder_decoder

        a__     : str  =num_query_tokens
        a__     : List[Any]  =self.vision_config.hidden_size
        a__     : str  =self.text_config.model_type in MODEL_FOR_CAUSAL_LM_MAPPING_NAMES
        a__     : List[Any]  =1.0
        a__     : List[str]  =0.02







      @classmethod
      def        _lowercase  (   cls		,				lowerCAmelCase__		,				lowerCAmelCase__		,				lowerCAmelCase__		,				**lowerCAmelCase__		,				)       ->						int:



        '''simple docstring'''




        return cls(
            vision_config=vision_config.to_dict()		,				qformer_config=qformer_config.to_dict()		,				text_config=text_config.to_dict()		,				**lowerCAmelCase__		,				)







      def        _lowercase  (   self							)       ->						Union[str, Any]:



        '''simple docstring'''




        a__     : int  =copy.deepcopy(self.__dict__							)
        a__     : int  =self.vision_config.to_dict()
        a__     : str  =self.qformer_config.to_dict()
        a__     : str  =self.text_config.to_dict()
        a__     : List[str]  =self.__class__.model_type
        return output






