



"""simple docstring"""


import os
from typing import BinaryIO, Optional, Union

import numpy as np
import pyarrow.parquet as pq

from .. import Audio, Dataset, Features, Image, NamedSplit, Value, config
from ..features.features import FeatureType, _visit
from ..formatting import query_table
from ..packaged_modules import _PACKAGED_DATASETS_MODULES
from ..packaged_modules.parquet.parquet import Parquet
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader






def    a__   (   __SCREAMING_SNAKE_CASE			)     ->     Optional[int]:
		__lowerCAmelCase:	Union[str, Any]   				=      np.inf

		def set_batch_size(__SCREAMING_SNAKE_CASE			) -> None:
				nonlocal batch_size
				if isinstance(_UpperCamelCase ,			_UpperCamelCase			):
						__lowerCAmelCase:	Optional[int]   				=      min(_UpperCamelCase ,			config.PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS			)
				elif isinstance(_UpperCamelCase ,			_UpperCamelCase			):
						__lowerCAmelCase:	List[str]   				=      min(_UpperCamelCase ,			config.PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS			)
				elif isinstance(_UpperCamelCase ,			_UpperCamelCase			) and feature.dtype == "binary":
						__lowerCAmelCase:	int   				=      min(_UpperCamelCase ,			config.PARQUET_ROW_GROUP_SIZE_FOR_BINARY_DATASETS			)

		_visit(_UpperCamelCase ,			_UpperCamelCase			)

		return None if batch_size is np.inf else batch_size





class    snake_case     (    _UpperCamelCase  ):

				def __init__(     self							:    Optional[int]					,  UpperCamelCase__							:    Any					,  UpperCamelCase__							:    str = None					,  UpperCamelCase__							:    str = None					,  UpperCamelCase__							:    Any = None					,  UpperCamelCase__							:    Optional[Any] = False					,  UpperCamelCase__							:    List[Any] = False					,  UpperCamelCase__							:    int = None					,  **UpperCamelCase__							:    List[str]					,  )->       int:



						'''simple docstring'''


						super().__init__(
						    _SCREAMING_SNAKE_CASE					,  split=_SCREAMING_SNAKE_CASE					,  features=_SCREAMING_SNAKE_CASE					,  cache_dir=_SCREAMING_SNAKE_CASE					,  keep_in_memory=_SCREAMING_SNAKE_CASE					,  streaming=_SCREAMING_SNAKE_CASE					,  num_proc=_SCREAMING_SNAKE_CASE					,  **_SCREAMING_SNAKE_CASE					,  )
						__lowerCAmelCase:	Optional[Any]   				=      path_or_paths if isinstance(_SCREAMING_SNAKE_CASE					,  _SCREAMING_SNAKE_CASE) else {self.split: path_or_paths}
						__lowerCAmelCase:	Optional[Any]   				=      _PACKAGED_DATASETS_MODULES["parquet"][1]
						__lowerCAmelCase:	Union[str, Any]   				=      Parquet(
						    cache_dir=_SCREAMING_SNAKE_CASE					,  data_files=_SCREAMING_SNAKE_CASE					,  features=_SCREAMING_SNAKE_CASE					,  hash=_SCREAMING_SNAKE_CASE					,  **_SCREAMING_SNAKE_CASE					,  )




				def 							lowercase_					(     self							:    Optional[Any])->       str:



						'''simple docstring'''


						if self.streaming:
								__lowerCAmelCase:	Any   				=      self.builder.as_streaming_dataset(split=self.split)
						# Build regular (map-style) dataset
						else:
								__lowerCAmelCase:	int   				=      None
								__lowerCAmelCase:	Any   				=      None
								__lowerCAmelCase:	List[str]   				=      None
								__lowerCAmelCase:	int   				=      None

								self.builder.download_and_prepare(
								    download_config=_SCREAMING_SNAKE_CASE					,  download_mode=_SCREAMING_SNAKE_CASE					,  verification_mode=_SCREAMING_SNAKE_CASE					,  base_path=_SCREAMING_SNAKE_CASE					,  num_proc=self.num_proc					,  )
								__lowerCAmelCase:	Optional[int]   				=      self.builder.as_dataset(
								    split=self.split					,  verification_mode=_SCREAMING_SNAKE_CASE					,  in_memory=self.keep_in_memory)
						return dataset





class    snake_case     :

				def __init__(     self							:    str					,  UpperCamelCase__							:    List[Any]					,  UpperCamelCase__							:    Optional[Any]					,  UpperCamelCase__							:    Any = None					,  **UpperCamelCase__							:    Any					,  )->       Dict:



						'''simple docstring'''


						__lowerCAmelCase:	int   				=      dataset
						__lowerCAmelCase:	int   				=      path_or_buf
						__lowerCAmelCase:	List[str]   				=      batch_size or get_writer_batch_size(dataset.features)
						__lowerCAmelCase:	str   				=      parquet_writer_kwargs

				def 							lowercase_					(     self							:    List[str])->       int:



						'''simple docstring'''


						__lowerCAmelCase:	int   				=      self.batch_size if self.batch_size else config.DEFAULT_MAX_BATCH_SIZE

						if isinstance(self.path_or_buf					,  (str, bytes, os.PathLike)):
								with open(self.path_or_buf					,  "wb+") as buffer:
										__lowerCAmelCase:	str   				=      self._write(file_obj=_SCREAMING_SNAKE_CASE					,  batch_size=_SCREAMING_SNAKE_CASE					,  **self.parquet_writer_kwargs)
						else:
								__lowerCAmelCase:	List[str]   				=      self._write(file_obj=self.path_or_buf					,  batch_size=_SCREAMING_SNAKE_CASE					,  **self.parquet_writer_kwargs)
						return written




				def 							lowercase_					(     self							:    List[str]					,  UpperCamelCase__							:    Optional[Any]					,  UpperCamelCase__							:    str					,  **UpperCamelCase__							:    Optional[Any])->       int:



						'''simple docstring'''


						__lowerCAmelCase:	List[str]   				=      0
						__lowerCAmelCase:	Optional[Any]   				=      parquet_writer_kwargs.pop("path_or_buf"					,  _SCREAMING_SNAKE_CASE)
						__lowerCAmelCase:	str   				=      self.dataset.features.arrow_schema

						__lowerCAmelCase:	List[str]   				=      pq.ParquetWriter(_SCREAMING_SNAKE_CASE					,  schema=_SCREAMING_SNAKE_CASE					,  **_SCREAMING_SNAKE_CASE)

						for offset in logging.tqdm(
						    range(0					,  len(self.dataset)					,  _SCREAMING_SNAKE_CASE)					,  unit="ba"					,  disable=not logging.is_progress_bar_enabled()					,  desc="Creating parquet from Arrow format"					,  ):
								__lowerCAmelCase:	Optional[Any]   				=      query_table(
								    table=self.dataset._data					,  key=slice(_SCREAMING_SNAKE_CASE					,  offset + batch_size)					,  indices=self.dataset._indices if self.dataset._indices is not None else None					,  )
								writer.write_table(_SCREAMING_SNAKE_CASE)
								written += batch.nbytes
						writer.close()
						return written





"""simple docstring"""


import copy
import os
from typing import Union

from ...configuration_utils import PretrainedConfig
from ...models.auto.modeling_auto import MODEL_FOR_CAUSAL_LM_MAPPING_NAMES
from ...utils import logging
from ..auto import CONFIG_MAPPING


__A						       =      logging.get_logger(__name__)

__A						       =      {
    "salesforce/blip2-opt-2.7b": "https://huggingface.co/salesforce/blip2-opt-2.7b/resolve/main/config.json",
}





class    snake_case     (    __snake_case  ):
				SCREAMING_SNAKE_CASE_       :				Union[str, Any]       				=  """blip_2_vision_model"""

				def __init__(     self							:    Optional[int]					,  UpperCamelCase__							:    Optional[int]=1_4_0_8					,  UpperCamelCase__							:    Tuple=6_1_4_4					,  UpperCamelCase__							:    Tuple=3_9					,  UpperCamelCase__							:    str=1_6					,  UpperCamelCase__							:    Optional[Any]=2_2_4					,  UpperCamelCase__							:    List[str]=1_4					,  UpperCamelCase__							:    Optional[int]="gelu"					,  UpperCamelCase__							:    Optional[int]=0.00001					,  UpperCamelCase__							:    Tuple=0.0					,  UpperCamelCase__							:    Dict=1e-10					,  UpperCamelCase__							:    List[str]=True					,  **UpperCamelCase__							:    List[str]					,  )->       int:



						'''simple docstring'''


						super().__init__(**UpperCamelCase__)

						__lowerCAmelCase:	List[str]   				=      hidden_size
						__lowerCAmelCase:	Tuple   				=      intermediate_size
						__lowerCAmelCase:	str   				=      num_hidden_layers
						__lowerCAmelCase:	Any   				=      num_attention_heads
						__lowerCAmelCase:	Any   				=      patch_size
						__lowerCAmelCase:	Tuple   				=      image_size
						__lowerCAmelCase:	str   				=      initializer_range
						__lowerCAmelCase:	List[str]   				=      attention_dropout
						__lowerCAmelCase:	List[Any]   				=      layer_norm_eps
						__lowerCAmelCase:	Any   				=      hidden_act
						__lowerCAmelCase:	int   				=      qkv_bias




				@classmethod
				def 							lowercase_					(     cls							:    List[str]					,  UpperCamelCase__							:    Union[str, os.PathLike]					,  **UpperCamelCase__							:    List[str])->       "PretrainedConfig":



						'''simple docstring'''


						cls._set_token_in_kwargs(UpperCamelCase__)

						__lowerCAmelCase				,			__lowerCAmelCase:	List[str]   				=      cls.get_config_dict(UpperCamelCase__					,  **UpperCamelCase__)

						# get the vision config dict if we are loading from Blip2Config
						if config_dict.get("model_type") == "blip-2":
								__lowerCAmelCase:	Tuple   				=      config_dict["vision_config"]

						if "model_type" in config_dict and hasattr(cls					,  "model_type") and config_dict["model_type"] != cls.model_type:
								logger.warning(
								    f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
								    f"{cls.model_type}. This is not supported for all configurations of models and can yield errors.")

						return cls.from_dict(UpperCamelCase__					,  **UpperCamelCase__)





class    snake_case     (    __snake_case  ):
				SCREAMING_SNAKE_CASE_       :				List[str]       				=  """blip_2_qformer"""

				def __init__(     self							:    Optional[int]					,  UpperCamelCase__							:    Dict=3_0_5_2_2					,  UpperCamelCase__							:    Union[str, Any]=7_6_8					,  UpperCamelCase__							:    Union[str, Any]=1_2					,  UpperCamelCase__							:    Union[str, Any]=1_2					,  UpperCamelCase__							:    Optional[int]=3_0_7_2					,  UpperCamelCase__							:    Optional[Any]="gelu"					,  UpperCamelCase__							:    Tuple=0.1					,  UpperCamelCase__							:    Optional[Any]=0.1					,  UpperCamelCase__							:    List[str]=5_1_2					,  UpperCamelCase__							:    List[Any]=0.02					,  UpperCamelCase__							:    Dict=1e-12					,  UpperCamelCase__							:    List[str]=0					,  UpperCamelCase__							:    Optional[int]="absolute"					,  UpperCamelCase__							:    List[Any]=2					,  UpperCamelCase__							:    Optional[Any]=1_4_0_8					,  **UpperCamelCase__							:    List[str]					,  )->       str:



						'''simple docstring'''


						super().__init__(pad_token_id=UpperCamelCase__					,  **UpperCamelCase__)

						__lowerCAmelCase:	Tuple   				=      vocab_size
						__lowerCAmelCase:	str   				=      hidden_size
						__lowerCAmelCase:	Dict   				=      num_hidden_layers
						__lowerCAmelCase:	List[Any]   				=      num_attention_heads
						__lowerCAmelCase:	List[Any]   				=      hidden_act
						__lowerCAmelCase:	Optional[int]   				=      intermediate_size
						__lowerCAmelCase:	str   				=      hidden_dropout_prob
						__lowerCAmelCase:	Optional[int]   				=      attention_probs_dropout_prob
						__lowerCAmelCase:	Optional[int]   				=      max_position_embeddings
						__lowerCAmelCase:	int   				=      initializer_range
						__lowerCAmelCase:	List[str]   				=      layer_norm_eps
						__lowerCAmelCase:	Optional[Any]   				=      position_embedding_type
						__lowerCAmelCase:	Dict   				=      cross_attention_frequency
						__lowerCAmelCase:	List[Any]   				=      encoder_hidden_size




				@classmethod
				def 							lowercase_					(     cls							:    int					,  UpperCamelCase__							:    Union[str, os.PathLike]					,  **UpperCamelCase__							:    Tuple)->       "PretrainedConfig":



						'''simple docstring'''


						cls._set_token_in_kwargs(UpperCamelCase__)

						__lowerCAmelCase				,			__lowerCAmelCase:	Optional[int]   				=      cls.get_config_dict(UpperCamelCase__					,  **UpperCamelCase__)

						# get the qformer config dict if we are loading from Blip2Config
						if config_dict.get("model_type") == "blip-2":
								__lowerCAmelCase:	Any   				=      config_dict["qformer_config"]

						if "model_type" in config_dict and hasattr(cls					,  "model_type") and config_dict["model_type"] != cls.model_type:
								logger.warning(
								    f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
								    f"{cls.model_type}. This is not supported for all configurations of models and can yield errors.")

						return cls.from_dict(UpperCamelCase__					,  **UpperCamelCase__)





class    snake_case     (    __snake_case  ):
				SCREAMING_SNAKE_CASE_       :				Dict       				=  """blip-2"""
				SCREAMING_SNAKE_CASE_       :				str       				=  True

				def __init__(     self							:    List[Any]					,  UpperCamelCase__							:    Tuple=None					,  UpperCamelCase__							:    Dict=None					,  UpperCamelCase__							:    Optional[int]=None					,  UpperCamelCase__							:    Union[str, Any]=3_2					,  **UpperCamelCase__							:    Dict)->       List[Any]:



						'''simple docstring'''


						super().__init__(**UpperCamelCase__)

						if vision_config is None:
								__lowerCAmelCase:	Dict   				=      {}
								logger.info("vision_config is None. initializing the Blip2VisionConfig with default values.")

						if qformer_config is None:
								__lowerCAmelCase:	Union[str, Any]   				=      {}
								logger.info("qformer_config is None. Initializing the Blip2QFormerConfig with default values.")

						if text_config is None:
								__lowerCAmelCase:	Dict   				=      {}
								logger.info("text_config is None. Initializing the text config with default values (`OPTConfig`).")

						__lowerCAmelCase:	List[str]   				=      BlipaVisionConfig(**UpperCamelCase__)
						__lowerCAmelCase:	str   				=      BlipaQFormerConfig(**UpperCamelCase__)
						__lowerCAmelCase:	Optional[int]   				=      text_config["model_type"] if "model_type" in text_config else "opt"
						__lowerCAmelCase:	Optional[int]   				=      CONFIG_MAPPING[text_model_type](**UpperCamelCase__)

						__lowerCAmelCase:	List[Any]   				=      self.text_config.tie_word_embeddings
						__lowerCAmelCase:	List[Any]   				=      self.text_config.is_encoder_decoder

						__lowerCAmelCase:	int   				=      num_query_tokens
						__lowerCAmelCase:	Union[str, Any]   				=      self.vision_config.hidden_size
						__lowerCAmelCase:	Any   				=      self.text_config.model_type in MODEL_FOR_CAUSAL_LM_MAPPING_NAMES
						__lowerCAmelCase:	Dict   				=      1.0
						__lowerCAmelCase:	Dict   				=      0.02

				@classmethod
				def 							lowercase_					(     cls							:    str					,  UpperCamelCase__							:    BlipaVisionConfig					,  UpperCamelCase__							:    BlipaQFormerConfig					,  UpperCamelCase__							:    PretrainedConfig					,  **UpperCamelCase__							:    Union[str, Any]					,  )->       Tuple:



						'''simple docstring'''


						return cls(
						    vision_config=vision_config.to_dict()					,  qformer_config=qformer_config.to_dict()					,  text_config=text_config.to_dict()					,  **UpperCamelCase__					,  )




				def 							lowercase_					(     self							:    Any)->       List[str]:



						'''simple docstring'''


						__lowerCAmelCase:	str   				=      copy.deepcopy(self.__dict__)
						__lowerCAmelCase:	Optional[int]   				=      self.vision_config.to_dict()
						__lowerCAmelCase:	List[Any]   				=      self.qformer_config.to_dict()
						__lowerCAmelCase:	str   				=      self.text_config.to_dict()
						__lowerCAmelCase:	Tuple   				=      self.__class__.model_type
						return output


