


"""simple docstring"""







from collections import OrderedDict
from typing import Any, Mapping, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...feature_extraction_utils import FeatureExtractionMixin
from ...onnx import OnnxConfig
from ...onnx.utils import compute_effective_axis_dimension
from ...tokenization_utils_base import PreTrainedTokenizerBase
from ...utils import TensorType, logging


lowerCAmelCase__				    =       logging.get_logger(__name__)

lowerCAmelCase__				    =       {
    '''deepmind/language-perceiver''': '''https://huggingface.co/deepmind/language-perceiver/resolve/main/config.json''',
    # See all Perceiver models at https://huggingface.co/models?filter=perceiver
}


class 				SCREAMING_SNAKE_CASE__    (		lowercase    ):
       """simple docstring"""


       a  :    Optional[int]       ="perceiver"

       def __init__(						self		,					snake_case__=256		,					snake_case__=1_280		,					snake_case__=768		,					snake_case__=1		,					snake_case__=26		,					snake_case__=8		,					snake_case__=8		,					snake_case__=None		,					snake_case__=None		,					snake_case__="kv"		,					snake_case__=1		,					snake_case__=1		,					snake_case__="gelu"		,					snake_case__=0.1		,					snake_case__=0.02		,					snake_case__=1e-12		,					snake_case__=True		,					snake_case__=262		,					snake_case__=2_048		,					snake_case__=56		,					snake_case__=[368, 496]		,					snake_case__=16		,					snake_case__=1_920		,					snake_case__=16		,					snake_case__=[1, 16, 224, 224]		,					**snake_case__		,					):






         """simple docstring"""





         super().__init__(**snake_case__						)

         lowerCAmelCase					:					Dict								=							num_latents
         lowerCAmelCase					:					Dict								=							d_latents
         lowerCAmelCase					:					Union[str, Any]								=							d_model
         lowerCAmelCase					:					List[Any]								=							num_blocks
         lowerCAmelCase					:					Optional[int]								=							num_self_attends_per_block
         lowerCAmelCase					:					Optional[Any]								=							num_self_attention_heads
         lowerCAmelCase					:					Union[str, Any]								=							num_cross_attention_heads
         lowerCAmelCase					:					Dict								=							qk_channels
         lowerCAmelCase					:					Union[str, Any]								=							v_channels
         lowerCAmelCase					:					Tuple								=							cross_attention_shape_for_attention
         lowerCAmelCase					:					int								=							self_attention_widening_factor
         lowerCAmelCase					:					Any								=							cross_attention_widening_factor
         lowerCAmelCase					:					List[Any]								=							hidden_act
         lowerCAmelCase					:					int								=							attention_probs_dropout_prob
         lowerCAmelCase					:					str								=							initializer_range
         lowerCAmelCase					:					Optional[int]								=							layer_norm_eps
         lowerCAmelCase					:					List[str]								=							use_query_residual
         # masked language modeling attributes
         lowerCAmelCase					:					List[Any]								=							vocab_size
         lowerCAmelCase					:					List[Any]								=							max_position_embeddings
         # image classification attributes
         lowerCAmelCase					:					Dict								=							image_size
         # flow attributes
         lowerCAmelCase					:					Union[str, Any]								=							train_size
         # multimodal autoencoding attributes
         lowerCAmelCase					:					Dict								=							num_frames
         lowerCAmelCase					:					Optional[Any]								=							audio_samples_per_frame
         lowerCAmelCase					:					List[Any]								=							samples_per_patch
         lowerCAmelCase					:					int								=							output_shape


class 				SCREAMING_SNAKE_CASE__    (		lowercase    ):
       """simple docstring"""


       @property
       def 	lowercase__							(						self						):






         """simple docstring"""





         if self.task == "multiple-choice":
           lowerCAmelCase					:					int								=							{0: "batch", 1: "choice", 2: "sequence"}
         else:
           lowerCAmelCase					:					Tuple								=							{0: "batch", 1: "sequence"}
         return OrderedDict(
             [
                 ("inputs", dynamic_axis),
                 ("attention_mask", dynamic_axis),
             ]						)

       @property
       def 	lowercase__							(						self						):






         """simple docstring"""





         return 1e-4







       def 	lowercase__							(						self		,					snake_case__		,					snake_case__ = -1		,					snake_case__ = -1		,					snake_case__ = -1		,					snake_case__ = False		,					snake_case__ = None		,					snake_case__ = 3		,					snake_case__ = 40		,					snake_case__ = 40		,					):






         """simple docstring"""





         if isinstance(snake_case__		,					snake_case__						):
           # If dynamic axis (-1) we forward with a fixed dimension of 2 samples to avoid optimizations made by ONNX
           lowerCAmelCase					:					List[str]								=							compute_effective_axis_dimension(
               snake_case__		,					fixed_dimension=OnnxConfig.default_fixed_batch		,					num_token_to_add=0						)
           # If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
           lowerCAmelCase					:					List[Any]								=							preprocessor.num_special_tokens_to_add(snake_case__						)
           lowerCAmelCase					:					str								=							compute_effective_axis_dimension(
               snake_case__		,					fixed_dimension=OnnxConfig.default_fixed_sequence		,					num_token_to_add=snake_case__						)
           # Generate dummy inputs according to compute batch and sequence
           lowerCAmelCase					:					List[Any]								=							[" ".join(["a"]						) * seq_length] * batch_size
           lowerCAmelCase					:					int								=							dict(preprocessor(snake_case__		,					return_tensors=snake_case__						)						)
           lowerCAmelCase					:					Optional[Any]								=							inputs.pop("input_ids"						)
           return inputs
         elif isinstance(snake_case__		,					snake_case__						) and preprocessor.model_input_names[0] == "pixel_values":
           # If dynamic axis (-1) we forward with a fixed dimension of 2 samples to avoid optimizations made by ONNX
           lowerCAmelCase					:					List[str]								=							compute_effective_axis_dimension(snake_case__		,					fixed_dimension=OnnxConfig.default_fixed_batch						)
           lowerCAmelCase					:					str								=							self._generate_dummy_images(snake_case__		,					snake_case__		,					snake_case__		,					snake_case__						)
           lowerCAmelCase					:					Union[str, Any]								=							dict(preprocessor(images=snake_case__		,					return_tensors=snake_case__						)						)
           lowerCAmelCase					:					List[str]								=							inputs.pop("pixel_values"						)
           return inputs
         else:
           raise ValueError(
               "Unable to generate dummy inputs for the model. Please provide a tokenizer or a preprocessor."						)









"""simple docstring"""







import multiprocessing
from typing import TYPE_CHECKING, Optional, Union

from .. import Dataset, Features, config
from ..formatting import query_table
from ..packaged_modules.sql.sql import Sql
from ..utils import logging
from .abc import AbstractDatasetInputStream


if TYPE_CHECKING:
     import sqlitea

     import sqlalchemy


class 				SCREAMING_SNAKE_CASE__    (		lowercase    ):
       """simple docstring"""


       def __init__(						self		,					snake_case__		,					snake_case__		,					snake_case__ = None		,					snake_case__ = None		,					snake_case__ = False		,					**snake_case__		,					):






         """simple docstring"""





         super().__init__(features=snake_case__		,					cache_dir=snake_case__		,					keep_in_memory=snake_case__		,					**snake_case__						)
         lowerCAmelCase					:					Dict								=							Sql(
             cache_dir=snake_case__		,					features=snake_case__		,					sql=snake_case__		,					con=snake_case__		,					**snake_case__		,					)







       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					Tuple								=							None
         lowerCAmelCase					:					Optional[int]								=							None
         lowerCAmelCase					:					str								=							None
         lowerCAmelCase					:					int								=							None

         self.builder.download_and_prepare(
             download_config=snake_case__		,					download_mode=snake_case__		,					verification_mode=snake_case__		,					base_path=snake_case__		,					)

         # Build dataset for splits
         lowerCAmelCase					:					str								=							self.builder.as_dataset(
             split="train"		,					verification_mode=snake_case__		,					in_memory=self.keep_in_memory						)
         return dataset


class 				SCREAMING_SNAKE_CASE__    :
       """simple docstring"""


       def __init__(						self		,					snake_case__		,					snake_case__		,					snake_case__		,					snake_case__ = None		,					snake_case__ = None		,					**snake_case__		,					):






         """simple docstring"""





         if num_proc is not None and num_proc <= 0:
           raise ValueError(f"""num_proc {num_proc} must be an integer > 0."""						)

         lowerCAmelCase					:					Tuple								=							dataset
         lowerCAmelCase					:					Tuple								=							name
         lowerCAmelCase					:					List[str]								=							con
         lowerCAmelCase					:					List[Any]								=							batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
         lowerCAmelCase					:					Optional[int]								=							num_proc
         lowerCAmelCase					:					Optional[int]								=							to_sql_kwargs

       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					List[str]								=							self.to_sql_kwargs.pop("sql"		,					snake_case__						)
         lowerCAmelCase					:					List[Any]								=							self.to_sql_kwargs.pop("con"		,					snake_case__						)
         lowerCAmelCase					:					Dict								=							self.to_sql_kwargs.pop("index"		,					snake_case__						)

         lowerCAmelCase					:					Dict								=							self._write(index=snake_case__		,					**self.to_sql_kwargs						)
         return written

       def 	lowercase__							(						self		,					snake_case__						):






         """simple docstring"""





         lowerCAmelCase						,  lowerCAmelCase						,  lowerCAmelCase					:					Optional[int]								=							args
         lowerCAmelCase					:					Optional[int]								=							{**to_sql_kwargs, "if_exists": "append"} if offset > 0 else to_sql_kwargs
         lowerCAmelCase					:					List[Any]								=							query_table(
             table=self.dataset.data		,					key=slice(snake_case__		,					offset + self.batch_size						)		,					indices=self.dataset._indices		,					)
         lowerCAmelCase					:					Tuple								=							batch.to_pandas()
         lowerCAmelCase					:					str								=							df.to_sql(self.name		,					self.con		,					index=snake_case__		,					**snake_case__						)
         return num_rows or len(snake_case__						)







       def 	lowercase__							(						self		,					snake_case__		,					**snake_case__						):






         """simple docstring"""





         lowerCAmelCase					:					Tuple								=							0

         if self.num_proc is None or self.num_proc == 1:
           for offset in logging.tqdm(
               range(0		,					len(self.dataset						)		,					self.batch_size						)		,					unit="ba"		,					disable=not logging.is_progress_bar_enabled()		,					desc="Creating SQL from Arrow format"		,					):
             written += self._batch_sql((offset, index, to_sql_kwargs)						)
         else:
           lowerCAmelCase						,  lowerCAmelCase					:					Dict								=							len(self.dataset						), self.batch_size
           with multiprocessing.Pool(self.num_proc						) as pool:
             for num_rows in logging.tqdm(
                 pool.imap(
                     self._batch_sql		,					[(offset, index, to_sql_kwargs) for offset in range(0		,					snake_case__		,					snake_case__						)]		,					)		,					total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size		,					unit="ba"		,					disable=not logging.is_progress_bar_enabled()		,					desc="Creating SQL from Arrow format"		,					):
               written += num_rows

         return written





