



import itertools
from dataclasses import dataclass
from typing import List, Optional

import pyarrow as pa
import pyarrow.parquet as pq

import datasets
from datasets.table import table_cast


__snake_case    :		str  =datasets.utils.logging.get_logger(__name__)



@dataclass
class 	lowerCamelCase__      (							datasets.BuilderConfig):




		'''simple docstring'''



		snake_case_     =1_0000
		snake_case_     =None
		snake_case_     =None



class 	lowerCamelCase__      (							datasets.ArrowBasedBuilder):




		'''simple docstring'''



		snake_case_     =ParquetConfig



		def 		lowerCAmelCase__				(self			)					->      Dict:

						"""simple docstring"""






						return datasets.DatasetInfo(features=self.config.features			)



		def 		lowerCAmelCase__				(self		,__lowerCamelCase			)					->      List[Any]:

						"""simple docstring"""






						if not self.config.data_files:
										raise ValueError(f"""At least one data file must be specified, but got data_files={self.config.data_files}"""			)
						lowerCAmelCase__					:    Any												=					dl_manager.download_and_extract(self.config.data_files			)
						if isinstance(__lowerCamelCase		,(str, list, tuple)			):
										lowerCAmelCase__					:    Optional[Any]												=					data_files
										if isinstance(__lowerCamelCase		,__lowerCamelCase			):
														lowerCAmelCase__					:    List[Any]												=					[files]
										# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
										lowerCAmelCase__					:    Dict												=					[dl_manager.iter_files(__lowerCamelCase			) for file in files]
										return [datasets.SplitGenerator(name=datasets.Split.TRAIN		,gen_kwargs={'''files''': files}			)]
						lowerCAmelCase__					:    int												=					[]
						for split_name, files in data_files.items():
										if isinstance(__lowerCamelCase		,__lowerCamelCase			):
														lowerCAmelCase__					:    int												=					[files]
										# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
										lowerCAmelCase__					:    Union[str, Any]												=					[dl_manager.iter_files(__lowerCamelCase			) for file in files]
										# Infer features is they are stoed in the arrow schema
										if self.info.features is None:
														for file in itertools.chain.from_iterable(__lowerCamelCase			):
																		with open(__lowerCamelCase		,'''rb'''			) as f:
																						lowerCAmelCase__					:    str												=					datasets.Features.from_arrow_schema(pq.read_schema(__lowerCamelCase			)			)
																		break
										splits.append(datasets.SplitGenerator(name=__lowerCamelCase		,gen_kwargs={'''files''': files}			)			)
						return splits



		def 		lowerCAmelCase__				(self		,__lowerCamelCase			)					->      pa.Table:

						"""simple docstring"""






						if self.info.features is not None:
										# more expensive cast to support nested features with keys in a different order
										# allows str <-> int/float or str to Audio for example
										lowerCAmelCase__					:    Dict												=					table_cast(__lowerCamelCase		,self.info.features.arrow_schema			)
						return pa_table



		def 		lowerCAmelCase__				(self		,__lowerCamelCase			)					->      List[Any]:

						"""simple docstring"""






						lowerCAmelCase__					:    int												=					self.info.features.arrow_schema if self.info.features is not None else None
						if self.info.features is not None and self.config.columns is not None:
										if sorted(field.name for field in schema			) != sorted(self.config.columns			):
														raise ValueError(
														    f"""Tried to load parquet data with columns '{self.config.columns}' with mismatching features '{self.info.features}'"""			)
						for file_idx, file in enumerate(itertools.chain.from_iterable(__lowerCamelCase			)			):
										with open(__lowerCamelCase		,'''rb'''			) as f:
														lowerCAmelCase__					:    str												=					pq.ParquetFile(__lowerCamelCase			)
														try:
																		for batch_idx, record_batch in enumerate(
																		    parquet_file.iter_batches(batch_size=self.config.batch_size		,columns=self.config.columns			)			):
																						lowerCAmelCase__					:    List[str]												=					pa.Table.from_batches([record_batch]			)
																						# Uncomment for debugging (will print the Arrow table size and elements)
																						# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																						# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																						yield f"""{file_idx}_{batch_idx}""", self._cast_table(__lowerCamelCase			)
														except ValueError as e:
																		logger.error(f"""Failed to read file '{file}' with error {type(__lowerCamelCase			)}: {e}"""			)
																		raise






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_torch_available,
)


__snake_case    :		Optional[int]  ={
    'configuration_vision_text_dual_encoder': ['VisionTextDualEncoderConfig'],
    'processing_vision_text_dual_encoder': ['VisionTextDualEncoderProcessor'],
}


try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__snake_case    :		int  =['VisionTextDualEncoderModel']


try:
							if not is_flax_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__snake_case    :		List[str]  =['FlaxVisionTextDualEncoderModel']

try:
							if not is_tf_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__snake_case    :		Tuple  =['TFVisionTextDualEncoderModel']


if TYPE_CHECKING:
							from .configuration_vision_text_dual_encoder import VisionTextDualEncoderConfig
							from .processing_vision_text_dual_encoder import VisionTextDualEncoderProcessor

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_vision_text_dual_encoder import VisionTextDualEncoderModel

							try:
														if not is_flax_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_flax_vision_text_dual_encoder import FlaxVisionTextDualEncoderModel

							try:
														if not is_tf_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_tf_vision_text_dual_encoder import TFVisionTextDualEncoderModel


else:
							import sys

							__snake_case    :		str  =_LazyModule(__name__, globals()['__file__'], _import_structure)

