





"""simple docstring"""




import itertools
from dataclasses import dataclass
from typing import List, Optional

import pyarrow as pa
import pyarrow.parquet as pq

import datasets
from datasets.table import table_cast


UpperCamelCase_									=datasets.utils.logging.get_logger(__name__)




@dataclass
class 			_a						(				datasets.BuilderConfig    ):
				UpperCamelCase								=   1_0000
				UpperCamelCase								=   None
				UpperCamelCase								=   None




class 			_a						(				datasets.ArrowBasedBuilder    ):
				UpperCamelCase								=   ParquetConfig






				def     snake_case     (			self		:   Any							)     ->				Tuple:




										'''simple docstring'''



										return datasets.DatasetInfo(features=self.config.features							)






				def     snake_case     (			self		:   str,			lowerCAmelCase__		:   List[str]							)     ->				int:




										'''simple docstring'''



										if not self.config.data_files:
																raise ValueError(f"""At least one data file must be specified, but got data_files={self.config.data_files}"""							)
										_UpperCamelCase      : Dict         =   dl_manager.download_and_extract(self.config.data_files							)
										if isinstance(lowerCAmelCase__,			(str, list, tuple)							):
																_UpperCamelCase      : Tuple         =   data_files
																if isinstance(lowerCAmelCase__,			lowerCAmelCase__							):
																						_UpperCamelCase      : int         =   [files]
																# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
																_UpperCamelCase      : List[str]         =   [dl_manager.iter_files(lowerCAmelCase__							) for file in files]
																return [datasets.SplitGenerator(name=datasets.Split.TRAIN,			gen_kwargs={'''files''': files}							)]
										_UpperCamelCase      : Optional[int]         =   []
										for split_name, files in data_files.items():
																if isinstance(lowerCAmelCase__,			lowerCAmelCase__							):
																						_UpperCamelCase      : Tuple         =   [files]
																# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
																_UpperCamelCase      : List[Any]         =   [dl_manager.iter_files(lowerCAmelCase__							) for file in files]
																# Infer features is they are stoed in the arrow schema
																if self.info.features is None:
																						for file in itertools.chain.from_iterable(lowerCAmelCase__							):
																												with open(lowerCAmelCase__,			'''rb'''							) as f:
																																		_UpperCamelCase      : int         =   datasets.Features.from_arrow_schema(pq.read_schema(lowerCAmelCase__							)							)
																												break
																splits.append(datasets.SplitGenerator(name=lowerCAmelCase__,			gen_kwargs={'''files''': files}							)							)
										return splits






				def     snake_case     (			self		:   Any,			lowerCAmelCase__		:   pa.Table							)     ->				pa.Table:




										'''simple docstring'''



										if self.info.features is not None:
																# more expensive cast to support nested features with keys in a different order
																# allows str <-> int/float or str to Audio for example
																_UpperCamelCase      : Union[str, Any]         =   table_cast(lowerCAmelCase__,			self.info.features.arrow_schema							)
										return pa_table






				def     snake_case     (			self		:   List[Any],			lowerCAmelCase__		:   Union[str, Any]							)     ->				str:




										'''simple docstring'''



										_UpperCamelCase      : str         =   self.info.features.arrow_schema if self.info.features is not None else None
										if self.info.features is not None and self.config.columns is not None:
																if sorted(field.name for field in schema							) != sorted(self.config.columns							):
																						raise ValueError(
																						    f"""Tried to load parquet data with columns '{self.config.columns}' with mismatching features '{self.info.features}'"""							)
										for file_idx, file in enumerate(itertools.chain.from_iterable(lowerCAmelCase__							)							):
																with open(lowerCAmelCase__,			'''rb'''							) as f:
																						_UpperCamelCase      : Any         =   pq.ParquetFile(lowerCAmelCase__							)
																						try:
																												for batch_idx, record_batch in enumerate(
																												    parquet_file.iter_batches(batch_size=self.config.batch_size,			columns=self.config.columns							)							):
																																		_UpperCamelCase      : int         =   pa.Table.from_batches([record_batch]							)
																																		# Uncomment for debugging (will print the Arrow table size and elements)
																																		# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																																		# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																																		yield f"""{file_idx}_{batch_idx}""", self._cast_table(lowerCAmelCase__							)
																						except ValueError as e:
																												logger.error(f"""Failed to read file '{file}' with error {type(lowerCAmelCase__							)}: {e}"""							)
																												raise










"""simple docstring"""




from typing import List, Optional, Union

from ...image_utils import ImageInput
from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
from ...utils import TensorType




class 			_a						(				_lowerCAmelCase    ):
				UpperCamelCase								=   ['''image_processor''', '''tokenizer''']
				UpperCamelCase								=   '''BlipImageProcessor'''
				UpperCamelCase								=   '''AutoTokenizer'''






				def __init__(			self		:   List[str],			lowerCAmelCase__		:   Optional[int],			lowerCAmelCase__		:   Optional[int]							)     ->				int:




										'''simple docstring'''



										_UpperCamelCase      : Any         =   False
										super().__init__(lowerCAmelCase__,			lowerCAmelCase__							)
										_UpperCamelCase      : Tuple         =   self.image_processor






				def __call__(			self		:   str,			lowerCAmelCase__		:   ImageInput = None,			lowerCAmelCase__		:   Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None,			lowerCAmelCase__		:   bool = True,			lowerCAmelCase__		:   Union[bool, str, PaddingStrategy] = False,			lowerCAmelCase__		:   Union[bool, str, TruncationStrategy] = None,			lowerCAmelCase__		:   Optional[int] = None,			lowerCAmelCase__		:   int = 0,			lowerCAmelCase__		:   Optional[int] = None,			lowerCAmelCase__		:   Optional[bool] = None,			lowerCAmelCase__		:   bool = False,			lowerCAmelCase__		:   bool = False,			lowerCAmelCase__		:   bool = False,			lowerCAmelCase__		:   bool = False,			lowerCAmelCase__		:   bool = False,			lowerCAmelCase__		:   bool = True,			lowerCAmelCase__		:   Optional[Union[str, TensorType]] = None,			**lowerCAmelCase__		:   Optional[Any],			)     ->				BatchEncoding:




										'''simple docstring'''



										if images is None and text is None:
																raise ValueError('''You have to specify either images or text.'''							)

										# Get only text
										if images is None:
																_UpperCamelCase      : int         =   self.tokenizer
																_UpperCamelCase      : List[str]         =   self.tokenizer(
																    text=lowerCAmelCase__,			add_special_tokens=lowerCAmelCase__,			padding=lowerCAmelCase__,			truncation=lowerCAmelCase__,			max_length=lowerCAmelCase__,			stride=lowerCAmelCase__,			pad_to_multiple_of=lowerCAmelCase__,			return_attention_mask=lowerCAmelCase__,			return_overflowing_tokens=lowerCAmelCase__,			return_special_tokens_mask=lowerCAmelCase__,			return_offsets_mapping=lowerCAmelCase__,			return_token_type_ids=lowerCAmelCase__,			return_length=lowerCAmelCase__,			verbose=lowerCAmelCase__,			return_tensors=lowerCAmelCase__,			**lowerCAmelCase__,			)
																return text_encoding

										# add pixel_values
										_UpperCamelCase      : List[str]         =   self.image_processor(lowerCAmelCase__,			return_tensors=lowerCAmelCase__							)

										if text is not None:
																_UpperCamelCase      : Any         =   self.tokenizer(
																    text=lowerCAmelCase__,			add_special_tokens=lowerCAmelCase__,			padding=lowerCAmelCase__,			truncation=lowerCAmelCase__,			max_length=lowerCAmelCase__,			stride=lowerCAmelCase__,			pad_to_multiple_of=lowerCAmelCase__,			return_attention_mask=lowerCAmelCase__,			return_overflowing_tokens=lowerCAmelCase__,			return_special_tokens_mask=lowerCAmelCase__,			return_offsets_mapping=lowerCAmelCase__,			return_token_type_ids=lowerCAmelCase__,			return_length=lowerCAmelCase__,			verbose=lowerCAmelCase__,			return_tensors=lowerCAmelCase__,			**lowerCAmelCase__,			)
										else:
																_UpperCamelCase      : List[Any]         =   None

										if text_encoding is not None:
																encoding_image_processor.update(lowerCAmelCase__							)

										return encoding_image_processor






				def     snake_case     (			self		:   List[Any],			*lowerCAmelCase__		:   List[str],			**lowerCAmelCase__		:   str							)     ->				Optional[int]:




										'''simple docstring'''



										return self.tokenizer.batch_decode(*lowerCAmelCase__,			**lowerCAmelCase__							)






				def     snake_case     (			self		:   List[Any],			*lowerCAmelCase__		:   Dict,			**lowerCAmelCase__		:   Any							)     ->				List[str]:




										'''simple docstring'''



										return self.tokenizer.decode(*lowerCAmelCase__,			**lowerCAmelCase__							)






				@property
				# Copied from transformers.models.blip.processing_blip.BlipProcessor.model_input_names
				def     snake_case     (			self		:   Any							)     ->				str:




										'''simple docstring'''



										_UpperCamelCase      : List[str]         =   self.tokenizer.model_input_names
										_UpperCamelCase      : Union[str, Any]         =   self.image_processor.model_input_names
										return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names							)							)



