



import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
			import torch

if is_vision_available():
			from PIL import Image

			from transformers import MobileNetVaImageProcessor

class        UpperCAmelCase_  (       unittest.TestCase	):

						'''simple docstring'''




						def __init__(   self				,							_A				,							_A=7				,							_A=3				,							_A=18				,							_A=30				,							_A=400				,							_A=True				,							_A=None				,							_A=True				,							_A=None				,							):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= size if size is not None else {'shortest_edge': 20}
													__SCREAMING_SNAKE_CASE									= crop_size if crop_size is not None else {'height': 18, 'width': 18}
													__SCREAMING_SNAKE_CASE									= parent
													__SCREAMING_SNAKE_CASE									= batch_size
													__SCREAMING_SNAKE_CASE									= num_channels
													__SCREAMING_SNAKE_CASE									= image_size
													__SCREAMING_SNAKE_CASE									= min_resolution
													__SCREAMING_SNAKE_CASE									= max_resolution
													__SCREAMING_SNAKE_CASE									= do_resize
													__SCREAMING_SNAKE_CASE									= size
													__SCREAMING_SNAKE_CASE									= do_center_crop
													__SCREAMING_SNAKE_CASE									= crop_size






						def 	_A     (   self		):







													'''simple docstring'''

													return {
													    "do_resize": self.do_resize,
													    "size": self.size,
													    "do_center_crop": self.do_center_crop,
													    "crop_size": self.crop_size,
													}







@require_torch
@require_vision
class        UpperCAmelCase_  (       UpperCamelCase_   ,   unittest.TestCase	):

						'''simple docstring'''




						UpperCamelCase__      :			List[Any]      =  MobileNetVaImageProcessor if is_vision_available() else None






						def 	_A     (   self		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= MobileNetVaImageProcessingTester(self		)






						@property
						def 	_A     (   self		):







													'''simple docstring'''

													return self.image_processor_tester.prepare_image_processor_dict()






						def 	_A     (   self		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= self.image_processing_class(**self.image_processor_dict		)
													self.assertTrue(hasattr(_A				,							'do_resize'		)		)
													self.assertTrue(hasattr(_A				,							'size'		)		)
													self.assertTrue(hasattr(_A				,							'do_center_crop'		)		)
													self.assertTrue(hasattr(_A				,							'crop_size'		)		)






						def 	_A     (   self		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= self.image_processing_class.from_dict(self.image_processor_dict		)
													self.assertEqual(image_processor.size				,							{'shortest_edge': 20}		)
													self.assertEqual(image_processor.crop_size				,							{'height': 18, 'width': 18}		)

													__SCREAMING_SNAKE_CASE									= self.image_processing_class.from_dict(self.image_processor_dict				,							size=42				,							crop_size=84		)
													self.assertEqual(image_processor.size				,							{'shortest_edge': 42}		)
													self.assertEqual(image_processor.crop_size				,							{'height': 84, 'width': 84}		)






						def 	_A     (   self		):







													'''simple docstring'''

													pass






						def 	_A     (   self		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= self.image_processing_class(**self.image_processor_dict		)
													# create random PIL images
													__SCREAMING_SNAKE_CASE									= prepare_image_inputs(self.image_processor_tester				,							equal_resolution=_A		)
													for image in image_inputs:
																				self.assertIsInstance(_A				,							Image.Image		)

													# Test not batched input
													__SCREAMING_SNAKE_CASE									= image_processing(image_inputs[0]				,							return_tensors='pt'		).pixel_values
													self.assertEqual(
													    encoded_images.shape				,							(
													        1,
													        self.image_processor_tester.num_channels,
													        self.image_processor_tester.crop_size['height'],
													        self.image_processor_tester.crop_size['width'],
													    )				,							)

													# Test batched
													__SCREAMING_SNAKE_CASE									= image_processing(_A				,							return_tensors='pt'		).pixel_values
													self.assertEqual(
													    encoded_images.shape				,							(
													        self.image_processor_tester.batch_size,
													        self.image_processor_tester.num_channels,
													        self.image_processor_tester.crop_size['height'],
													        self.image_processor_tester.crop_size['width'],
													    )				,							)






						def 	_A     (   self		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= self.image_processing_class(**self.image_processor_dict		)
													# create random numpy tensors
													__SCREAMING_SNAKE_CASE									= prepare_image_inputs(self.image_processor_tester				,							equal_resolution=_A				,							numpify=_A		)
													for image in image_inputs:
																				self.assertIsInstance(_A				,							np.ndarray		)

													# Test not batched input
													__SCREAMING_SNAKE_CASE									= image_processing(image_inputs[0]				,							return_tensors='pt'		).pixel_values
													self.assertEqual(
													    encoded_images.shape				,							(
													        1,
													        self.image_processor_tester.num_channels,
													        self.image_processor_tester.crop_size['height'],
													        self.image_processor_tester.crop_size['width'],
													    )				,							)

													# Test batched
													__SCREAMING_SNAKE_CASE									= image_processing(_A				,							return_tensors='pt'		).pixel_values
													self.assertEqual(
													    encoded_images.shape				,							(
													        self.image_processor_tester.batch_size,
													        self.image_processor_tester.num_channels,
													        self.image_processor_tester.crop_size['height'],
													        self.image_processor_tester.crop_size['width'],
													    )				,							)






						def 	_A     (   self		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= self.image_processing_class(**self.image_processor_dict		)
													# create random PyTorch tensors
													__SCREAMING_SNAKE_CASE									= prepare_image_inputs(self.image_processor_tester				,							equal_resolution=_A				,							torchify=_A		)
													for image in image_inputs:
																				self.assertIsInstance(_A				,							torch.Tensor		)

													# Test not batched input
													__SCREAMING_SNAKE_CASE									= image_processing(image_inputs[0]				,							return_tensors='pt'		).pixel_values
													self.assertEqual(
													    encoded_images.shape				,							(
													        1,
													        self.image_processor_tester.num_channels,
													        self.image_processor_tester.crop_size['height'],
													        self.image_processor_tester.crop_size['width'],
													    )				,							)

													# Test batched
													__SCREAMING_SNAKE_CASE									= image_processing(_A				,							return_tensors='pt'		).pixel_values
													self.assertEqual(
													    encoded_images.shape				,							(
													        self.image_processor_tester.batch_size,
													        self.image_processor_tester.num_channels,
													        self.image_processor_tester.crop_size['height'],
													        self.image_processor_tester.crop_size['width'],
													    )				,							)











import multiprocessing
from typing import TYPE_CHECKING, Optional, Union

from .. import Dataset, Features, config
from ..formatting import query_table
from ..packaged_modules.sql.sql import Sql
from ..utils import logging
from .abc import AbstractDatasetInputStream


if TYPE_CHECKING:
			import sqlitea

			import sqlalchemy

class        UpperCAmelCase_  (       UpperCamelCase_	):

						'''simple docstring'''




						def __init__(   self				,							_A				,							_A				,							_A = None				,							_A = None				,							_A = False				,							**_A				,							):







													'''simple docstring'''

													super().__init__(features=_A				,							cache_dir=_A				,							keep_in_memory=_A				,							**_A		)
													__SCREAMING_SNAKE_CASE									= Sql(
													    cache_dir=_A				,							features=_A				,							sql=_A				,							con=_A				,							**_A				,							)






						def 	_A     (   self		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= None
													__SCREAMING_SNAKE_CASE									= None
													__SCREAMING_SNAKE_CASE									= None
													__SCREAMING_SNAKE_CASE									= None

													self.builder.download_and_prepare(
													    download_config=_A				,							download_mode=_A				,							verification_mode=_A				,							base_path=_A				,							)

													# Build dataset for splits
													__SCREAMING_SNAKE_CASE									= self.builder.as_dataset(
													    split='train'				,							verification_mode=_A				,							in_memory=self.keep_in_memory		)
													return dataset







class        UpperCAmelCase_  :

						'''simple docstring'''




						def __init__(   self				,							_A				,							_A				,							_A				,							_A = None				,							_A = None				,							**_A				,							):







													'''simple docstring'''

													if num_proc is not None and num_proc <= 0:
																				raise ValueError(f"""num_proc {num_proc} must be an integer > 0."""		)

													__SCREAMING_SNAKE_CASE									= dataset
													__SCREAMING_SNAKE_CASE									= name
													__SCREAMING_SNAKE_CASE									= con
													__SCREAMING_SNAKE_CASE									= batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
													__SCREAMING_SNAKE_CASE									= num_proc
													__SCREAMING_SNAKE_CASE									= to_sql_kwargs






						def 	_A     (   self		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= self.to_sql_kwargs.pop('sql'				,							_A		)
													__SCREAMING_SNAKE_CASE									= self.to_sql_kwargs.pop('con'				,							_A		)
													__SCREAMING_SNAKE_CASE									= self.to_sql_kwargs.pop('index'				,							_A		)

													__SCREAMING_SNAKE_CASE									= self._write(index=_A				,							**self.to_sql_kwargs		)
													return written






						def 	_A     (   self				,							_A		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE							,	__SCREAMING_SNAKE_CASE							,	__SCREAMING_SNAKE_CASE									= args
													__SCREAMING_SNAKE_CASE									= {**to_sql_kwargs, 'if_exists': 'append'} if offset > 0 else to_sql_kwargs
													__SCREAMING_SNAKE_CASE									= query_table(
													    table=self.dataset.data				,							key=slice(_A				,							offset + self.batch_size		)				,							indices=self.dataset._indices				,							)
													__SCREAMING_SNAKE_CASE									= batch.to_pandas()
													__SCREAMING_SNAKE_CASE									= df.to_sql(self.name				,							self.con				,							index=_A				,							**_A		)
													return num_rows or len(_A		)






						def 	_A     (   self				,							_A				,							**_A		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= 0

													if self.num_proc is None or self.num_proc == 1:
																				for offset in logging.tqdm(
																				    range(0				,							len(self.dataset		)				,							self.batch_size		)				,							unit='ba'				,							disable=not logging.is_progress_bar_enabled()				,							desc='Creating SQL from Arrow format'				,							):
																											written += self._batch_sql((offset, index, to_sql_kwargs)		)
													else:
																				__SCREAMING_SNAKE_CASE							,	__SCREAMING_SNAKE_CASE									= len(self.dataset		), self.batch_size
																				with multiprocessing.Pool(self.num_proc		) as pool:
																											for num_rows in logging.tqdm(
																											    pool.imap(
																											        self._batch_sql				,							[(offset, index, to_sql_kwargs) for offset in range(0				,							_A				,							_A		)]				,							)				,							total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size				,							unit='ba'				,							disable=not logging.is_progress_bar_enabled()				,							desc='Creating SQL from Arrow format'				,							):
																																		written += num_rows

													return written






