


import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


snake_case							:	Tuple					      =				datasets.utils.logging.get_logger(__name__)







@dataclass
class 	snake_case_       (datasets.BuilderConfig				):
					UpperCAmelCase__ :   Optional[datasets.Features]   				=      None
					UpperCAmelCase__ :   str   				=      "utf-8"
					UpperCAmelCase__ :   Optional[str]   				=      None
					UpperCAmelCase__ :   Optional[str]   				=      None
					UpperCAmelCase__ :   bool   				=      True  # deprecated
					UpperCAmelCase__ :   Optional[int]   				=      None  # deprecated
					UpperCAmelCase__ :   int   				=      1_0 << 2_0  # 10MB
					UpperCAmelCase__ :   Optional[bool]   				=      None







class 	snake_case_       (datasets.ArrowBasedBuilder				):
					UpperCAmelCase__ :   Optional[Any]   				=      JsonConfig
					def 			lowerCamelCase__(				self       :Optional[int]				) ->   List[str]:
												if self.config.block_size is not None:
																			logger.warning('The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead'				)
																			a__					=					self.config.block_size
												if self.config.use_threads is not True:
																			logger.warning(
																			    'The JSON loader parameter `use_threads` is deprecated and doesn\'t have any effect anymore.'				)
												if self.config.newlines_in_values is not None:
																			raise ValueError('The JSON loader parameter `newlines_in_values` is no longer supported'				)
												return datasets.DatasetInfo(features=self.config.features				)
					def 			lowerCamelCase__(				self       :Any      ,__snake_case       :Any				) ->   List[str]:
												if not self.config.data_files:
																			raise ValueError(F'At least one data file must be specified, but got data_files={self.config.data_files}'				)
												a__					=					dl_manager.download_and_extract(self.config.data_files				)
												if isinstance(__snake_case      ,(str, list, tuple)				):
																			a__					=					data_files
																			if isinstance(__snake_case      ,__snake_case				):
																										a__					=					[files]
																			a__					=					[dl_manager.iter_files(__snake_case				) for file in files]
																			return [datasets.SplitGenerator(name=datasets.Split.TRAIN      ,gen_kwargs={'files': files}				)]
												a__					=					[]
												for split_name, files in data_files.items():
																			if isinstance(__snake_case      ,__snake_case				):
																										a__					=					[files]
																			a__					=					[dl_manager.iter_files(__snake_case				) for file in files]
																			splits.append(datasets.SplitGenerator(name=__snake_case      ,gen_kwargs={'files': files}				)				)
												return splits
					def 			lowerCamelCase__(				self       :Optional[int]      ,__snake_case       :pa.Table				) ->   pa.Table:
												if self.config.features is not None:
																			# adding missing columns
																			for column_name in set(self.config.features				) - set(pa_table.column_names				):
																										a__					=					self.config.features.arrow_schema.field(__snake_case				).type
																										a__					=					pa_table.append_column(__snake_case      ,pa.array([None] * len(__snake_case				)      ,type=__snake_case				)				)
																			# more expensive cast to support nested structures with keys in a different order
																			# allows str <-> int/float or str to Audio for example
																			a__					=					table_cast(__snake_case      ,self.config.features.arrow_schema				)
												return pa_table



					def 			lowerCamelCase__(				self       :int      ,__snake_case       :Optional[Any]				) ->   List[Any]:
												for file_idx, file in enumerate(itertools.chain.from_iterable(__snake_case				)				):
																			# If the file is one json object and if we need to look at the list of items in one specific field
																			if self.config.field is not None:
																										with open(__snake_case      ,encoding=self.config.encoding      ,errors=self.config.encoding_errors				) as f:
																																	a__					=					json.load(__snake_case				)

																										# We keep only the field we are interested in
																										a__					=					dataset[self.config.field]

																										# We accept two format: a list of dicts or a dict of lists
																										if isinstance(__snake_case      ,(list, tuple)				):
																																	a__					=					set().union(*[row.keys() for row in dataset]				)
																																	a__					=					{col: [row.get(__snake_case				) for row in dataset] for col in keys}
																										else:
																																	a__					=					dataset
																										a__					=					pa.Table.from_pydict(__snake_case				)
																										yield file_idx, self._cast_table(__snake_case				)

																			# If the file has one json object per line
																			else:
																										with open(__snake_case      ,'rb'				) as f:
																																	a__					=					0
																																	# Use block_size equal to the chunk size divided by 32 to leverage multithreading
																																	# Set a default minimum value of 16kB if the chunk size is really small
																																	a__					=					max(self.config.chunksize // 32      ,16 << 10				)
																																	a__					=					(
																																	    self.config.encoding_errors if self.config.encoding_errors is not None else 'strict'
																																	)
																																	while True:
																																								a__					=					f.read(self.config.chunksize				)
																																								if not batch:
																																															break
																																								# Finish current line
																																								try:
																																															batch += f.readline()
																																								except (AttributeError, io.UnsupportedOperation):
																																															batch += readline(__snake_case				)
																																								# PyArrow only accepts utf-8 encoded bytes
																																								if self.config.encoding != "utf-8":
																																															a__					=					batch.decode(self.config.encoding      ,errors=__snake_case				).encode('utf-8'				)
																																								try:
																																															while True:
																																																						try:
																																																													a__					=					paj.read_json(
																																																													    io.BytesIO(__snake_case				)      ,read_options=paj.ReadOptions(block_size=__snake_case				)				)
																																																													break
																																																						except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
																																																													if (
																																																													    isinstance(__snake_case      ,pa.ArrowInvalid				)
																																																													    and "straddling" not in str(__snake_case				)
																																																													    or block_size > len(__snake_case				)
																																																													):
																																																																				raise
																																																													else:
																																																																				# Increase the block size in case it was too small.
																																																																				# The block size will be reset for the next file.
																																																																				logger.debug(
																																																																				    F'Batch of {len(__snake_case				)} bytes couldn\'t be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}.'				)
																																																																				block_size *= 2
																																								except pa.ArrowInvalid as e:
																																															try:
																																																						with open(
																																																						    __snake_case      ,encoding=self.config.encoding      ,errors=self.config.encoding_errors				) as f:
																																																													a__					=					json.load(__snake_case				)
																																															except json.JSONDecodeError:
																																																						logger.error(F'Failed to read file \'{file}\' with error {type(__snake_case				)}: {e}'				)
																																																						raise e
																																															# If possible, parse the file as a list of json objects and exit the loop
																																															if isinstance(__snake_case      ,__snake_case				):  # list is the only sequence type supported in JSON
																																																						try:
																																																													a__					=					set().union(*[row.keys() for row in dataset]				)
																																																													a__					=					{col: [row.get(__snake_case				) for row in dataset] for col in keys}
																																																													a__					=					pa.Table.from_pydict(__snake_case				)
																																																						except (pa.ArrowInvalid, AttributeError) as e:
																																																													logger.error(F'Failed to read file \'{file}\' with error {type(__snake_case				)}: {e}'				)
																																																													raise ValueError(F'Not able to read records in the JSON file at {file}.'				) from None
																																																						yield file_idx, self._cast_table(__snake_case				)
																																																						break
																																															else:
																																																						logger.error(F'Failed to read file \'{file}\' with error {type(__snake_case				)}: {e}'				)
																																																						raise ValueError(
																																																						    F'Not able to read records in the JSON file at {file}. '
																																																						    F'You should probably indicate the field of the JSON file containing your records. '
																																																						    F'This JSON file contain the following fields: {str(list(dataset.keys()				)				)}. '
																																																						    F'Select the correct one and provide it as `field=\'XXX\'` to the dataset loading method. '				) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																																								yield (file_idx, batch_idx), self._cast_table(__snake_case				)
																																								batch_idx += 1








from ...configuration_utils import PretrainedConfig
from ...utils import logging


snake_case							:	Optional[int]					      =				logging.get_logger(__name__)

snake_case							:	Union[str, Any]					      =				{
    '''uclanlp/visualbert-vqa''': '''https://huggingface.co/uclanlp/visualbert-vqa/resolve/main/config.json''',
    '''uclanlp/visualbert-vqa-pre''': '''https://huggingface.co/uclanlp/visualbert-vqa-pre/resolve/main/config.json''',
    '''uclanlp/visualbert-vqa-coco-pre''': (
        '''https://huggingface.co/uclanlp/visualbert-vqa-coco-pre/resolve/main/config.json'''
    ),
    '''uclanlp/visualbert-vcr''': '''https://huggingface.co/uclanlp/visualbert-vcr/resolve/main/config.json''',
    '''uclanlp/visualbert-vcr-pre''': '''https://huggingface.co/uclanlp/visualbert-vcr-pre/resolve/main/config.json''',
    '''uclanlp/visualbert-vcr-coco-pre''': (
        '''https://huggingface.co/uclanlp/visualbert-vcr-coco-pre/resolve/main/config.json'''
    ),
    '''uclanlp/visualbert-nlvr2''': '''https://huggingface.co/uclanlp/visualbert-nlvr2/resolve/main/config.json''',
    '''uclanlp/visualbert-nlvr2-pre''': '''https://huggingface.co/uclanlp/visualbert-nlvr2-pre/resolve/main/config.json''',
    '''uclanlp/visualbert-nlvr2-coco-pre''': (
        '''https://huggingface.co/uclanlp/visualbert-nlvr2-coco-pre/resolve/main/config.json'''
    )
    # See all VisualBERT models at https://huggingface.co/models?filter=visual_bert
}







class 	snake_case_       (lowerCamelCase_				):
					UpperCAmelCase__ :   Dict   				=      '''visual_bert'''
					def __init__(				self       :Optional[int]      ,__snake_case       :Any=3_05_22      ,__snake_case       :str=7_68      ,__snake_case       :Any=5_12      ,__snake_case       :Any=12      ,__snake_case       :int=12      ,__snake_case       :str=30_72      ,__snake_case       :int="gelu"      ,__snake_case       :Optional[int]=0.1      ,__snake_case       :str=0.1      ,__snake_case       :Union[str, Any]=5_12      ,__snake_case       :Tuple=2      ,__snake_case       :Union[str, Any]=0.02      ,__snake_case       :Optional[Any]=1E-12      ,__snake_case       :Optional[Any]=False      ,__snake_case       :int=True      ,__snake_case       :Any=1      ,__snake_case       :Optional[int]=0      ,__snake_case       :Tuple=2      ,**__snake_case       :Any      ,) ->   Union[str, Any]:
												super().__init__(pad_token_id=__snake_case      ,bos_token_id=__snake_case      ,eos_token_id=__snake_case      ,**__snake_case				)

												a__					=					vocab_size
												a__					=					max_position_embeddings
												a__					=					hidden_size
												a__					=					visual_embedding_dim
												a__					=					num_hidden_layers
												a__					=					num_attention_heads
												a__					=					intermediate_size
												a__					=					hidden_act
												a__					=					hidden_dropout_prob
												a__					=					attention_probs_dropout_prob
												a__					=					initializer_range
												a__					=					type_vocab_size
												a__					=					layer_norm_eps
												a__					=					bypass_transformer
												a__					=					special_visual_initialize




