



import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


lowerCAmelCase__		:					Union[str, Any]							=datasets.utils.logging.get_logger(__name__)

@dataclass
class        UpperCAmelCase_  (       datasets.BuilderConfig	):

						'''simple docstring'''




						UpperCamelCase__      :			Optional[datasets.Features]      =  None
						UpperCamelCase__      :			str      =  "utf-8"
						UpperCamelCase__      :			Optional[str]      =  None
						UpperCamelCase__      :			Optional[str]      =  None
						UpperCamelCase__      :			bool      =  True  # deprecated
						UpperCamelCase__      :			Optional[int]      =  None  # deprecated
						UpperCamelCase__      :			int      =  10 << 20  # 10MB
						UpperCamelCase__      :			Optional[bool]      =  None







class        UpperCAmelCase_  (       datasets.ArrowBasedBuilder	):

						'''simple docstring'''




						UpperCamelCase__      :			Any      =  JsonConfig






						def 	_A     (   self		):







													'''simple docstring'''

													if self.config.block_size is not None:
																				logger.warning('The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead'		)
																				__SCREAMING_SNAKE_CASE									= self.config.block_size
													if self.config.use_threads is not True:
																				logger.warning(
																				    'The JSON loader parameter `use_threads` is deprecated and doesn\'t have any effect anymore.'		)
													if self.config.newlines_in_values is not None:
																				raise ValueError('The JSON loader parameter `newlines_in_values` is no longer supported'		)
													return datasets.DatasetInfo(features=self.config.features		)






						def 	_A     (   self				,							_A		):







													'''simple docstring'''

													if not self.config.data_files:
																				raise ValueError(f"""At least one data file must be specified, but got data_files={self.config.data_files}"""		)
													__SCREAMING_SNAKE_CASE									= dl_manager.download_and_extract(self.config.data_files		)
													if isinstance(_A				,							(str, list, tuple)		):
																				__SCREAMING_SNAKE_CASE									= data_files
																				if isinstance(_A				,							_A		):
																											__SCREAMING_SNAKE_CASE									= [files]
																				__SCREAMING_SNAKE_CASE									= [dl_manager.iter_files(_A		) for file in files]
																				return [datasets.SplitGenerator(name=datasets.Split.TRAIN				,							gen_kwargs={'files': files}		)]
													__SCREAMING_SNAKE_CASE									= []
													for split_name, files in data_files.items():
																				if isinstance(_A				,							_A		):
																											__SCREAMING_SNAKE_CASE									= [files]
																				__SCREAMING_SNAKE_CASE									= [dl_manager.iter_files(_A		) for file in files]
																				splits.append(datasets.SplitGenerator(name=_A				,							gen_kwargs={'files': files}		)		)
													return splits






						def 	_A     (   self				,							_A		):







													'''simple docstring'''

													if self.config.features is not None:
																				# adding missing columns
																				for column_name in set(self.config.features		) - set(pa_table.column_names		):
																											__SCREAMING_SNAKE_CASE									= self.config.features.arrow_schema.field(_A		).type
																											__SCREAMING_SNAKE_CASE									= pa_table.append_column(_A				,							pa.array([None] * len(_A		)				,							type=_A		)		)
																				# more expensive cast to support nested structures with keys in a different order
																				# allows str <-> int/float or str to Audio for example
																				__SCREAMING_SNAKE_CASE									= table_cast(_A				,							self.config.features.arrow_schema		)
													return pa_table






						def 	_A     (   self				,							_A		):







													'''simple docstring'''

													for file_idx, file in enumerate(itertools.chain.from_iterable(_A		)		):
																				# If the file is one json object and if we need to look at the list of items in one specific field
																				if self.config.field is not None:
																											with open(_A				,							encoding=self.config.encoding				,							errors=self.config.encoding_errors		) as f:
																																		__SCREAMING_SNAKE_CASE									= json.load(_A		)

																											# We keep only the field we are interested in
																											__SCREAMING_SNAKE_CASE									= dataset[self.config.field]

																											# We accept two format: a list of dicts or a dict of lists
																											if isinstance(_A				,							(list, tuple)		):
																																		__SCREAMING_SNAKE_CASE									= set().union(*[row.keys() for row in dataset]		)
																																		__SCREAMING_SNAKE_CASE									= {col: [row.get(_A		) for row in dataset] for col in keys}
																											else:
																																		__SCREAMING_SNAKE_CASE									= dataset
																											__SCREAMING_SNAKE_CASE									= pa.Table.from_pydict(_A		)
																											yield file_idx, self._cast_table(_A		)

																				# If the file has one json object per line
																				else:
																											with open(_A				,							'rb'		) as f:
																																		__SCREAMING_SNAKE_CASE									= 0
																																		# Use block_size equal to the chunk size divided by 32 to leverage multithreading
																																		# Set a default minimum value of 16kB if the chunk size is really small
																																		__SCREAMING_SNAKE_CASE									= max(self.config.chunksize // 32				,							16 << 10		)
																																		__SCREAMING_SNAKE_CASE									= (
																																		    self.config.encoding_errors if self.config.encoding_errors is not None else 'strict'
																																		)
																																		while True:
																																									__SCREAMING_SNAKE_CASE									= f.read(self.config.chunksize		)
																																									if not batch:
																																																break
																																									# Finish current line
																																									try:
																																																batch += f.readline()
																																									except (AttributeError, io.UnsupportedOperation):
																																																batch += readline(_A		)
																																									# PyArrow only accepts utf-8 encoded bytes
																																									if self.config.encoding != "utf-8":
																																																__SCREAMING_SNAKE_CASE									= batch.decode(self.config.encoding				,							errors=_A		).encode('utf-8'		)
																																									try:
																																																while True:
																																																							try:
																																																														__SCREAMING_SNAKE_CASE									= paj.read_json(
																																																														    io.BytesIO(_A		)				,							read_options=paj.ReadOptions(block_size=_A		)		)
																																																														break
																																																							except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
																																																														if (
																																																														    isinstance(_A				,							pa.ArrowInvalid		)
																																																														    and "straddling" not in str(_A		)
																																																														    or block_size > len(_A		)
																																																														):
																																																																					raise
																																																														else:
																																																																					# Increase the block size in case it was too small.
																																																																					# The block size will be reset for the next file.
																																																																					logger.debug(
																																																																					    f"""Batch of {len(_A		)} bytes couldn't be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}."""		)
																																																																					block_size *= 2
																																									except pa.ArrowInvalid as e:
																																																try:
																																																							with open(
																																																							    _A				,							encoding=self.config.encoding				,							errors=self.config.encoding_errors		) as f:
																																																														__SCREAMING_SNAKE_CASE									= json.load(_A		)
																																																except json.JSONDecodeError:
																																																							logger.error(f"""Failed to read file '{file}' with error {type(_A		)}: {e}"""		)
																																																							raise e
																																																# If possible, parse the file as a list of json objects and exit the loop
																																																if isinstance(_A				,							_A		):  # list is the only sequence type supported in JSON
																																																							try:
																																																														__SCREAMING_SNAKE_CASE									= set().union(*[row.keys() for row in dataset]		)
																																																														__SCREAMING_SNAKE_CASE									= {col: [row.get(_A		) for row in dataset] for col in keys}
																																																														__SCREAMING_SNAKE_CASE									= pa.Table.from_pydict(_A		)
																																																							except (pa.ArrowInvalid, AttributeError) as e:
																																																														logger.error(f"""Failed to read file '{file}' with error {type(_A		)}: {e}"""		)
																																																														raise ValueError(f"""Not able to read records in the JSON file at {file}."""		) from None
																																																							yield file_idx, self._cast_table(_A		)
																																																							break
																																																else:
																																																							logger.error(f"""Failed to read file '{file}' with error {type(_A		)}: {e}"""		)
																																																							raise ValueError(
																																																							    f"""Not able to read records in the JSON file at {file}. """
																																																							    f"""You should probably indicate the field of the JSON file containing your records. """
																																																							    f"""This JSON file contain the following fields: {str(list(dataset.keys()		)		)}. """
																																																							    f"""Select the correct one and provide it as `field='XXX'` to the dataset loading method. """		) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																																									yield (file_idx, batch_idx), self._cast_table(_A		)
																																									batch_idx += 1











import itertools
import random
import unittest

import numpy as np

from transformers import ASTFeatureExtractor
from transformers.testing_utils import require_torch, require_torchaudio
from transformers.utils.import_utils import is_torch_available

from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin


lowerCAmelCase__		:					Dict							=random.Random()

if is_torch_available():
			import torch






def      __lowercase       (			a__							,       a__=1.0							,       a__=None							,       a__=None       )    ->			Any:
							if rng is None:
														__SCREAMING_SNAKE_CASE									= global_rng

							__SCREAMING_SNAKE_CASE									= []
							for batch_idx in range(shape[0]       ):
														values.append([]       )
														for _ in range(shape[1]       ):
																					values[-1].append(rng.random() * scale       )

							return values

class        UpperCAmelCase_  (       unittest.TestCase	):

						'''simple docstring'''




						def __init__(   self				,							_A				,							_A=7				,							_A=400				,							_A=2_000				,							_A=1				,							_A=0.0				,							_A=16_000				,							_A=True				,							_A=True				,							):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= parent
													__SCREAMING_SNAKE_CASE									= batch_size
													__SCREAMING_SNAKE_CASE									= min_seq_length
													__SCREAMING_SNAKE_CASE									= max_seq_length
													__SCREAMING_SNAKE_CASE									= (self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
													__SCREAMING_SNAKE_CASE									= feature_size
													__SCREAMING_SNAKE_CASE									= padding_value
													__SCREAMING_SNAKE_CASE									= sampling_rate
													__SCREAMING_SNAKE_CASE									= return_attention_mask
													__SCREAMING_SNAKE_CASE									= do_normalize






						def 	_A     (   self		):







													'''simple docstring'''

													return {
													    "feature_size": self.feature_size,
													    "padding_value": self.padding_value,
													    "sampling_rate": self.sampling_rate,
													    "return_attention_mask": self.return_attention_mask,
													    "do_normalize": self.do_normalize,
													}






						def 	_A     (   self				,							_A=False				,							_A=False		):







													'''simple docstring'''

													def _flatten(_A		):
																				return list(itertools.chain(*_A		)		)

													if equal_length:
																				__SCREAMING_SNAKE_CASE									= floats_list((self.batch_size, self.max_seq_length)		)
													else:
																				# make sure that inputs increase in size
																				__SCREAMING_SNAKE_CASE									= [
																				    _flatten(floats_list((x, self.feature_size)		)		)
																				    for x in range(self.min_seq_length				,							self.max_seq_length				,							self.seq_length_diff		)
																				]

													if numpify:
																				__SCREAMING_SNAKE_CASE									= [np.asarray(_A		) for x in speech_inputs]

													return speech_inputs







@require_torch
@require_torchaudio
class        UpperCAmelCase_  (       UpperCamelCase_   ,   unittest.TestCase	):

						'''simple docstring'''




						UpperCamelCase__      :			Optional[int]      =  ASTFeatureExtractor






						def 	_A     (   self		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= ASTFeatureExtractionTester(self		)






						def 	_A     (   self		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()		)
													# create three inputs of length 800, 1000, and 1200
													__SCREAMING_SNAKE_CASE									= [floats_list((1, x)		)[0] for x in range(800				,							1_400				,							200		)]
													__SCREAMING_SNAKE_CASE									= [np.asarray(_A		) for speech_input in speech_inputs]

													# Test not batched input
													__SCREAMING_SNAKE_CASE									= feat_extract(speech_inputs[0]				,							return_tensors='np'		).input_values
													__SCREAMING_SNAKE_CASE									= feat_extract(np_speech_inputs[0]				,							return_tensors='np'		).input_values
													self.assertTrue(np.allclose(_A				,							_A				,							atol=1e-3		)		)

													# Test batched
													__SCREAMING_SNAKE_CASE									= feat_extract(_A				,							padding=_A				,							return_tensors='np'		).input_values
													__SCREAMING_SNAKE_CASE									= feat_extract(_A				,							padding=_A				,							return_tensors='np'		).input_values
													for enc_seq_a, enc_seq_a in zip(_A				,							_A		):
																				self.assertTrue(np.allclose(_A				,							_A				,							atol=1e-3		)		)

													# Test 2-D numpy arrays are batched.
													__SCREAMING_SNAKE_CASE									= [floats_list((1, x)		)[0] for x in (800, 800, 800)]
													__SCREAMING_SNAKE_CASE									= np.asarray(_A		)
													__SCREAMING_SNAKE_CASE									= feat_extract(_A				,							return_tensors='np'		).input_values
													__SCREAMING_SNAKE_CASE									= feat_extract(_A				,							return_tensors='np'		).input_values
													for enc_seq_a, enc_seq_a in zip(_A				,							_A		):
																				self.assertTrue(np.allclose(_A				,							_A				,							atol=1e-3		)		)






						@require_torch
						def 	_A     (   self		):







													'''simple docstring'''

													import torch

													__SCREAMING_SNAKE_CASE									= self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()		)
													__SCREAMING_SNAKE_CASE									= np.random.rand(100		).astype(np.floataa		)
													__SCREAMING_SNAKE_CASE									= np_speech_inputs.tolist()

													for inputs in [py_speech_inputs, np_speech_inputs]:
																				__SCREAMING_SNAKE_CASE									= feature_extractor.pad([{'input_values': inputs}]				,							return_tensors='np'		)
																				self.assertTrue(np_processed.input_values.dtype == np.floataa		)
																				__SCREAMING_SNAKE_CASE									= feature_extractor.pad([{'input_values': inputs}]				,							return_tensors='pt'		)
																				self.assertTrue(pt_processed.input_values.dtype == torch.floataa		)






						def 	_A     (   self				,							_A		):







													'''simple docstring'''

													from datasets import load_dataset

													__SCREAMING_SNAKE_CASE									= load_dataset('hf-internal-testing/librispeech_asr_dummy'				,							'clean'				,							split='validation'		)
													# automatic decoding with librispeech
													__SCREAMING_SNAKE_CASE									= ds.sort('id'		).select(range(_A		)		)[:num_samples]['audio']

													return [x["array"] for x in speech_samples]






						@require_torch
						def 	_A     (   self		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= torch.tensor(
													    [-0.9_8_9_4, -1.2_7_7_6, -0.9_0_6_6, -1.2_7_7_6, -0.9_3_4_9, -1.2_6_0_9, -1.0_3_8_6, -1.2_7_7_6,
													     -1.1_5_6_1, -1.2_7_7_6, -1.2_0_5_2, -1.2_7_2_3, -1.2_1_9_0, -1.2_1_3_2, -1.2_7_7_6, -1.1_1_3_3,
													     -1.1_9_5_3, -1.1_3_4_3, -1.1_5_8_4, -1.2_2_0_3, -1.1_7_7_0, -1.2_4_7_4, -1.2_3_8_1, -1.1_9_3_6,
													     -0.9_2_7_0, -0.8_3_1_7, -0.8_0_4_9, -0.7_7_0_6, -0.7_5_6_5, -0.7_8_6_9]		)
													# fmt: on

													__SCREAMING_SNAKE_CASE									= self._load_datasamples(1		)
													__SCREAMING_SNAKE_CASE									= ASTFeatureExtractor()
													__SCREAMING_SNAKE_CASE									= feature_extractor(_A				,							return_tensors='pt'		).input_values
													self.assertEquals(input_values.shape				,							(1, 1_024, 128)		)
													self.assertTrue(torch.allclose(input_values[0, 0, :30]				,							_A				,							atol=1e-4		)		)






