



import itertools
import random
import unittest

import numpy as np

from transformers import is_speech_available
from transformers.testing_utils import require_torch, require_torchaudio

from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin


if is_speech_available():
     from transformers import SpeechaTextFeatureExtractor

SCREAMING_SNAKE_CASE_:Tuple					=						random.Random()




def      __UpperCamelCase     (  _lowerCAmelCase				,					_lowerCAmelCase=1.0				,					_lowerCAmelCase=None				,					_lowerCAmelCase=None		)	->  Union[str, Any]:



    """simple docstring"""






    if rng is None:
        A			:	int										=       global_rng

    A			:	Tuple										=       []
    for batch_idx in range(shape[0]		):
        values.append([]		)
        for _ in range(shape[1]		):
            values[-1].append(rng.random() * scale		)

    return values






@require_torch
@require_torchaudio
class SCREAMING_SNAKE_CASE__					(		unittest.TestCase   ):




    '''simple docstring'''






    def __init__(	self,		lowerCamelCase__,		lowerCamelCase__=7,		lowerCamelCase__=400,		lowerCamelCase__=2000,		lowerCamelCase__=24,		lowerCamelCase__=24,		lowerCamelCase__=0.0,		lowerCamelCase__=1_6000,		lowerCamelCase__=True,		lowerCamelCase__=True,		):
        A			:	Optional[Any]										=       parent
        A			:	Optional[Any]										=       batch_size
        A			:	int										=       min_seq_length
        A			:	Union[str, Any]										=       max_seq_length
        A			:	int										=       (self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
        A			:	Dict										=       feature_size
        A			:	Any										=       num_mel_bins
        A			:	int										=       padding_value
        A			:	int										=       sampling_rate
        A			:	int										=       return_attention_mask
        A			:	List[str]										=       do_normalize






    def       _lowerCAmelCase (	self		):
        return {
            "feature_size": self.feature_size,
            "num_mel_bins": self.num_mel_bins,
            "padding_value": self.padding_value,
            "sampling_rate": self.sampling_rate,
            "return_attention_mask": self.return_attention_mask,
            "do_normalize": self.do_normalize,
        }






    def       _lowerCAmelCase (	self,		lowerCamelCase__=False,		lowerCamelCase__=False		):
        def _flatten(lowerCamelCase__		):
            return list(itertools.chain(*lowerCamelCase__		)		)

        if equal_length:
            A			:	Optional[Any]										=       [floats_list((self.max_seq_length, self.feature_size)		) for _ in range(self.batch_size		)]
        else:
            # make sure that inputs increase in size
            A			:	List[Any]										=       [
                floats_list((x, self.feature_size)		)
                for x in range(self.min_seq_length,		self.max_seq_length,		self.seq_length_diff		)
            ]
        if numpify:
            A			:	Dict										=       [np.asarray(lowerCamelCase__		) for x in speech_inputs]
        return speech_inputs






@require_torch
@require_torchaudio
class SCREAMING_SNAKE_CASE__					(		SCREAMING_SNAKE_CASE__			,						unittest.TestCase   ):




    '''simple docstring'''






    __lowerCamelCase      :			Optional[Any] =				SpeechaTextFeatureExtractor if is_speech_available() else None






    def       _lowerCAmelCase (	self		):
        A			:	List[str]										=       SpeechaTextFeatureExtractionTester(self		)






    def       _lowerCAmelCase (	self,		lowerCamelCase__		):
        self.assertTrue(np.all(np.mean(lowerCamelCase__,		axis=0		) < 1e-3		)		)
        self.assertTrue(np.all(np.abs(np.var(lowerCamelCase__,		axis=0		) - 1		) < 1e-3		)		)






    def       _lowerCAmelCase (	self		):
        # Tests that all call wrap to encode_plus and batch_encode_plus
        A			:	str										=       self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()		)
        # create three inputs of length 800, 1000, and 1200
        A			:	Optional[Any]										=       [floats_list((1, x)		)[0] for x in range(800,		1400,		200		)]
        A			:	Optional[int]										=       [np.asarray(lowerCamelCase__		) for speech_input in speech_inputs]

        # Test feature size
        A			:	List[Any]										=       feature_extractor(lowerCamelCase__,		padding=lowerCamelCase__,		return_tensors="""np"""		).input_features
        self.assertTrue(input_features.ndim == 3		)
        self.assertTrue(input_features.shape[-1] == feature_extractor.feature_size		)

        # Test not batched input
        A			:	Union[str, Any]										=       feature_extractor(speech_inputs[0],		return_tensors="""np"""		).input_features
        A			:	Union[str, Any]										=       feature_extractor(np_speech_inputs[0],		return_tensors="""np"""		).input_features
        self.assertTrue(np.allclose(lowerCamelCase__,		lowerCamelCase__,		atol=1e-3		)		)

        # Test batched
        A			:	int										=       feature_extractor(lowerCamelCase__,		return_tensors="""np"""		).input_features
        A			:	str										=       feature_extractor(lowerCamelCase__,		return_tensors="""np"""		).input_features
        for enc_seq_a, enc_seq_a in zip(lowerCamelCase__,		lowerCamelCase__		):
            self.assertTrue(np.allclose(lowerCamelCase__,		lowerCamelCase__,		atol=1e-3		)		)

        # Test 2-D numpy arrays are batched.
        A			:	Union[str, Any]										=       [floats_list((1, x)		)[0] for x in (800, 800, 800)]
        A			:	Any										=       np.asarray(lowerCamelCase__		)
        A			:	Dict										=       feature_extractor(lowerCamelCase__,		return_tensors="""np"""		).input_features
        A			:	Tuple										=       feature_extractor(lowerCamelCase__,		return_tensors="""np"""		).input_features
        for enc_seq_a, enc_seq_a in zip(lowerCamelCase__,		lowerCamelCase__		):
            self.assertTrue(np.allclose(lowerCamelCase__,		lowerCamelCase__,		atol=1e-3		)		)






    def       _lowerCAmelCase (	self		):
        A			:	Optional[int]										=       self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()		)
        A			:	List[Any]										=       [floats_list((1, x)		)[0] for x in range(800,		1400,		200		)]

        A			:	Dict										=       ["""longest""", """max_length""", """do_not_pad"""]
        A			:	Optional[int]										=       [None, 16, None]
        for max_length, padding in zip(lowerCamelCase__,		lowerCamelCase__		):
            A			:	Tuple										=       feature_extractor(
                lowerCamelCase__,		padding=lowerCamelCase__,		max_length=lowerCamelCase__,		return_attention_mask=lowerCamelCase__		)
            A			:	Tuple										=       inputs.input_features
            A			:	List[str]										=       inputs.attention_mask
            A			:	Any										=       [np.sum(lowerCamelCase__		) for x in attention_mask]

            self._check_zero_mean_unit_variance(input_features[0][: fbank_feat_lengths[0]]		)
            self._check_zero_mean_unit_variance(input_features[1][: fbank_feat_lengths[1]]		)
            self._check_zero_mean_unit_variance(input_features[2][: fbank_feat_lengths[2]]		)






    def       _lowerCAmelCase (	self		):
        A			:	int										=       self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()		)
        A			:	Union[str, Any]										=       [floats_list((1, x)		)[0] for x in range(800,		1400,		200		)]

        A			:	Tuple										=       ["""longest""", """max_length""", """do_not_pad"""]
        A			:	Union[str, Any]										=       [None, 16, None]
        for max_length, padding in zip(lowerCamelCase__,		lowerCamelCase__		):
            A			:	int										=       feature_extractor(
                lowerCamelCase__,		max_length=lowerCamelCase__,		padding=lowerCamelCase__,		return_tensors="""np""",		return_attention_mask=lowerCamelCase__		)
            A			:	Optional[int]										=       inputs.input_features
            A			:	int										=       inputs.attention_mask
            A			:	List[Any]										=       [np.sum(lowerCamelCase__		) for x in attention_mask]

            self._check_zero_mean_unit_variance(input_features[0][: fbank_feat_lengths[0]]		)
            self.assertTrue(input_features[0][fbank_feat_lengths[0] :].sum() < 1e-6		)
            self._check_zero_mean_unit_variance(input_features[1][: fbank_feat_lengths[1]]		)
            self.assertTrue(input_features[0][fbank_feat_lengths[1] :].sum() < 1e-6		)
            self._check_zero_mean_unit_variance(input_features[2][: fbank_feat_lengths[2]]		)






    def       _lowerCAmelCase (	self		):
        A			:	List[str]										=       self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()		)
        A			:	Any										=       [floats_list((1, x)		)[0] for x in range(800,		1400,		200		)]
        A			:	Dict										=       feature_extractor(
            lowerCamelCase__,		padding="""max_length""",		max_length=4,		truncation=lowerCamelCase__,		return_tensors="""np""",		return_attention_mask=lowerCamelCase__,		)
        A			:	Union[str, Any]										=       inputs.input_features
        A			:	Tuple										=       inputs.attention_mask
        A			:	List[Any]										=       np.sum(attention_mask == 1,		axis=1		)

        self._check_zero_mean_unit_variance(input_features[0, : fbank_feat_lengths[0]]		)
        self._check_zero_mean_unit_variance(input_features[1]		)
        self._check_zero_mean_unit_variance(input_features[2]		)






    def       _lowerCAmelCase (	self		):
        A			:	Tuple										=       self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()		)
        A			:	Tuple										=       [floats_list((1, x)		)[0] for x in range(800,		1400,		200		)]
        A			:	Any										=       feature_extractor(
            lowerCamelCase__,		padding="""longest""",		max_length=4,		truncation=lowerCamelCase__,		return_tensors="""np""",		return_attention_mask=lowerCamelCase__,		)
        A			:	Optional[Any]										=       inputs.input_features
        A			:	Optional[int]										=       inputs.attention_mask
        A			:	Tuple										=       np.sum(attention_mask == 1,		axis=1		)

        self._check_zero_mean_unit_variance(input_features[0, : fbank_feat_lengths[0]]		)
        self._check_zero_mean_unit_variance(input_features[1, : fbank_feat_lengths[1]]		)
        self._check_zero_mean_unit_variance(input_features[2]		)

        # make sure that if max_length < longest -> then pad to max_length
        self.assertEqual(input_features.shape,		(3, 4, 24)		)

        A			:	List[str]										=       [floats_list((1, x)		)[0] for x in range(800,		1400,		200		)]
        A			:	Any										=       feature_extractor(
            lowerCamelCase__,		padding="""longest""",		max_length=16,		truncation=lowerCamelCase__,		return_tensors="""np""",		return_attention_mask=lowerCamelCase__,		)
        A			:	int										=       inputs.input_features
        A			:	Optional[int]										=       inputs.attention_mask
        A			:	List[Any]										=       np.sum(attention_mask == 1,		axis=1		)

        self._check_zero_mean_unit_variance(input_features[0, : fbank_feat_lengths[0]]		)
        self._check_zero_mean_unit_variance(input_features[1, : fbank_feat_lengths[1]]		)
        self._check_zero_mean_unit_variance(input_features[2]		)

        # make sure that if max_length < longest -> then pad to max_length
        self.assertEqual(input_features.shape,		(3, 6, 24)		)






    def       _lowerCAmelCase (	self		):
        import torch

        A			:	Any										=       self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()		)
        A			:	Dict										=       np.random.rand(100,		32		).astype(np.floataa		)
        A			:	List[Any]										=       np_speech_inputs.tolist()

        for inputs in [py_speech_inputs, np_speech_inputs]:
            A			:	Any										=       feature_extractor.pad([{"""input_features""": inputs}],		return_tensors="""np"""		)
            self.assertTrue(np_processed.input_features.dtype == np.floataa		)
            A			:	List[Any]										=       feature_extractor.pad([{"""input_features""": inputs}],		return_tensors="""pt"""		)
            self.assertTrue(pt_processed.input_features.dtype == torch.floataa		)






    def       _lowerCAmelCase (	self,		lowerCamelCase__		):
        from datasets import load_dataset

        A			:	str										=       load_dataset("""hf-internal-testing/librispeech_asr_dummy""",		"""clean""",		split="""validation"""		)
        # automatic decoding with librispeech
        A			:	List[Any]										=       ds.sort("""id"""		).select(range(lowerCamelCase__		)		)[:num_samples]["""audio"""]

        return [x["array"] for x in speech_samples]






    def       _lowerCAmelCase (	self		):
        # fmt: off
        A			:	Optional[int]										=       np.array([
            -1.5745, -1.7713, -1.7020, -1.6069, -1.2250, -1.1105, -0.9072, -0.8241,
            -1.2310, -0.8098, -0.3320, -0.4101, -0.7985, -0.4996, -0.8213, -0.9128,
            -1.0420, -1.1286, -1.0440, -0.7999, -0.8405, -1.2275, -1.5443, -1.4625,
        ]		)
        # fmt: on

        A			:	Optional[Any]										=       self._load_datasamples(1		)
        A			:	int										=       self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()		)
        A			:	List[Any]										=       feature_extractor(lowerCamelCase__,		return_tensors="""pt"""		).input_features
        self.assertEquals(input_features.shape,		(1, 584, 24)		)
        self.assertTrue(np.allclose(input_features[0, 0, :30],		lowerCamelCase__,		atol=1e-4		)		)







import inspect
import re
from hashlib import shaaaa
from typing import Dict, List

from .arrow import arrow
from .audiofolder import audiofolder
from .csv import csv
from .imagefolder import imagefolder
from .json import json
from .pandas import pandas
from .parquet import parquet
from .sql import sql  # noqa F401
from .text import text




def      __UpperCamelCase     (  _lowerCAmelCase		)	->  str:



    """simple docstring"""






    A			:	int										=       []
    for line in lines:
        A			:	int										=       re.sub(R"""#.*"""				,					""""""				,					_lowerCAmelCase		)  # remove comments
        if line:
            filtered_lines.append(_lowerCAmelCase		)
    A			:	Tuple										=       """\n""".join(_lowerCAmelCase		)

    # Make a hash from all this code
    A			:	Union[str, Any]										=       full_str.encode("""utf-8"""		)
    return shaaaa(_lowerCAmelCase		).hexdigest()


# get importable module names and hash for caching
SCREAMING_SNAKE_CASE_:List[Any]					=						{
    """csv""": (csv.__name__, _hash_python_lines(inspect.getsource(csv).splitlines())),
    """json""": (json.__name__, _hash_python_lines(inspect.getsource(json).splitlines())),
    """pandas""": (pandas.__name__, _hash_python_lines(inspect.getsource(pandas).splitlines())),
    """parquet""": (parquet.__name__, _hash_python_lines(inspect.getsource(parquet).splitlines())),
    """arrow""": (arrow.__name__, _hash_python_lines(inspect.getsource(arrow).splitlines())),
    """text""": (text.__name__, _hash_python_lines(inspect.getsource(text).splitlines())),
    """imagefolder""": (imagefolder.__name__, _hash_python_lines(inspect.getsource(imagefolder).splitlines())),
    """audiofolder""": (audiofolder.__name__, _hash_python_lines(inspect.getsource(audiofolder).splitlines())),
}

# Used to infer the module to use based on the data files extensions
SCREAMING_SNAKE_CASE_:Optional[Any]					=						{
    """.csv""": ("""csv""", {}),
    """.tsv""": ("""csv""", {"""sep""": """\t"""}),
    """.json""": ("""json""", {}),
    """.jsonl""": ("""json""", {}),
    """.parquet""": ("""parquet""", {}),
    """.arrow""": ("""arrow""", {}),
    """.txt""": ("""text""", {}),
}
_EXTENSION_TO_MODULE.update({ext: ("""imagefolder""", {}) for ext in imagefolder.ImageFolder.EXTENSIONS})
_EXTENSION_TO_MODULE.update({ext.upper(): ("""imagefolder""", {}) for ext in imagefolder.ImageFolder.EXTENSIONS})
_EXTENSION_TO_MODULE.update({ext: ("""audiofolder""", {}) for ext in audiofolder.AudioFolder.EXTENSIONS})
_EXTENSION_TO_MODULE.update({ext.upper(): ("""audiofolder""", {}) for ext in audiofolder.AudioFolder.EXTENSIONS})
SCREAMING_SNAKE_CASE_:Optional[int]					=						{"""imagefolder""", """audiofolder"""}

# Used to filter data files based on extensions given a module name
SCREAMING_SNAKE_CASE_:Dict[str, List[str]]					=						{}
for _ext, (_module, _) in _EXTENSION_TO_MODULE.items():
     _MODULE_TO_EXTENSIONS.setdefault(_module, []).append(_ext)

_MODULE_TO_EXTENSIONS["imagefolder"].append(""".zip""")




_MODULE_TO_EXTENSIONS["audiofolder"].append(""".zip""")


