


'''simple docstring'''






import time
from dataclasses import dataclass
from multiprocessing import Pool
from unittest import TestCase
from unittest.mock import patch

import multiprocess
import numpy as np
import pytest

from datasets.utils.py_utils import (
    NestedDataStructure,
    asdict,
    iflatmap_unordered,
    map_nested,
    temp_seed,
    temporary_assignment,
    zip_dict,
)

from .utils import require_tf, require_torch






def 		_A			(lowerCAmelCase__					:str						)				->					int:  # picklable for multiprocessing




     '''simple docstring'''



     return x.sum()






def 		_A			(lowerCAmelCase__					:int						)				->					int:  # picklable for multiprocessing




     '''simple docstring'''



     return i + 1




@dataclass
class 	a       :
 _lowerCAmelCase	   =		4_2
 _lowerCAmelCase	   =		4_2





class 	a       (       _SCREAMING_SNAKE_CASE    ):
 def 							__UpperCAmelCase						(	self  )   ->							Optional[Any]:
      _a      =							{}
      _a      =							[]
      _a      =							1
      _a      =							[1, 2]
      _a      =							{'a': 1, 'b': 2}
      _a      =							{'a': [1, 2], 'b': [3, 4]}
      _a      =							{'a': {'1': 1}, 'b': 2}
      _a      =							{'a': 1, 'b': 2, 'c': 3, 'd': 4}
      _a      =							{}
      _a      =							[]
      _a      =							2
      _a      =							[2, 3]
      _a      =							{'a': 2, 'b': 3}
      _a      =							{'a': [2, 3], 'b': [4, 5]}
      _a      =							{'a': {'1': 2}, 'b': 3}
      _a      =							{'a': 2, 'b': 3, 'c': 4, 'd': 5}
      self.assertEqual(map_nested(__magic_name__      ,			__magic_name__  )      ,			__magic_name__  )
      self.assertEqual(map_nested(__magic_name__      ,			__magic_name__  )      ,			__magic_name__  )
      self.assertEqual(map_nested(__magic_name__      ,			__magic_name__  )      ,			__magic_name__  )
      self.assertEqual(map_nested(__magic_name__      ,			__magic_name__  )      ,			__magic_name__  )
      self.assertEqual(map_nested(__magic_name__      ,			__magic_name__  )      ,			__magic_name__  )
      self.assertEqual(map_nested(__magic_name__      ,			__magic_name__  )      ,			__magic_name__  )
      self.assertEqual(map_nested(__magic_name__      ,			__magic_name__  )      ,			__magic_name__  )
      self.assertEqual(map_nested(__magic_name__      ,			__magic_name__  )      ,			__magic_name__  )

      _a      =							2
      self.assertEqual(map_nested(__magic_name__      ,			__magic_name__      ,			num_proc=__magic_name__  )      ,			__magic_name__  )
      self.assertEqual(map_nested(__magic_name__      ,			__magic_name__      ,			num_proc=__magic_name__  )      ,			__magic_name__  )
      self.assertEqual(map_nested(__magic_name__      ,			__magic_name__      ,			num_proc=__magic_name__  )      ,			__magic_name__  )
      self.assertEqual(map_nested(__magic_name__      ,			__magic_name__      ,			num_proc=__magic_name__  )      ,			__magic_name__  )
      self.assertEqual(map_nested(__magic_name__      ,			__magic_name__      ,			num_proc=__magic_name__  )      ,			__magic_name__  )
      self.assertEqual(map_nested(__magic_name__      ,			__magic_name__      ,			num_proc=__magic_name__  )      ,			__magic_name__  )
      self.assertEqual(map_nested(__magic_name__      ,			__magic_name__      ,			num_proc=__magic_name__  )      ,			__magic_name__  )
      self.assertEqual(map_nested(__magic_name__      ,			__magic_name__      ,			num_proc=__magic_name__  )      ,			__magic_name__  )

      _a      =							{'a': np.eye(2  ), 'b': np.zeros(3  ), 'c': np.ones(2  )}
      _a      =							{'a': 2, 'b': 0, 'c': 2}
      _a      =							{
          'a': np.eye(2  ).astype(__magic_name__  ),
          'b': np.zeros(3  ).astype(__magic_name__  ),
          'c': np.ones(2  ).astype(__magic_name__  ),
      }
      self.assertEqual(map_nested(__magic_name__      ,			__magic_name__      ,			map_numpy=__magic_name__  )      ,			__magic_name__  )
      self.assertEqual(
          {k: v.tolist() for k, v in map_nested(__magic_name__      ,			__magic_name__      ,			map_numpy=__magic_name__  ).items()}      ,			{k: v.tolist() for k, v in expected_map_nested_sna_int.items()}      ,			)
      self.assertEqual(map_nested(__magic_name__      ,			__magic_name__      ,			map_numpy=__magic_name__      ,			num_proc=__magic_name__  )      ,			__magic_name__  )
      self.assertEqual(
          {k: v.tolist() for k, v in map_nested(__magic_name__      ,			__magic_name__      ,			map_numpy=__magic_name__      ,			num_proc=__magic_name__  ).items()}      ,			{k: v.tolist() for k, v in expected_map_nested_sna_int.items()}      ,			)
      with self.assertRaises(__magic_name__  ):  # can't pickle a local lambda
           map_nested(lambda __magic_name__  : x + 1      ,			__magic_name__      ,			num_proc=__magic_name__  )
 def 							__UpperCAmelCase						(	self  )   ->							List[Any]:
      _a      =							{'a': 1, 'b': 2}
      _a      =							{'a': 3, 'b': 4}
      _a      =							{'a': 5, 'b': 6}
      _a      =							sorted([('a', (1, 3, 5)), ('b', (2, 4, 6))]  )
      self.assertEqual(sorted(zip_dict(__magic_name__      ,			__magic_name__      ,			__magic_name__  )  )      ,			__magic_name__  )




 def 							__UpperCAmelCase						(	self  )   ->							Union[str, Any]:




      class 	a       :
       _lowerCAmelCase	   =		"""bar"""





      _a      =							Foo()
      self.assertEqual(foo.my_attr      ,			'bar'  )
      with temporary_assignment(__magic_name__      ,			'my_attr'      ,			'BAR'  ):
           self.assertEqual(foo.my_attr      ,			'BAR'  )
      self.assertEqual(foo.my_attr      ,			'bar'  )






@pytest.mark.parametrize(
    'iterable_length, num_proc, expected_num_proc'			,  [
        (1, None, 1),
        (1, 1, 1),
        (2, None, 1),
        (2, 1, 1),
        (2, 2, 1),
        (2, 3, 1),
        (3, 2, 1),
        (16, 16, 16),
        (16, 17, 16),
        (17, 16, 16),
    ]			,  )
def 		_A			(lowerCAmelCase__					:int			,  lowerCAmelCase__					:str			,  lowerCAmelCase__					:str						)				->					str:




     '''simple docstring'''



     with patch('datasets.utils.py_utils._single_map_nested'						) as mock_single_map_nested, patch(
         'datasets.parallel.parallel.Pool'						) as mock_multiprocessing_pool:
          _a      =							{f'{i}': i for i in range(lowerCAmelCase__						)}
          _a      =							map_nested(lambda lowerCAmelCase__						: x + 10			,  lowerCAmelCase__			,  num_proc=lowerCAmelCase__			,  parallel_min_length=16						)
          if expected_num_proc == 1:
               assert mock_single_map_nested.called
               assert not mock_multiprocessing_pool.called
          else:
               assert not mock_single_map_nested.called
               assert mock_multiprocessing_pool.called
               assert mock_multiprocessing_pool.call_args[0][0] == expected_num_proc




class 	a       (       _SCREAMING_SNAKE_CASE    ):
 @require_tf
 def 							__UpperCAmelCase						(	self  )   ->							Union[str, Any]:
      import tensorflow as tf
      from tensorflow.keras import layers

      _a      =							layers.Dense(2  )

      def gen_random_output():
           _a      =							tf.random.uniform((1, 3)  )
           return model(__magic_name__  ).numpy()

      with temp_seed(42      ,			set_tensorflow=__magic_name__  ):
           _a      =							gen_random_output()
      with temp_seed(42      ,			set_tensorflow=__magic_name__  ):
           _a      =							gen_random_output()
      _a      =							gen_random_output()

      np.testing.assert_equal(__magic_name__      ,			__magic_name__  )
      self.assertGreater(np.abs(outa - outa  ).sum()      ,			0  )
 @require_torch
 def 							__UpperCAmelCase						(	self  )   ->							Any:
      import torch

      def gen_random_output():
           _a      =							torch.nn.Linear(3      ,			2  )
           _a      =							torch.rand(1      ,			3  )
           return model(__magic_name__  ).detach().numpy()

      with temp_seed(42      ,			set_pytorch=__magic_name__  ):
           _a      =							gen_random_output()
      with temp_seed(42      ,			set_pytorch=__magic_name__  ):
           _a      =							gen_random_output()
      _a      =							gen_random_output()

      np.testing.assert_equal(__magic_name__      ,			__magic_name__  )
      self.assertGreater(np.abs(outa - outa  ).sum()      ,			0  )




 def 							__UpperCAmelCase						(	self  )   ->							str:
      def gen_random_output():
           return np.random.rand(1      ,			3  )

      with temp_seed(42  ):
           _a      =							gen_random_output()
      with temp_seed(42  ):
           _a      =							gen_random_output()
      _a      =							gen_random_output()

      np.testing.assert_equal(__magic_name__      ,			__magic_name__  )
      self.assertGreater(np.abs(outa - outa  ).sum()      ,			0  )






@pytest.mark.parametrize('input_data'			,  [{}]						)
def 		_A			(lowerCAmelCase__					:Optional[int]						)				->					List[str]:




     '''simple docstring'''



     _a      =							NestedDataStructure(lowerCAmelCase__						).data
     assert output_data == input_data






@pytest.mark.parametrize(
    'data, expected_output'			,  [
        ({}, []),
        ([], []),
        ('foo', ['foo']),
        (['foo', 'bar'], ['foo', 'bar']),
        ([['foo', 'bar']], ['foo', 'bar']),
        ([[['foo'], ['bar']]], ['foo', 'bar']),
        ([[['foo'], 'bar']], ['foo', 'bar']),
        ({'a': 1, 'b': 2}, [1, 2]),
        ({'a': [1, 2], 'b': [3, 4]}, [1, 2, 3, 4]),
        ({'a': [[1, 2]], 'b': [[3, 4]]}, [1, 2, 3, 4]),
        ({'a': [[1, 2]], 'b': [3, 4]}, [1, 2, 3, 4]),
        ({'a': [[[1], [2]]], 'b': [[[3], [4]]]}, [1, 2, 3, 4]),
        ({'a': [[[1], [2]]], 'b': [[3, 4]]}, [1, 2, 3, 4]),
        ({'a': [[[1], [2]]], 'b': [3, 4]}, [1, 2, 3, 4]),
        ({'a': [[[1], [2]]], 'b': [3, [4]]}, [1, 2, 3, 4]),
        ({'a': {'1': 1}, 'b': 2}, [1, 2]),
        ({'a': {'1': [1]}, 'b': 2}, [1, 2]),
        ({'a': {'1': [1]}, 'b': [2]}, [1, 2]),
    ]			,  )
def 		_A			(lowerCAmelCase__					:Optional[int]			,  lowerCAmelCase__					:List[str]						)				->					str:




     '''simple docstring'''



     _a      =							NestedDataStructure(lowerCAmelCase__						).flatten()
     assert output == expected_output






def 		_A			()				->					str:




     '''simple docstring'''



     _a      =							A(x=1			,  y='foobar'						)
     _a      =							{'x': 1, 'y': 'foobar'}
     assert asdict(lowerCAmelCase__						) == expected_output

     _a      =							{'a': {'b': A(x=10			,  y='foo'						)}, 'c': [A(x=20			,  y='bar'						)]}
     _a      =							{'a': {'b': {'x': 10, 'y': 'foo'}}, 'c': [{'x': 20, 'y': 'bar'}]}
     assert asdict(lowerCAmelCase__						) == expected_output

     with pytest.raises(lowerCAmelCase__						):
          asdict([1, A(x=10			,  y='foo'						)]						)






def 		_A			(lowerCAmelCase__					:str						)				->					Union[str, Any]:




     '''simple docstring'''



     return text.split()






def 		_A			(lowerCAmelCase__					:Dict						)				->					str:




     '''simple docstring'''



     yield (time.time(), content)
     time.sleep(2						)
     yield (time.time(), content)






def 		_A			()				->					Union[str, Any]:




     '''simple docstring'''



     with Pool(2						) as pool:
          _a      =							list(iflatmap_unordered(lowerCAmelCase__			,  _split_text			,  kwargs_iterable=[{'text': 'hello there'}] * 10						)						)
          assert out.count('hello'						) == 10
          assert out.count('there'						) == 10
          assert len(lowerCAmelCase__						) == 20

     # check multiprocess from pathos (uses dill for pickling)
     with multiprocess.Pool(2						) as pool:
          _a      =							list(iflatmap_unordered(lowerCAmelCase__			,  _split_text			,  kwargs_iterable=[{'text': 'hello there'}] * 10						)						)
          assert out.count('hello'						) == 10
          assert out.count('there'						) == 10
          assert len(lowerCAmelCase__						) == 20

     # check that we get items as fast as possible
     with Pool(2						) as pool:
          _a      =							[]
          for yield_time, content in iflatmap_unordered(
              lowerCAmelCase__			,  _aseconds_generator_of_aitems_with_timing			,  kwargs_iterable=[{'content': 'a'}, {'content': 'b'}]						):
               assert yield_time < time.time() + 0.1, "we should each item directly after it was yielded"
               out.append(lowerCAmelCase__						)
          assert out.count('a'						) == 2
          assert out.count('b'						) == 2
          assert len(lowerCAmelCase__						) == 4





'''simple docstring'''






from typing import Dict, List, Optional, Union

import numpy as np

from .feature_extraction_utils import BatchFeature, FeatureExtractionMixin
from .utils import PaddingStrategy, TensorType, is_tf_tensor, is_torch_tensor, logging, to_numpy


a_   :      List[str]				    =      logging.get_logger(__name__)




class 	a       (       _SCREAMING_SNAKE_CASE    ):
 def __init__(	self ,			__magic_name__ ,			__magic_name__ ,			__magic_name__ ,			**__magic_name__  )   ->							Any:
      _a      =							feature_size
      _a      =							sampling_rate
      _a      =							padding_value

      _a      =							kwargs.pop('padding_side' ,			'right'  )
      _a      =							kwargs.pop('return_attention_mask' ,			__magic_name__  )

      super().__init__(**__magic_name__  )
 def 							__UpperCAmelCase						(	self ,			__magic_name__ ,			__magic_name__ = True ,			__magic_name__ = None ,			__magic_name__ = False ,			__magic_name__ = None ,			__magic_name__ = None ,			__magic_name__ = None ,			)   ->							BatchFeature:
      # If we have a list of dicts, let's convert it in a dict of lists
      # We do this to allow using this method as a collate_fn function in PyTorch Dataloader
      if isinstance(__magic_name__ ,			(list, tuple)  ) and isinstance(processed_features[0] ,			(dict, BatchFeature)  ):
           _a      =							{
               key: [example[key] for example in processed_features] for key in processed_features[0].keys()
           }

      # The model's main input name, usually `input_values`, has be passed for padding
      if self.model_input_names[0] not in processed_features:
           raise ValueError(
               'You should supply an instance of `transformers.BatchFeature` or list of `transformers.BatchFeature`'
               f' to this method that includes {self.model_input_names[0]}, but you provided'
               f' {list(processed_features.keys()  )}'  )

      _a      =							processed_features[self.model_input_names[0]]
      _a      =							(
          return_attention_mask if return_attention_mask is not None else self.return_attention_mask
      )

      if len(__magic_name__  ) == 0:
           if return_attention_mask:
                _a      =							[]
           return processed_features

      # If we have PyTorch/TF tensors or lists as inputs, we cast them as Numpy arrays
      # and rebuild them afterwards if no return_tensors is specified
      # Note that we lose the specific device the tensor may be on for PyTorch

      _a      =							required_input[0]
      if isinstance(__magic_name__ ,			(list, tuple)  ):
           # first_element might be an empty list/tuple in some edge cases so we grab the first non empty element.
           _a      =							0
           while len(required_input[index]  ) == 0:
                index += 1
           if index < len(__magic_name__  ):
                _a      =							required_input[index][0]

      if return_tensors is None:
           if is_tf_tensor(__magic_name__  ):
                _a      =							'tf'
           elif is_torch_tensor(__magic_name__  ):
                _a      =							'pt'
           elif isinstance(__magic_name__ ,			(int, float, list, tuple, np.ndarray)  ):
                _a      =							'np'
           else:
                raise ValueError(
                    f'type of {first_element} unknown: {type(__magic_name__  )}. '
                    'Should be one of a python, numpy, pytorch or tensorflow object.'  )

      for key, value in processed_features.items():
           if isinstance(value[0] ,			(int, float)  ):
                _a      =							to_numpy(__magic_name__  )
           else:
                _a      =							[to_numpy(__magic_name__  ) for v in value]

        # Convert padding_strategy in PaddingStrategy
      _a      =							self._get_padding_strategies(padding=__magic_name__ ,			max_length=__magic_name__  )

      _a      =							processed_features[self.model_input_names[0]]

      _a      =							len(__magic_name__  )
      if not all(len(__magic_name__  ) == batch_size for v in processed_features.values()  ):
           raise ValueError('Some items in the output dictionary have a different batch size than others.'  )

      _a      =							[]
      for i in range(__magic_name__  ):
           _a      =							{k: v[i] for k, v in processed_features.items()}
           # truncation
           _a      =							self._truncate(
               __magic_name__ ,			max_length=__magic_name__ ,			pad_to_multiple_of=__magic_name__ ,			truncation=__magic_name__ ,			)
           truncated_inputs.append(__magic_name__  )

      if padding_strategy == PaddingStrategy.LONGEST:
           # make sure that `max_length` cannot be longer than the longest truncated length
           _a      =							max(len(input_slice[self.model_input_names[0]]  ) for input_slice in truncated_inputs  )
           _a      =							PaddingStrategy.MAX_LENGTH

      _a      =							{}
      for i in range(__magic_name__  ):
           # padding
           _a      =							self._pad(
               truncated_inputs[i] ,			max_length=__magic_name__ ,			padding_strategy=__magic_name__ ,			pad_to_multiple_of=__magic_name__ ,			return_attention_mask=__magic_name__ ,			)

           for key, value in outputs.items():
                if key not in batch_outputs:
                     _a      =							[]
                if value.dtype is np.dtype(np.floataa  ):
                     _a      =							value.astype(np.floataa  )
                batch_outputs[key].append(__magic_name__  )

      return BatchFeature(__magic_name__ ,			tensor_type=__magic_name__  )
 def 							__UpperCAmelCase						(	self ,			__magic_name__ ,			__magic_name__ = None ,			__magic_name__ = PaddingStrategy.DO_NOT_PAD ,			__magic_name__ = None ,			__magic_name__ = None ,			)   ->							dict:
      _a      =							processed_features[self.model_input_names[0]]

      if padding_strategy == PaddingStrategy.LONGEST:
           _a      =							len(__magic_name__  )

      if max_length is not None and pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0):
           _a      =							((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of

      _a      =							padding_strategy != PaddingStrategy.DO_NOT_PAD and len(__magic_name__  ) < max_length

      if return_attention_mask and "attention_mask" not in processed_features:
           _a      =							np.ones(len(__magic_name__  ) ,			dtype=np.intaa  )

      if needs_to_be_padded:
           _a      =							max_length - len(__magic_name__  )
           if self.padding_side == "right":
                if return_attention_mask:
                     _a      =							np.pad(
                         processed_features['attention_mask'] ,			(0, difference)  )
                _a      =							((0, difference), (0, 0)) if self.feature_size > 1 else (0, difference)
                _a      =							np.pad(
                    __magic_name__ ,			__magic_name__ ,			'constant' ,			constant_values=self.padding_value  )
           elif self.padding_side == "left":
                if return_attention_mask:
                     _a      =							np.pad(
                         processed_features['attention_mask'] ,			(difference, 0)  )
                _a      =							((difference, 0), (0, 0)) if self.feature_size > 1 else (difference, 0)
                _a      =							np.pad(
                    __magic_name__ ,			__magic_name__ ,			'constant' ,			constant_values=self.padding_value  )
           else:
                raise ValueError('Invalid padding strategy:' + str(self.padding_side  )  )

      return processed_features
 def 							__UpperCAmelCase						(	self ,			__magic_name__ ,			__magic_name__ = None ,			__magic_name__ = None ,			__magic_name__ = None ,			)   ->							List[str]:
      if not truncation:
           return processed_features
      elif truncation and max_length is None:
           raise ValueError('When setting ``truncation=True``, make sure that ``max_length`` is defined.'  )

      _a      =							processed_features[self.model_input_names[0]]

      # find `max_length` that fits `pad_to_multiple_of`
      if max_length is not None and pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0):
           _a      =							((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of

      _a      =							len(__magic_name__  ) > max_length

      if needs_to_be_truncated:
           _a      =							processed_features[self.model_input_names[0]][:max_length]
           if "attention_mask" in processed_features:
                _a      =							processed_features['attention_mask'][:max_length]

      return processed_features




 def 							__UpperCAmelCase						(	self ,			__magic_name__=False ,			__magic_name__=None  )   ->							Dict:

      # Get padding strategy
      if padding is not False:
           if padding is True:
                _a      =							PaddingStrategy.LONGEST  # Default to pad to the longest sequence in the batch
           elif not isinstance(__magic_name__ ,			__magic_name__  ):
                _a      =							PaddingStrategy(__magic_name__  )
           elif isinstance(__magic_name__ ,			__magic_name__  ):
                _a      =							padding
      else:
           _a      =							PaddingStrategy.DO_NOT_PAD

      # Set max length if needed
      if max_length is None:
           if padding_strategy == PaddingStrategy.MAX_LENGTH:
                raise ValueError(
                    f'When setting ``padding={PaddingStrategy.MAX_LENGTH}``, make sure that max_length is defined'  )

        # Test if we have a padding value
      if padding_strategy != PaddingStrategy.DO_NOT_PAD and (self.padding_value is None):
           raise ValueError(
               'Asking to pad but the feature_extractor does not have a padding value. Please select a value to use'
               ' as `padding_value`. For example: `feature_extractor.padding_value = 0.0`.'  )

      return padding_strategy

