

'''simple docstring'''




import time
from dataclasses import dataclass
from multiprocessing import Pool
from unittest import TestCase
from unittest.mock import patch

import multiprocess
import numpy as np
import pytest

from datasets.utils.py_utils import (
    NestedDataStructure,
    asdict,
    iflatmap_unordered,
    map_nested,
    temp_seed,
    temporary_assignment,
    zip_dict,
)

from .utils import require_tf, require_torch




def 						UpperCamelCase	(			a			)				->     Dict:  # picklable for multiprocessing






     '''simple docstring'''
     return x.sum()




def 						UpperCamelCase	(			a			)				->     Optional[int]:  # picklable for multiprocessing






     '''simple docstring'''
     return i + 1






@dataclass
class   _SCREAMING_SNAKE_CASE   :
 __SCREAMING_SNAKE_CASE					:int
 __SCREAMING_SNAKE_CASE					:str






class   _SCREAMING_SNAKE_CASE   (   __a     ):



 def 			snake_case__  (   self		:   List[Any]      ):
      __magic_name__						       =    {}
      __magic_name__						       =    []
      __magic_name__						       =    1
      __magic_name__						       =    [1, 2]
      __magic_name__						       =    {'''a''': 1, '''b''': 2}
      __magic_name__						       =    {'''a''': [1, 2], '''b''': [3, 4]}
      __magic_name__						       =    {'''a''': {'''1''': 1}, '''b''': 2}
      __magic_name__						       =    {'''a''': 1, '''b''': 2, '''c''': 3, '''d''': 4}
      __magic_name__						       =    {}
      __magic_name__						       =    []
      __magic_name__						       =    2
      __magic_name__						       =    [2, 3]
      __magic_name__						       =    {'''a''': 2, '''b''': 3}
      __magic_name__						       =    {'''a''': [2, 3], '''b''': [4, 5]}
      __magic_name__						       =    {'''a''': {'''1''': 2}, '''b''': 3}
      __magic_name__						       =    {'''a''': 2, '''b''': 3, '''c''': 4, '''d''': 5}
      self.assertEqual(map_nested(a__			,				a__      )			,				a__      )
      self.assertEqual(map_nested(a__			,				a__      )			,				a__      )
      self.assertEqual(map_nested(a__			,				a__      )			,				a__      )
      self.assertEqual(map_nested(a__			,				a__      )			,				a__      )
      self.assertEqual(map_nested(a__			,				a__      )			,				a__      )
      self.assertEqual(map_nested(a__			,				a__      )			,				a__      )
      self.assertEqual(map_nested(a__			,				a__      )			,				a__      )
      self.assertEqual(map_nested(a__			,				a__      )			,				a__      )

      __magic_name__						       =    2
      self.assertEqual(map_nested(a__			,				a__			,				num_proc=a__      )			,				a__      )
      self.assertEqual(map_nested(a__			,				a__			,				num_proc=a__      )			,				a__      )
      self.assertEqual(map_nested(a__			,				a__			,				num_proc=a__      )			,				a__      )
      self.assertEqual(map_nested(a__			,				a__			,				num_proc=a__      )			,				a__      )
      self.assertEqual(map_nested(a__			,				a__			,				num_proc=a__      )			,				a__      )
      self.assertEqual(map_nested(a__			,				a__			,				num_proc=a__      )			,				a__      )
      self.assertEqual(map_nested(a__			,				a__			,				num_proc=a__      )			,				a__      )
      self.assertEqual(map_nested(a__			,				a__			,				num_proc=a__      )			,				a__      )

      __magic_name__						       =    {'''a''': np.eye(2      ), '''b''': np.zeros(3      ), '''c''': np.ones(2      )}
      __magic_name__						       =    {'''a''': 2, '''b''': 0, '''c''': 2}
      __magic_name__						       =    {
          '''a''': np.eye(2      ).astype(a__      ),
          '''b''': np.zeros(3      ).astype(a__      ),
          '''c''': np.ones(2      ).astype(a__      ),
      }
      self.assertEqual(map_nested(a__			,				a__			,				map_numpy=a__      )			,				a__      )
      self.assertEqual(
          {k: v.tolist() for k, v in map_nested(a__			,				a__			,				map_numpy=a__      ).items()}			,				{k: v.tolist() for k, v in expected_map_nested_sna_int.items()}			,				)
      self.assertEqual(map_nested(a__			,				a__			,				map_numpy=a__			,				num_proc=a__      )			,				a__      )
      self.assertEqual(
          {k: v.tolist() for k, v in map_nested(a__			,				a__			,				map_numpy=a__			,				num_proc=a__      ).items()}			,				{k: v.tolist() for k, v in expected_map_nested_sna_int.items()}			,				)
      with self.assertRaises(a__      ):  # can't pickle a local lambda
           map_nested(lambda a__      : x + 1			,				a__			,				num_proc=a__      )



 def 			snake_case__  (   self		:   int      ):
      __magic_name__						       =    {'''a''': 1, '''b''': 2}
      __magic_name__						       =    {'''a''': 3, '''b''': 4}
      __magic_name__						       =    {'''a''': 5, '''b''': 6}
      __magic_name__						       =    sorted([('''a''', (1, 3, 5)), ('''b''', (2, 4, 6))]      )
      self.assertEqual(sorted(zip_dict(a__			,				a__			,				a__      )      )			,				a__      )



 def 			snake_case__  (   self		:   Tuple      ):






      class   _SCREAMING_SNAKE_CASE   :
       __SCREAMING_SNAKE_CASE					:int      	=					"""bar"""


      __magic_name__						       =    Foo()
      self.assertEqual(foo.my_attr			,				'''bar'''      )
      with temporary_assignment(a__			,				'''my_attr'''			,				'''BAR'''      ):
           self.assertEqual(foo.my_attr			,				'''BAR'''      )
      self.assertEqual(foo.my_attr			,				'''bar'''      )




@pytest.mark.parametrize(
    '''iterable_length, num_proc, expected_num_proc'''		,						[
        (1, None, 1),
        (1, 1, 1),
        (2, None, 1),
        (2, 1, 1),
        (2, 2, 1),
        (2, 3, 1),
        (3, 2, 1),
        (16, 16, 16),
        (16, 17, 16),
        (17, 16, 16),
    ]		,						)
def 						UpperCamelCase	(			a		,						a		,						a			)				->     str:






     '''simple docstring'''
     with patch('''datasets.utils.py_utils._single_map_nested'''			) as mock_single_map_nested, patch(
         '''datasets.parallel.parallel.Pool'''			) as mock_multiprocessing_pool:
          __magic_name__						       =    {F'''{i}''': i for i in range(a			)}
          __magic_name__						       =    map_nested(lambda a			: x + 10		,						a		,						num_proc=a		,						parallel_min_length=16			)
          if expected_num_proc == 1:
               assert mock_single_map_nested.called
               assert not mock_multiprocessing_pool.called
          else:
               assert not mock_single_map_nested.called
               assert mock_multiprocessing_pool.called
               assert mock_multiprocessing_pool.call_args[0][0] == expected_num_proc






class   _SCREAMING_SNAKE_CASE   (   __a     ):



 @require_tf
 def 			snake_case__  (   self		:   Optional[Any]      ):
      import tensorflow as tf
      from tensorflow.keras import layers

      __magic_name__						       =    layers.Dense(2      )

      def gen_random_output():
           __magic_name__						       =    tf.random.uniform((1, 3)      )
           return model(a__      ).numpy()

      with temp_seed(42			,				set_tensorflow=a__      ):
           __magic_name__						       =    gen_random_output()
      with temp_seed(42			,				set_tensorflow=a__      ):
           __magic_name__						       =    gen_random_output()
      __magic_name__						       =    gen_random_output()

      np.testing.assert_equal(a__			,				a__      )
      self.assertGreater(np.abs(outa - outa      ).sum()			,				0      )



 @require_torch
 def 			snake_case__  (   self		:   Optional[int]      ):
      import torch

      def gen_random_output():
           __magic_name__						       =    torch.nn.Linear(3			,				2      )
           __magic_name__						       =    torch.rand(1			,				3      )
           return model(a__      ).detach().numpy()

      with temp_seed(42			,				set_pytorch=a__      ):
           __magic_name__						       =    gen_random_output()
      with temp_seed(42			,				set_pytorch=a__      ):
           __magic_name__						       =    gen_random_output()
      __magic_name__						       =    gen_random_output()

      np.testing.assert_equal(a__			,				a__      )
      self.assertGreater(np.abs(outa - outa      ).sum()			,				0      )



 def 			snake_case__  (   self		:   Optional[Any]      ):
      def gen_random_output():
           return np.random.rand(1			,				3      )

      with temp_seed(42      ):
           __magic_name__						       =    gen_random_output()
      with temp_seed(42      ):
           __magic_name__						       =    gen_random_output()
      __magic_name__						       =    gen_random_output()

      np.testing.assert_equal(a__			,				a__      )
      self.assertGreater(np.abs(outa - outa      ).sum()			,				0      )




@pytest.mark.parametrize('''input_data'''		,						[{}]			)
def 						UpperCamelCase	(			a			)				->     str:






     '''simple docstring'''
     __magic_name__						       =    NestedDataStructure(a			).data
     assert output_data == input_data




@pytest.mark.parametrize(
    '''data, expected_output'''		,						[
        ({}, []),
        ([], []),
        ('''foo''', ['''foo''']),
        (['''foo''', '''bar'''], ['''foo''', '''bar''']),
        ([['''foo''', '''bar''']], ['''foo''', '''bar''']),
        ([[['''foo'''], ['''bar''']]], ['''foo''', '''bar''']),
        ([[['''foo'''], '''bar''']], ['''foo''', '''bar''']),
        ({'''a''': 1, '''b''': 2}, [1, 2]),
        ({'''a''': [1, 2], '''b''': [3, 4]}, [1, 2, 3, 4]),
        ({'''a''': [[1, 2]], '''b''': [[3, 4]]}, [1, 2, 3, 4]),
        ({'''a''': [[1, 2]], '''b''': [3, 4]}, [1, 2, 3, 4]),
        ({'''a''': [[[1], [2]]], '''b''': [[[3], [4]]]}, [1, 2, 3, 4]),
        ({'''a''': [[[1], [2]]], '''b''': [[3, 4]]}, [1, 2, 3, 4]),
        ({'''a''': [[[1], [2]]], '''b''': [3, 4]}, [1, 2, 3, 4]),
        ({'''a''': [[[1], [2]]], '''b''': [3, [4]]}, [1, 2, 3, 4]),
        ({'''a''': {'''1''': 1}, '''b''': 2}, [1, 2]),
        ({'''a''': {'''1''': [1]}, '''b''': 2}, [1, 2]),
        ({'''a''': {'''1''': [1]}, '''b''': [2]}, [1, 2]),
    ]		,						)
def 						UpperCamelCase	(			a		,						a			)				->     Dict:






     '''simple docstring'''
     __magic_name__						       =    NestedDataStructure(a			).flatten()
     assert output == expected_output




def 						UpperCamelCase	(			)				->     List[Any]:






     '''simple docstring'''
     __magic_name__						       =    A(x=1		,						y='''foobar'''			)
     __magic_name__						       =    {'''x''': 1, '''y''': '''foobar'''}
     assert asdict(a			) == expected_output

     __magic_name__						       =    {'''a''': {'''b''': A(x=10		,						y='''foo'''			)}, '''c''': [A(x=20		,						y='''bar'''			)]}
     __magic_name__						       =    {'''a''': {'''b''': {'''x''': 10, '''y''': '''foo'''}}, '''c''': [{'''x''': 20, '''y''': '''bar'''}]}
     assert asdict(a			) == expected_output

     with pytest.raises(a			):
          asdict([1, A(x=10		,						y='''foo'''			)]			)




def 						UpperCamelCase	(			a			)				->     Optional[Any]:






     '''simple docstring'''
     return text.split()




def 						UpperCamelCase	(			a			)				->     Any:






     '''simple docstring'''
     yield (time.time(), content)
     time.sleep(2			)
     yield (time.time(), content)




def 						UpperCamelCase	(			)				->     List[str]:






     '''simple docstring'''
     with Pool(2			) as pool:
          __magic_name__						       =    list(iflatmap_unordered(a		,						_split_text		,						kwargs_iterable=[{'''text''': '''hello there'''}] * 10			)			)
          assert out.count('''hello'''			) == 10
          assert out.count('''there'''			) == 10
          assert len(a			) == 20

     # check multiprocess from pathos (uses dill for pickling)
     with multiprocess.Pool(2			) as pool:
          __magic_name__						       =    list(iflatmap_unordered(a		,						_split_text		,						kwargs_iterable=[{'''text''': '''hello there'''}] * 10			)			)
          assert out.count('''hello'''			) == 10
          assert out.count('''there'''			) == 10
          assert len(a			) == 20

     # check that we get items as fast as possible
     with Pool(2			) as pool:
          __magic_name__						       =    []
          for yield_time, content in iflatmap_unordered(
              a		,						_aseconds_generator_of_aitems_with_timing		,						kwargs_iterable=[{'''content''': '''a'''}, {'''content''': '''b'''}]			):
               assert yield_time < time.time() + 0.1, "we should each item directly after it was yielded"
               out.append(a			)
          assert out.count('''a'''			) == 2
          assert out.count('''b'''			) == 2
          assert len(a			) == 4









'''simple docstring'''




import re

from flax.core.frozen_dict import freeze
from flax.traverse_util import flatten_dict, unflatten_dict
from jax.experimental import PartitionSpec as P


# Sentinels
_lowerCAmelCase         =						object()

# For specifying empty leaf dict `{}`
_lowerCAmelCase         =						object()




def 						UpperCamelCase	(			a		,						a			)				->     Union[str, Any]:






     '''simple docstring'''
     __magic_name__						       =    tuple((re.compile(x + '''$'''			) for x in qs)			)
     for i in range(len(a			) - len(a			) + 1			):
          __magic_name__						       =    [x.match(a			) for x, y in zip(a		,						ks[i:]			)]
          if matches and all(a			):
               return True
     return False




def 						UpperCamelCase	(			a			)				->     Tuple:






     '''simple docstring'''
     def replace(a		,						a			):
          for rule, replacement in rules:
               if _match(a		,						a			):
                    return replacement
          return val

     return replace




def 						UpperCamelCase	(			)				->     Union[str, Any]:






     '''simple docstring'''
     return [
         # embeddings
         (("transformer", "wpe", "embedding"), P('''mp'''		,						a			)),
         (("transformer", "wte", "embedding"), P('''mp'''		,						a			)),
         # atention
         (("attention", "(q_proj|k_proj|v_proj)", "kernel"), P(a		,						'''mp'''			)),
         (("attention", "out_proj", "kernel"), P('''mp'''		,						a			)),
         (("attention", "out_proj", "bias"), None),
         # mlp
         (("mlp", "c_fc", "kernel"), P(a		,						'''mp'''			)),
         (("mlp", "c_fc", "bias"), P('''mp'''			)),
         (("mlp", "c_proj", "kernel"), P('''mp'''		,						a			)),
         (("mlp", "c_proj", "bias"), None),
         # layer norms
         ((r"ln_\d+", "bias"), None),
         ((r"\d+", r"ln_\d+", "scale"), None),
         (("ln_f", "bias"), None),
         (("ln_f", "scale"), None),
     ]




def 						UpperCamelCase	(			a			)				->     str:






     '''simple docstring'''
     __magic_name__						       =    _get_partition_rules()
     __magic_name__						       =    _replacement_rules(a			)
     __magic_name__						       =    {k: _unmatched for k in flatten_dict(a			)}
     __magic_name__						       =    {k: replace(a		,						a			) for k, v in initd.items()}
     assert _unmatched not in result.values(), "Incomplete partition spec."
     return freeze(unflatten_dict(a			)			)






