


"""simple docstring"""







import pytest

from datasets.utils.sharding import _distribute_shards, _number_of_shards_in_gen_kwargs, _split_gen_kwargs
@pytest.mark.parametrize(
    "kwargs, expected"    ,				[
        ({"num_shards": 0, "max_num_jobs": 1}, []),
        ({"num_shards": 1_0, "max_num_jobs": 1}, [range(1_0    )]),
        ({"num_shards": 1_0, "max_num_jobs": 1_0}, [range(SCREAMING_SNAKE_CASE    ,				i + 1    ) for i in range(1_0    )]),
        ({"num_shards": 1, "max_num_jobs": 1_0}, [range(1    )]),
        ({"num_shards": 1_0, "max_num_jobs": 3}, [range(0    ,				4    ), range(4    ,				7    ), range(7    ,				1_0    )]),
        ({"num_shards": 3, "max_num_jobs": 1_0}, [range(0    ,				1    ), range(1    ,				2    ), range(2    ,				3    )]),
    ]    ,				)
def  a__      (    SCREAMING_SNAKE_CASE   :   Any    ,				SCREAMING_SNAKE_CASE   :   str    ):


  '''simple docstring'''







  lowerCAmelCase					:					List[str]								=							_distribute_shards(**SCREAMING_SNAKE_CASE    )
  assert out == expected
@pytest.mark.parametrize(
    "gen_kwargs, max_num_jobs, expected"    ,				[
        ({"foo": 0}, 1_0, [{"foo": 0}]),
        ({"shards": [0, 1, 2, 3]}, 1, [{"shards": [0, 1, 2, 3]}]),
        ({"shards": [0, 1, 2, 3]}, 4, [{"shards": [0]}, {"shards": [1]}, {"shards": [2]}, {"shards": [3]}]),
        ({"shards": [0, 1]}, 4, [{"shards": [0]}, {"shards": [1]}]),
        ({"shards": [0, 1, 2, 3]}, 2, [{"shards": [0, 1]}, {"shards": [2, 3]}]),
    ]    ,				)
def  a__      (    SCREAMING_SNAKE_CASE   :   Any    ,				SCREAMING_SNAKE_CASE   :   Any    ,				SCREAMING_SNAKE_CASE   :   Optional[int]    ):


  '''simple docstring'''







  lowerCAmelCase					:					List[str]								=							_split_gen_kwargs(SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE    )
  assert out == expected





@pytest.mark.parametrize(
    "gen_kwargs, expected"    ,				[
        ({"foo": 0}, 1),
        ({"shards": [0]}, 1),
        ({"shards": [0, 1, 2, 3]}, 4),
        ({"shards": [0, 1, 2, 3], "foo": 0}, 4),
        ({"shards": [0, 1, 2, 3], "other": (0, 1)}, 4),
        ({"shards": [0, 1, 2, 3], "shards2": [0, 1]}, RuntimeError),
    ]    ,				)
def  a__      (    SCREAMING_SNAKE_CASE   :   List[str]    ,				SCREAMING_SNAKE_CASE   :   Any    ):


  '''simple docstring'''







  if expected is RuntimeError:
    with pytest.raises(SCREAMING_SNAKE_CASE    ):
      _number_of_shards_in_gen_kwargs(SCREAMING_SNAKE_CASE    )
  else:
    lowerCAmelCase					:					Optional[Any]								=							_number_of_shards_in_gen_kwargs(SCREAMING_SNAKE_CASE    )
    assert out == expected









"""simple docstring"""







import pytest

from datasets.parallel import ParallelBackendConfig, parallel_backend
from datasets.utils.py_utils import map_nested

from .utils import require_dill_gt_0_3_2, require_joblibspark, require_not_windows
def  a__      (    SCREAMING_SNAKE_CASE   :   str    ):  # picklable for multiprocessing


  '''simple docstring'''







  return i + 1
@require_dill_gt_0_3_2
@require_joblibspark
@require_not_windows
def  a__      (    ):


  '''simple docstring'''







  with parallel_backend("spark"    ):
    assert ParallelBackendConfig.backend_name == "spark"

  lowerCAmelCase					:					List[str]								=							[1, 2, 3]
  with pytest.raises(SCREAMING_SNAKE_CASE    ):
    with parallel_backend("unsupported backend"    ):
      map_nested(SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE    ,				num_proc=2    )

  with pytest.raises(SCREAMING_SNAKE_CASE    ):
    with parallel_backend("unsupported backend"    ):
      map_nested(SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE    ,				num_proc=-1    )





@require_dill_gt_0_3_2
@require_joblibspark
@require_not_windows
@pytest.mark.parametrize("num_proc"    ,				[2, -1]    )
def  a__      (    SCREAMING_SNAKE_CASE   :   Tuple    ):


  '''simple docstring'''







  lowerCAmelCase					:					Tuple								=							[1, 2]
  lowerCAmelCase					:					int								=							{"a": 1, "b": 2}
  lowerCAmelCase					:					List[str]								=							{"a": [1, 2], "b": [3, 4]}
  lowerCAmelCase					:					Dict								=							{"a": {"1": 1}, "b": 2}
  lowerCAmelCase					:					Tuple								=							{"a": 1, "b": 2, "c": 3, "d": 4}
  lowerCAmelCase					:					Any								=							[2, 3]
  lowerCAmelCase					:					Any								=							{"a": 2, "b": 3}
  lowerCAmelCase					:					Optional[int]								=							{"a": [2, 3], "b": [4, 5]}
  lowerCAmelCase					:					Optional[int]								=							{"a": {"1": 2}, "b": 3}
  lowerCAmelCase					:					str								=							{"a": 2, "b": 3, "c": 4, "d": 5}

  with parallel_backend("spark"    ):
    assert map_nested(SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE    ,				num_proc=SCREAMING_SNAKE_CASE    ) == expected_map_nested_sa
    assert map_nested(SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE    ,				num_proc=SCREAMING_SNAKE_CASE    ) == expected_map_nested_sa
    assert map_nested(SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE    ,				num_proc=SCREAMING_SNAKE_CASE    ) == expected_map_nested_sa
    assert map_nested(SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE    ,				num_proc=SCREAMING_SNAKE_CASE    ) == expected_map_nested_sa
    assert map_nested(SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE    ,				num_proc=SCREAMING_SNAKE_CASE    ) == expected_map_nested_sa





