



'''simple docstring'''






import pytest

from datasets.utils.sharding import _distribute_shards, _number_of_shards_in_gen_kwargs, _split_gen_kwargs
@pytest.mark.parametrize(
    'kwargs, expected'			,      [
        ({'num_shards': 0, 'max_num_jobs': 1}, []),
        ({'num_shards': 10, 'max_num_jobs': 1}, [range(10    )]),
        ({'num_shards': 10, 'max_num_jobs': 10}, [range(_A			,      i + 1    ) for i in range(10    )]),
        ({'num_shards': 1, 'max_num_jobs': 10}, [range(1    )]),
        ({'num_shards': 10, 'max_num_jobs': 3}, [range(0			,      4    ), range(4			,      7    ), range(7			,      10    )]),
        ({'num_shards': 3, 'max_num_jobs': 10}, [range(0			,      1    ), range(1			,      2    ), range(2			,      3    )]),
    ]			,      )
def 				lowerCamelCase__	(						_A			,      _A    ):
  a						: List[Any]					       =					_distribute_shards(**_A    )
  assert out == expected
@pytest.mark.parametrize(
    'gen_kwargs, max_num_jobs, expected'			,      [
        ({'foo': 0}, 10, [{'foo': 0}]),
        ({'shards': [0, 1, 2, 3]}, 1, [{'shards': [0, 1, 2, 3]}]),
        ({'shards': [0, 1, 2, 3]}, 4, [{'shards': [0]}, {'shards': [1]}, {'shards': [2]}, {'shards': [3]}]),
        ({'shards': [0, 1]}, 4, [{'shards': [0]}, {'shards': [1]}]),
        ({'shards': [0, 1, 2, 3]}, 2, [{'shards': [0, 1]}, {'shards': [2, 3]}]),
    ]			,      )
def 				lowerCamelCase__	(						_A			,      _A			,      _A    ):
  a						: str					       =					_split_gen_kwargs(_A			,      _A    )
  assert out == expected

@pytest.mark.parametrize(
    'gen_kwargs, expected'			,      [
        ({'foo': 0}, 1),
        ({'shards': [0]}, 1),
        ({'shards': [0, 1, 2, 3]}, 4),
        ({'shards': [0, 1, 2, 3], 'foo': 0}, 4),
        ({'shards': [0, 1, 2, 3], 'other': (0, 1)}, 4),
        ({'shards': [0, 1, 2, 3], 'shards2': [0, 1]}, RuntimeError),
    ]			,      )
def 				lowerCamelCase__	(						_A			,      _A    ):
  if expected is RuntimeError:
    with pytest.raises(_A    ):
      _number_of_shards_in_gen_kwargs(_A    )
  else:
    a						: str					       =					_number_of_shards_in_gen_kwargs(_A    )
    assert out == expected




'''simple docstring'''






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tokenizers_available,
    is_torch_available,
)


lowerCAmelCase: List[Any]							=						{}

try:
     if not is_sentencepiece_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     lowerCAmelCase: Optional[int]							=						['NllbTokenizer']

try:
     if not is_tokenizers_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     lowerCAmelCase: Dict							=						['NllbTokenizerFast']


if TYPE_CHECKING:
     try:
          if not is_sentencepiece_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .tokenization_nllb import NllbTokenizer

     try:
          if not is_tokenizers_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .tokenization_nllb_fast import NllbTokenizerFast

else:
     import sys

     lowerCAmelCase: Tuple							=						_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)