


"""simple docstring"""







import enum
import os
from hashlib import shaaaa
from typing import Optional

from .. import config
from .logging import get_logger


lowerCAmelCase__				    =       get_logger(__name__)


class 				SCREAMING_SNAKE_CASE__    (		enum.Enum    ):
       """simple docstring"""


       a  :    Dict       ="all_checks"
       a  :    Optional[int]       ="basic_checks"
       a  :    Optional[Any]       ="no_checks"


class 				SCREAMING_SNAKE_CASE__    (		UpperCamelCase__    ):
       """simple docstring"""


       pass


class 				SCREAMING_SNAKE_CASE__    (		UpperCamelCase__    ):
       """simple docstring"""


       pass


class 				SCREAMING_SNAKE_CASE__    (		UpperCamelCase__    ):
       """simple docstring"""


       pass


class 				SCREAMING_SNAKE_CASE__    (		UpperCamelCase__    ):
       """simple docstring"""


       pass

def  a__      (    SCREAMING_SNAKE_CASE   :   Optional[dict]    ,				SCREAMING_SNAKE_CASE   :   dict    ,				SCREAMING_SNAKE_CASE   :   List[str]=None    ):


  '''simple docstring'''







  if expected_checksums is None:
    logger.info("Unable to verify checksums."    )
    return
  if len(set(UpperCamelCase__    ) - set(UpperCamelCase__    )    ) > 0:
    raise ExpectedMoreDownloadedFiles(str(set(UpperCamelCase__    ) - set(UpperCamelCase__    )    )    )
  if len(set(UpperCamelCase__    ) - set(UpperCamelCase__    )    ) > 0:
    raise UnexpectedDownloadedFile(str(set(UpperCamelCase__    ) - set(UpperCamelCase__    )    )    )
  lowerCAmelCase					:					List[Any]								=							[url for url in expected_checksums if expected_checksums[url] != recorded_checksums[url]]
  lowerCAmelCase					:					Union[str, Any]								=							" for " + verification_name if verification_name is not None else ""
  if len(UpperCamelCase__    ) > 0:
    raise NonMatchingChecksumError(
        f"""Checksums didn't match{for_verification_name}:\n"""
        f"""{bad_urls}\n"""
        "Set `verification_mode=\'no_checks\'` to skip checksums verification and ignore this error"    )
  logger.info("All the checksums matched successfully" + for_verification_name    )


class 				SCREAMING_SNAKE_CASE__    (		UpperCamelCase__    ):
       """simple docstring"""


       pass


class 				SCREAMING_SNAKE_CASE__    (		UpperCamelCase__    ):
       """simple docstring"""


       pass


class 				SCREAMING_SNAKE_CASE__    (		UpperCamelCase__    ):
       """simple docstring"""


       pass


class 				SCREAMING_SNAKE_CASE__    (		UpperCamelCase__    ):
       """simple docstring"""


       pass

def  a__      (    SCREAMING_SNAKE_CASE   :   Optional[dict]    ,				SCREAMING_SNAKE_CASE   :   dict    ):


  '''simple docstring'''







  if expected_splits is None:
    logger.info("Unable to verify splits sizes."    )
    return
  if len(set(UpperCamelCase__    ) - set(UpperCamelCase__    )    ) > 0:
    raise ExpectedMoreSplits(str(set(UpperCamelCase__    ) - set(UpperCamelCase__    )    )    )
  if len(set(UpperCamelCase__    ) - set(UpperCamelCase__    )    ) > 0:
    raise UnexpectedSplits(str(set(UpperCamelCase__    ) - set(UpperCamelCase__    )    )    )
  lowerCAmelCase					:					Dict								=							[
      {"expected": expected_splits[name], "recorded": recorded_splits[name]}
      for name in expected_splits
      if expected_splits[name].num_examples != recorded_splits[name].num_examples
  ]
  if len(UpperCamelCase__    ) > 0:
    raise NonMatchingSplitsSizesError(str(UpperCamelCase__    )    )
  logger.info("All the splits matched successfully."    )
def  a__      (    SCREAMING_SNAKE_CASE   :   str    ,				SCREAMING_SNAKE_CASE   :   bool = True    ):


  '''simple docstring'''







  if record_checksum:
    lowerCAmelCase					:					Any								=							shaaaa()
    with open(UpperCamelCase__    ,				"rb"    ) as f:
      for chunk in iter(lambda: f.read(1 << 2_0    )    ,				B""    ):
        m.update(UpperCamelCase__    )
      lowerCAmelCase					:					Optional[int]								=							m.hexdigest()
  else:
    lowerCAmelCase					:					Optional[Any]								=							None
  return {"num_bytes": os.path.getsize(UpperCamelCase__    ), "checksum": checksum}





def  a__      (    SCREAMING_SNAKE_CASE   :   Union[str, Any]    ):


  '''simple docstring'''







  if dataset_size and config.IN_MEMORY_MAX_SIZE:
    return dataset_size < config.IN_MEMORY_MAX_SIZE
  else:
    return False









"""simple docstring"""







import pytest

from datasets.parallel import ParallelBackendConfig, parallel_backend
from datasets.utils.py_utils import map_nested

from .utils import require_dill_gt_0_3_2, require_joblibspark, require_not_windows
def  a__      (    SCREAMING_SNAKE_CASE   :   str    ):  # picklable for multiprocessing


  '''simple docstring'''







  return i + 1
@require_dill_gt_0_3_2
@require_joblibspark
@require_not_windows
def  a__      (    ):


  '''simple docstring'''







  with parallel_backend("spark"    ):
    assert ParallelBackendConfig.backend_name == "spark"

  lowerCAmelCase					:					List[str]								=							[1, 2, 3]
  with pytest.raises(SCREAMING_SNAKE_CASE    ):
    with parallel_backend("unsupported backend"    ):
      map_nested(SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE    ,				num_proc=2    )

  with pytest.raises(SCREAMING_SNAKE_CASE    ):
    with parallel_backend("unsupported backend"    ):
      map_nested(SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE    ,				num_proc=-1    )





@require_dill_gt_0_3_2
@require_joblibspark
@require_not_windows
@pytest.mark.parametrize("num_proc"    ,				[2, -1]    )
def  a__      (    SCREAMING_SNAKE_CASE   :   Tuple    ):


  '''simple docstring'''







  lowerCAmelCase					:					Tuple								=							[1, 2]
  lowerCAmelCase					:					int								=							{"a": 1, "b": 2}
  lowerCAmelCase					:					List[str]								=							{"a": [1, 2], "b": [3, 4]}
  lowerCAmelCase					:					Dict								=							{"a": {"1": 1}, "b": 2}
  lowerCAmelCase					:					Tuple								=							{"a": 1, "b": 2, "c": 3, "d": 4}
  lowerCAmelCase					:					Any								=							[2, 3]
  lowerCAmelCase					:					Any								=							{"a": 2, "b": 3}
  lowerCAmelCase					:					Optional[int]								=							{"a": [2, 3], "b": [4, 5]}
  lowerCAmelCase					:					Optional[int]								=							{"a": {"1": 2}, "b": 3}
  lowerCAmelCase					:					str								=							{"a": 2, "b": 3, "c": 4, "d": 5}

  with parallel_backend("spark"    ):
    assert map_nested(SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE    ,				num_proc=SCREAMING_SNAKE_CASE    ) == expected_map_nested_sa
    assert map_nested(SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE    ,				num_proc=SCREAMING_SNAKE_CASE    ) == expected_map_nested_sa
    assert map_nested(SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE    ,				num_proc=SCREAMING_SNAKE_CASE    ) == expected_map_nested_sa
    assert map_nested(SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE    ,				num_proc=SCREAMING_SNAKE_CASE    ) == expected_map_nested_sa
    assert map_nested(SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE    ,				num_proc=SCREAMING_SNAKE_CASE    ) == expected_map_nested_sa





