


"""simple docstring"""







# Lint as: python3
import sys
from collections.abc import Mapping
from typing import TYPE_CHECKING

import numpy as np
import pyarrow as pa

from .. import config
from ..utils.py_utils import map_nested
from .formatting import TensorFormatter


if TYPE_CHECKING:
     import torch


class 				SCREAMING_SNAKE_CASE__    (		TensorFormatter[Mapping, "torch.Tensor", Mapping]    ):
       """simple docstring"""


       def __init__(						self		,					snake_case__=None		,					**snake_case__						):






         """simple docstring"""





         super().__init__(features=snake_case__						)
         lowerCAmelCase					:					Optional[Any]								=							torch_tensor_kwargs
         import torch  # noqa import torch at initialization

       def 	lowercase__							(						self		,					snake_case__						):






         """simple docstring"""





         import torch

         if isinstance(snake_case__		,					snake_case__						) and column:
           if all(
               isinstance(snake_case__		,					torch.Tensor						) and x.shape == column[0].shape and x.dtype == column[0].dtype
               for x in column						):
             return torch.stack(snake_case__						)
         return column

       def 	lowercase__							(						self		,					snake_case__						):






         """simple docstring"""





         import torch

         if isinstance(snake_case__		,					(str, bytes, type(snake_case__						))						):
           return value
         elif isinstance(snake_case__		,					(np.character, np.ndarray)						) and np.issubdtype(value.dtype		,					np.character						):
           return value.tolist()

         lowerCAmelCase					:					Dict								=							{}

         if isinstance(snake_case__		,					(np.number, np.ndarray)						) and np.issubdtype(value.dtype		,					np.integer						):
           lowerCAmelCase					:					List[Any]								=							{"dtype": torch.intaa}
         elif isinstance(snake_case__		,					(np.number, np.ndarray)						) and np.issubdtype(value.dtype		,					np.floating						):
           lowerCAmelCase					:					Union[str, Any]								=							{"dtype": torch.floataa}
         elif config.PIL_AVAILABLE and "PIL" in sys.modules:
           import PIL.Image

           if isinstance(snake_case__		,					PIL.Image.Image						):
             lowerCAmelCase					:					Tuple								=							np.asarray(snake_case__						)
         return torch.tensor(snake_case__		,					**{**default_dtype, **self.torch_tensor_kwargs}						)

       def 	lowercase__							(						self		,					snake_case__						):






         """simple docstring"""





         import torch

         # support for torch, tf, jax etc.
         if hasattr(snake_case__		,					"__array__"						) and not isinstance(snake_case__		,					torch.Tensor						):
           lowerCAmelCase					:					Dict								=							data_struct.__array__()
         # support for nested types like struct of list of struct
         if isinstance(snake_case__		,					np.ndarray						):
           if data_struct.dtype == object:  # torch tensors cannot be instantied from an array of objects
             return self._consolidate([self.recursive_tensorize(snake_case__						) for substruct in data_struct]						)
         elif isinstance(snake_case__		,					(list, tuple)						):
           return self._consolidate([self.recursive_tensorize(snake_case__						) for substruct in data_struct]						)
         return self._tensorize(snake_case__						)

       def 	lowercase__							(						self		,					snake_case__						):






         """simple docstring"""





         return map_nested(self._recursive_tensorize		,					snake_case__		,					map_list=snake_case__						)

       def 	lowercase__							(						self		,					snake_case__						):






         """simple docstring"""





         lowerCAmelCase					:					List[Any]								=							self.numpy_arrow_extractor().extract_row(snake_case__						)
         lowerCAmelCase					:					str								=							self.python_features_decoder.decode_row(snake_case__						)
         return self.recursive_tensorize(snake_case__						)

       def 	lowercase__							(						self		,					snake_case__						):






         """simple docstring"""





         lowerCAmelCase					:					Any								=							self.numpy_arrow_extractor().extract_column(snake_case__						)
         lowerCAmelCase					:					Optional[Any]								=							self.python_features_decoder.decode_column(snake_case__		,					pa_table.column_names[0]						)
         lowerCAmelCase					:					int								=							self.recursive_tensorize(snake_case__						)
         lowerCAmelCase					:					str								=							self._consolidate(snake_case__						)
         return column







       def 	lowercase__							(						self		,					snake_case__						):






         """simple docstring"""





         lowerCAmelCase					:					Optional[Any]								=							self.numpy_arrow_extractor().extract_batch(snake_case__						)
         lowerCAmelCase					:					Union[str, Any]								=							self.python_features_decoder.decode_batch(snake_case__						)
         lowerCAmelCase					:					int								=							self.recursive_tensorize(snake_case__						)
         for column_name in batch:
           lowerCAmelCase					:					List[str]								=							self._consolidate(batch[column_name]						)
         return batch









"""simple docstring"""







import pytest

from datasets.parallel import ParallelBackendConfig, parallel_backend
from datasets.utils.py_utils import map_nested

from .utils import require_dill_gt_0_3_2, require_joblibspark, require_not_windows
def  a__      (    SCREAMING_SNAKE_CASE   :   str    ):  # picklable for multiprocessing


  '''simple docstring'''







  return i + 1
@require_dill_gt_0_3_2
@require_joblibspark
@require_not_windows
def  a__      (    ):


  '''simple docstring'''







  with parallel_backend("spark"    ):
    assert ParallelBackendConfig.backend_name == "spark"

  lowerCAmelCase					:					List[str]								=							[1, 2, 3]
  with pytest.raises(SCREAMING_SNAKE_CASE    ):
    with parallel_backend("unsupported backend"    ):
      map_nested(SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE    ,				num_proc=2    )

  with pytest.raises(SCREAMING_SNAKE_CASE    ):
    with parallel_backend("unsupported backend"    ):
      map_nested(SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE    ,				num_proc=-1    )





@require_dill_gt_0_3_2
@require_joblibspark
@require_not_windows
@pytest.mark.parametrize("num_proc"    ,				[2, -1]    )
def  a__      (    SCREAMING_SNAKE_CASE   :   Tuple    ):


  '''simple docstring'''







  lowerCAmelCase					:					Tuple								=							[1, 2]
  lowerCAmelCase					:					int								=							{"a": 1, "b": 2}
  lowerCAmelCase					:					List[str]								=							{"a": [1, 2], "b": [3, 4]}
  lowerCAmelCase					:					Dict								=							{"a": {"1": 1}, "b": 2}
  lowerCAmelCase					:					Tuple								=							{"a": 1, "b": 2, "c": 3, "d": 4}
  lowerCAmelCase					:					Any								=							[2, 3]
  lowerCAmelCase					:					Any								=							{"a": 2, "b": 3}
  lowerCAmelCase					:					Optional[int]								=							{"a": [2, 3], "b": [4, 5]}
  lowerCAmelCase					:					Optional[int]								=							{"a": {"1": 2}, "b": 3}
  lowerCAmelCase					:					str								=							{"a": 2, "b": 3, "c": 4, "d": 5}

  with parallel_backend("spark"    ):
    assert map_nested(SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE    ,				num_proc=SCREAMING_SNAKE_CASE    ) == expected_map_nested_sa
    assert map_nested(SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE    ,				num_proc=SCREAMING_SNAKE_CASE    ) == expected_map_nested_sa
    assert map_nested(SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE    ,				num_proc=SCREAMING_SNAKE_CASE    ) == expected_map_nested_sa
    assert map_nested(SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE    ,				num_proc=SCREAMING_SNAKE_CASE    ) == expected_map_nested_sa
    assert map_nested(SCREAMING_SNAKE_CASE    ,				SCREAMING_SNAKE_CASE    ,				num_proc=SCREAMING_SNAKE_CASE    ) == expected_map_nested_sa





