




"""simple docstring"""


import io
import json

import fsspec
import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.json import JsonDatasetReader, JsonDatasetWriter

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases




def  UpperCAmelCase__	(snake_case__							: List[str]		,       snake_case__							: int   ):
  """simple docstring"""

  assert isinstance(__lowerCamelCase		,       __lowerCamelCase   )
  assert dataset.num_rows == 4
  assert dataset.num_columns == 3
  assert dataset.column_names == ["col_1", "col_2", "col_3"]
  for feature, expected_dtype in expected_features.items():
    assert dataset.features[feature].dtype == expected_dtype




@pytest.mark.parametrize("""keep_in_memory"""		,       [False, True]   )
def  UpperCAmelCase__	(snake_case__							: Dict		,       snake_case__							: Union[str, Any]		,       snake_case__							: int   ):
  """simple docstring"""

  _snake_case							:		Union[str, Any]								=  tmp_path / "cache"
  _snake_case							:		str								=  {"col_1": "string", "col_2": "int64", "col_3": "float64"}
  with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
    _snake_case							:		str								=  JsonDatasetReader(__lowerCamelCase		,       cache_dir=__lowerCamelCase		,       keep_in_memory=__lowerCamelCase   ).read()
  _check_json_dataset(__lowerCamelCase		,       __lowerCamelCase   )




@pytest.mark.parametrize(
    """features"""		,       [
        None,
        {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""},
        {"""col_1""": """string""", """col_2""": """string""", """col_3""": """string"""},
        {"""col_1""": """int32""", """col_2""": """int32""", """col_3""": """int32"""},
        {"""col_1""": """float32""", """col_2""": """float32""", """col_3""": """float32"""},
    ]		,       )
def  UpperCAmelCase__	(snake_case__							: Union[str, Any]		,       snake_case__							: Union[str, Any]		,       snake_case__							: Tuple   ):
  """simple docstring"""

  _snake_case							:		Optional[Any]								=  tmp_path / "cache"
  _snake_case							:		Optional[int]								=  {"col_1": "string", "col_2": "int64", "col_3": "float64"}
  _snake_case							:		int								=  features.copy() if features else default_expected_features
  _snake_case							:		Optional[Any]								=  (
      Features({feature: Value(__lowerCamelCase   ) for feature, dtype in features.items()}   ) if features is not None else None
  )
  _snake_case							:		Any								=  JsonDatasetReader(__lowerCamelCase		,       features=__lowerCamelCase		,       cache_dir=__lowerCamelCase   ).read()
  _check_json_dataset(__lowerCamelCase		,       __lowerCamelCase   )




@pytest.mark.parametrize(
    """features"""		,       [
        None,
        {"""col_3""": """float64""", """col_1""": """string""", """col_2""": """int64"""},
    ]		,       )
def  UpperCAmelCase__	(snake_case__							: Optional[Any]		,       snake_case__							: Dict		,       snake_case__							: Any   ):
  """simple docstring"""

  _snake_case							:		List[str]								=  tmp_path / "cache"
  _snake_case							:		Dict								=  {"col_3": "float64", "col_1": "string", "col_2": "int64"}
  _snake_case							:		Any								=  features.copy() if features else default_expected_features
  _snake_case							:		Dict								=  (
      Features({feature: Value(__lowerCamelCase   ) for feature, dtype in features.items()}   ) if features is not None else None
  )
  _snake_case							:		Dict								=  JsonDatasetReader(__lowerCamelCase		,       features=__lowerCamelCase		,       cache_dir=__lowerCamelCase   ).read()
  assert isinstance(__lowerCamelCase		,       __lowerCamelCase   )
  assert dataset.num_rows == 2
  assert dataset.num_columns == 3
  assert dataset.column_names == ["col_3", "col_1", "col_2"]
  for feature, expected_dtype in expected_features.items():
    assert dataset.features[feature].dtype == expected_dtype




def  UpperCAmelCase__	(snake_case__							: List[Any]		,       snake_case__							: Union[str, Any]   ):
  """simple docstring"""

  _snake_case							:		str								=  {"col_2": "int64", "col_3": "float64", "col_1": "string"}
  _snake_case							:		str								=  features.copy()
  _snake_case							:		Union[str, Any]								=  (
      Features({feature: Value(__lowerCamelCase   ) for feature, dtype in features.items()}   ) if features is not None else None
  )
  _snake_case							:		Any								=  tmp_path / "cache"
  _snake_case							:		str								=  JsonDatasetReader(__lowerCamelCase		,       features=__lowerCamelCase		,       cache_dir=__lowerCamelCase   ).read()
  assert isinstance(__lowerCamelCase		,       __lowerCamelCase   )
  assert dataset.num_rows == 2
  assert dataset.num_columns == 3
  assert dataset.column_names == ["col_2", "col_3", "col_1"]
  for feature, expected_dtype in expected_features.items():
    assert dataset.features[feature].dtype == expected_dtype




@pytest.mark.parametrize("""split"""		,       [None, NamedSplit("""train"""   ), """train""", """test"""]   )
def  UpperCAmelCase__	(snake_case__							: Optional[int]		,       snake_case__							: List[Any]		,       snake_case__							: Union[str, Any]   ):
  """simple docstring"""

  _snake_case							:		Dict								=  tmp_path / "cache"
  _snake_case							:		str								=  {"col_1": "string", "col_2": "int64", "col_3": "float64"}
  _snake_case							:		Dict								=  JsonDatasetReader(__lowerCamelCase		,       cache_dir=__lowerCamelCase		,       split=__lowerCamelCase   ).read()
  _check_json_dataset(__lowerCamelCase		,       __lowerCamelCase   )
  assert dataset.split == split if split else "train"




@pytest.mark.parametrize("""path_type"""		,       [str, list]   )
def  UpperCAmelCase__	(snake_case__							: Dict		,       snake_case__							: List[str]		,       snake_case__							: Tuple   ):
  """simple docstring"""

  if issubclass(__lowerCamelCase		,       __lowerCamelCase   ):
    _snake_case							:		Dict								=  jsonl_path
  elif issubclass(__lowerCamelCase		,       __lowerCamelCase   ):
    _snake_case							:		List[Any]								=  [jsonl_path]
  _snake_case							:		str								=  tmp_path / "cache"
  _snake_case							:		str								=  {"col_1": "string", "col_2": "int64", "col_3": "float64"}
  _snake_case							:		str								=  JsonDatasetReader(__lowerCamelCase		,       cache_dir=__lowerCamelCase   ).read()
  _check_json_dataset(__lowerCamelCase		,       __lowerCamelCase   )




def  UpperCAmelCase__	(snake_case__							: List[str]		,       snake_case__							: Union[str, Any]		,       snake_case__							: int=("train",)   ):
  """simple docstring"""

  assert isinstance(__lowerCamelCase		,       __lowerCamelCase   )
  for split in splits:
    _snake_case							:		List[str]								=  dataset_dict[split]
    assert dataset.num_rows == 4
    assert dataset.num_columns == 3
    assert dataset.column_names == ["col_1", "col_2", "col_3"]
    for feature, expected_dtype in expected_features.items():
      assert dataset.features[feature].dtype == expected_dtype




@pytest.mark.parametrize("""keep_in_memory"""		,       [False, True]   )
def  UpperCAmelCase__	(snake_case__							: Optional[int]		,       snake_case__							: Union[str, Any]		,       snake_case__							: List[Any]   ):
  """simple docstring"""

  _snake_case							:		Tuple								=  tmp_path / "cache"
  _snake_case							:		Optional[Any]								=  {"col_1": "string", "col_2": "int64", "col_3": "float64"}
  with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
    _snake_case							:		List[str]								=  JsonDatasetReader({"""train""": jsonl_path}		,       cache_dir=__lowerCamelCase		,       keep_in_memory=__lowerCamelCase   ).read()
  _check_json_datasetdict(__lowerCamelCase		,       __lowerCamelCase   )




@pytest.mark.parametrize(
    """features"""		,       [
        None,
        {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""},
        {"""col_1""": """string""", """col_2""": """string""", """col_3""": """string"""},
        {"""col_1""": """int32""", """col_2""": """int32""", """col_3""": """int32"""},
        {"""col_1""": """float32""", """col_2""": """float32""", """col_3""": """float32"""},
    ]		,       )
def  UpperCAmelCase__	(snake_case__							: Any		,       snake_case__							: Optional[int]		,       snake_case__							: List[str]   ):
  """simple docstring"""

  _snake_case							:		int								=  tmp_path / "cache"
  _snake_case							:		str								=  {"col_1": "string", "col_2": "int64", "col_3": "float64"}
  _snake_case							:		Any								=  features.copy() if features else default_expected_features
  _snake_case							:		Union[str, Any]								=  (
      Features({feature: Value(__lowerCamelCase   ) for feature, dtype in features.items()}   ) if features is not None else None
  )
  _snake_case							:		Optional[Any]								=  JsonDatasetReader({"""train""": jsonl_path}		,       features=__lowerCamelCase		,       cache_dir=__lowerCamelCase   ).read()
  _check_json_datasetdict(__lowerCamelCase		,       __lowerCamelCase   )




@pytest.mark.parametrize("""split"""		,       [None, NamedSplit("""train"""   ), """train""", """test"""]   )
def  UpperCAmelCase__	(snake_case__							: int		,       snake_case__							: List[Any]		,       snake_case__							: Dict   ):
  """simple docstring"""

  if split:
    _snake_case							:		Dict								=  {split: jsonl_path}
  else:
    _snake_case							:		Any								=  "train"
    _snake_case							:		Union[str, Any]								=  {"train": jsonl_path, "test": jsonl_path}
  _snake_case							:		Any								=  tmp_path / "cache"
  _snake_case							:		str								=  {"col_1": "string", "col_2": "int64", "col_3": "float64"}
  _snake_case							:		Dict								=  JsonDatasetReader(__lowerCamelCase		,       cache_dir=__lowerCamelCase   ).read()
  _check_json_datasetdict(__lowerCamelCase		,       __lowerCamelCase		,       splits=list(path.keys()   )   )
  assert all(dataset[split].split == split for split in path.keys()   )




def  UpperCAmelCase__	(snake_case__							: Any   ):
  """simple docstring"""

  return json.load(__lowerCamelCase   )




def  UpperCAmelCase__	(snake_case__							: Dict   ):
  """simple docstring"""

  return [json.loads(__lowerCamelCase   ) for line in buffer]






class  lowercase:







 '''simple docstring'''





 @pytest.mark.parametrize("""lines, load_json_function""",				[(True, load_json_lines), (False, load_json)]		)
 def 			UpperCamelCase_       (    self:		int,				a_:		str,				a_:		Optional[int],				a_:		int		):




   '''simple docstring'''



   with io.BytesIO() as buffer:
     JsonDatasetWriter(lowercase_,				lowercase_,				lines=lowercase_		).write()
     buffer.seek(0		)
     _snake_case							:		Dict								=  load_json_function(lowercase_		)
   assert isinstance(lowercase_,				lowercase_		)
   assert isinstance(exported_content[0],				lowercase_		)
   assert len(lowercase_		) == 10


 @pytest.mark.parametrize(
     """orient, container, keys, len_at""",				[
         ("""records""", list, {"""tokens""", """labels""", """answers""", """id"""}, None),
         ("""split""", dict, {"""columns""", """data"""}, """data"""),
         ("""index""", dict, set("""0123456789"""		), None),
         ("""columns""", dict, {"""tokens""", """labels""", """answers""", """id"""}, """tokens"""),
         ("""values""", list, None, None),
         ("""table""", dict, {"""schema""", """data"""}, """data"""),
     ],				)
 def 			UpperCamelCase_       (    self:		List[str],				a_:		Optional[Any],				a_:		Optional[int],				a_:		str,				a_:		Dict,				a_:		Any		):




   '''simple docstring'''



   with io.BytesIO() as buffer:
     JsonDatasetWriter(lowercase_,				lowercase_,				lines=lowercase_,				orient=lowercase_		).write()
     buffer.seek(0		)
     _snake_case							:		Optional[Any]								=  load_json(lowercase_		)
   assert isinstance(lowercase_,				lowercase_		)
   if keys:
     if container is dict:
       assert exported_content.keys() == keys
     else:
       assert exported_content[0].keys() == keys
   else:
     assert not hasattr(lowercase_,				"""keys"""		) and not hasattr(exported_content[0],				"""keys"""		)
   if len_at:
     assert len(exported_content[len_at]		) == 10
   else:
     assert len(lowercase_		) == 10


 @pytest.mark.parametrize("""lines, load_json_function""",				[(True, load_json_lines), (False, load_json)]		)
 def 			UpperCamelCase_       (    self:		List[str],				a_:		List[str],				a_:		List[str],				a_:		Any		):




   '''simple docstring'''



   with io.BytesIO() as buffer:
     JsonDatasetWriter(lowercase_,				lowercase_,				lines=lowercase_,				num_proc=2		).write()
     buffer.seek(0		)
     _snake_case							:		Optional[Any]								=  load_json_function(lowercase_		)
   assert isinstance(lowercase_,				lowercase_		)
   assert isinstance(exported_content[0],				lowercase_		)
   assert len(lowercase_		) == 10


 @pytest.mark.parametrize(
     """orient, container, keys, len_at""",				[
         ("""records""", list, {"""tokens""", """labels""", """answers""", """id"""}, None),
         ("""split""", dict, {"""columns""", """data"""}, """data"""),
         ("""index""", dict, set("""0123456789"""		), None),
         ("""columns""", dict, {"""tokens""", """labels""", """answers""", """id"""}, """tokens"""),
         ("""values""", list, None, None),
         ("""table""", dict, {"""schema""", """data"""}, """data"""),
     ],				)
 def 			UpperCamelCase_       (    self:		Optional[Any],				a_:		str,				a_:		Union[str, Any],				a_:		List[str],				a_:		List[str],				a_:		Tuple		):




   '''simple docstring'''



   with io.BytesIO() as buffer:
     JsonDatasetWriter(lowercase_,				lowercase_,				lines=lowercase_,				orient=lowercase_,				num_proc=2		).write()
     buffer.seek(0		)
     _snake_case							:		List[Any]								=  load_json(lowercase_		)
   assert isinstance(lowercase_,				lowercase_		)
   if keys:
     if container is dict:
       assert exported_content.keys() == keys
     else:
       assert exported_content[0].keys() == keys
   else:
     assert not hasattr(lowercase_,				"""keys"""		) and not hasattr(exported_content[0],				"""keys"""		)
   if len_at:
     assert len(exported_content[len_at]		) == 10
   else:
     assert len(lowercase_		) == 10


 def 			UpperCamelCase_       (    self:		Any,				a_:		int		):




   '''simple docstring'''



   with pytest.raises(lowercase_		):
     with io.BytesIO() as buffer:
       JsonDatasetWriter(lowercase_,				lowercase_,				num_proc=0		)


 @pytest.mark.parametrize("""compression, extension""",				[("""gzip""", """gz"""), ("""bz2""", """bz2"""), ("""xz""", """xz""")]		)
 def 			UpperCamelCase_       (    self:		Optional[Any],				a_:		Any,				a_:		Any,				a_:		List[str],				a_:		Any,				a_:		Any		):




   '''simple docstring'''



   _snake_case							:		List[Any]								=  tmp_path_factory.mktemp("""data"""		) / f"test.json.{extension}"
   _snake_case							:		str								=  str(shared_datadir / f"test_file.json.{extension}"		)
   JsonDatasetWriter(lowercase_,				lowercase_,				compression=lowercase_		).write()

   with fsspec.open(lowercase_,				"""rb""",				compression="""infer"""		) as f:
     _snake_case							:		List[str]								=  f.read()
   with fsspec.open(lowercase_,				"""rb""",				compression="""infer"""		) as f:
     _snake_case							:		Dict								=  f.read()
   assert exported_content == original_content








"""simple docstring"""


from collections.abc import Generator
from math import sin


def  UpperCAmelCase__	(snake_case__							: bytes   ):
  """simple docstring"""

  if len(snake_case__   ) != 32:
    raise ValueError("""Input must be of length 32"""   )

  _snake_case							:		Optional[int]								=  B""""""
  for i in [3, 2, 1, 0]:
    little_endian += string_aa[8 * i : 8 * i + 8]
  return little_endian


def  UpperCAmelCase__	(snake_case__							: int   ):
  """simple docstring"""

  if i < 0:
    raise ValueError("""Input must be non-negative"""   )

  _snake_case							:		Optional[Any]								=  format(snake_case__		,       """08x"""   )[-8:]
  _snake_case							:		Optional[Any]								=  B""""""
  for i in [3, 2, 1, 0]:
    little_endian_hex += hex_rep[2 * i : 2 * i + 2].encode("""utf-8"""   )
  return little_endian_hex


def  UpperCAmelCase__	(snake_case__							: bytes   ):
  """simple docstring"""

  _snake_case							:		Union[str, Any]								=  B""""""
  for char in message:
    bit_string += format(snake_case__		,       """08b"""   ).encode("""utf-8"""   )
  _snake_case							:		List[Any]								=  format(len(snake_case__   )		,       """064b"""   ).encode("""utf-8"""   )

  # Pad bit_string to a multiple of 512 chars
  bit_string += b"1"
  while len(snake_case__   ) % 5_12 != 4_48:
    bit_string += b"0"
  bit_string += to_little_endian(start_len[32:]   ) + to_little_endian(start_len[:32]   )

  return bit_string


def  UpperCAmelCase__	(snake_case__							: bytes   ):
  """simple docstring"""

  if len(snake_case__   ) % 5_12 != 0:
    raise ValueError("""Input must have length that's a multiple of 512"""   )

  for pos in range(0		,       len(snake_case__   )		,       5_12   ):
    _snake_case							:		List[str]								=  bit_string[pos : pos + 5_12]
    _snake_case							:		List[str]								=  []
    for i in range(0		,       5_12		,       32   ):
      block_words.append(int(to_little_endian(block[i : i + 32]   )		,       2   )   )
    yield block_words


def  UpperCAmelCase__	(snake_case__							: int   ):
  """simple docstring"""

  if i < 0:
    raise ValueError("""Input must be non-negative"""   )

  _snake_case							:		Optional[int]								=  format(snake_case__		,       """032b"""   )
  _snake_case							:		str								=  """"""
  for c in i_str:
    new_str += "1" if c == "0" else "0"
  return int(snake_case__		,       2   )


def  UpperCAmelCase__	(snake_case__							: int		,       snake_case__							: int   ):
  """simple docstring"""

  return (a + b) % 2**32


def  UpperCAmelCase__	(snake_case__							: int		,       snake_case__							: int   ):
  """simple docstring"""

  if i < 0:
    raise ValueError("""Input must be non-negative"""   )
  if shift < 0:
    raise ValueError("""Shift must be non-negative"""   )
  return ((i << shift) ^ (i >> (32 - shift))) % 2**32


def  UpperCAmelCase__	(snake_case__							: bytes   ):
  """simple docstring"""

  _snake_case							:		Any								=  preprocess(snake_case__   )

  _snake_case							:		Optional[Any]								=  [int(2**32 * abs(sin(i + 1   )   )   ) for i in range(64   )]

  # Starting states
  _snake_case							:		Union[str, Any]								=  0x6745_2301
  _snake_case							:		List[Any]								=  0xEFCD_AB89
  _snake_case							:		Optional[Any]								=  0x98BA_DCFE
  _snake_case							:		Optional[int]								=  0x1032_5476

  _snake_case							:		Tuple								=  [
      7,
      12,
      17,
      22,
      7,
      12,
      17,
      22,
      7,
      12,
      17,
      22,
      7,
      12,
      17,
      22,
      5,
      9,
      14,
      20,
      5,
      9,
      14,
      20,
      5,
      9,
      14,
      20,
      5,
      9,
      14,
      20,
      4,
      11,
      16,
      23,
      4,
      11,
      16,
      23,
      4,
      11,
      16,
      23,
      4,
      11,
      16,
      23,
      6,
      10,
      15,
      21,
      6,
      10,
      15,
      21,
      6,
      10,
      15,
      21,
      6,
      10,
      15,
      21,
  ]

  # Process bit string in chunks, each with 16 32-char words
  for block_words in get_block_words(snake_case__   ):
    _snake_case							:		Tuple								=  aa
    _snake_case							:		str								=  ba
    _snake_case							:		int								=  ca
    _snake_case							:		Dict								=  da

    # Hash current chunk
    for i in range(64   ):
      if i <= 15:
        # f = (b & c) | (not_32(b) & d)     # Alternate definition for f
        _snake_case							:		int								=  d ^ (b & (c ^ d))
        _snake_case							:		Dict								=  i
      elif i <= 31:
        # f = (d & b) | (not_32(d) & c)     # Alternate definition for f
        _snake_case							:		Tuple								=  c ^ (d & (b ^ c))
        _snake_case							:		int								=  (5 * i + 1) % 16
      elif i <= 47:
        _snake_case							:		List[Any]								=  b ^ c ^ d
        _snake_case							:		Union[str, Any]								=  (3 * i + 5) % 16
      else:
        _snake_case							:		Tuple								=  c ^ (b | not_aa(snake_case__   ))
        _snake_case							:		Optional[int]								=  (7 * i) % 16
      _snake_case							:		Dict								=  (f + a + added_consts[i] + block_words[g]) % 2**32
      _snake_case							:		List[str]								=  d
      _snake_case							:		List[Any]								=  c
      _snake_case							:		str								=  b
      _snake_case							:		List[str]								=  sum_aa(snake_case__		,       left_rotate_aa(snake_case__		,       shift_amounts[i]   )   )

    # Add hashed chunk to running total
    _snake_case							:		Union[str, Any]								=  sum_aa(snake_case__		,       snake_case__   )
    _snake_case							:		str								=  sum_aa(snake_case__		,       snake_case__   )
    _snake_case							:		Any								=  sum_aa(snake_case__		,       snake_case__   )
    _snake_case							:		List[str]								=  sum_aa(snake_case__		,       snake_case__   )

  _snake_case							:		Any								=  reformat_hex(snake_case__   ) + reformat_hex(snake_case__   ) + reformat_hex(snake_case__   ) + reformat_hex(snake_case__   )
  return digest


if __name__ == "__main__":
     import doctest

     doctest.testmod()


