


import copy
import os
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pyarrow as pa
import pyarrow.parquet as pq
import pytest

from datasets.arrow_writer import ArrowWriter, OptimizedTypedSequence, ParquetWriter, TypedSequence
from datasets.features import ArrayaD, ClassLabel, Features, Image, Value
from datasets.features.features import ArrayaDExtensionType, cast_to_python_objects
from datasets.keyhash import DuplicatedKeysError, InvalidKeyError

from .utils import require_pil

class     a		(A_				):




							"""simple docstring"""


							def    __snake_case (							self	:  int    )     ->   Tuple:
														__snake_case		:      Optional[Any]										=       pa.array(TypedSequence([1, 2, 3]    )    )
														self.assertEqual(arr.type   ,     pa.intaa()    )

							def    __snake_case (							self	:  str    )     ->   List[str]:


														with self.assertRaises(snake_case__    ):
																					__snake_case		:      Tuple										=       pa.array(TypedSequence([1, 2, 3]    )   ,     type=pa.intaa()    )

							def    __snake_case (							self	:  Optional[int]    )     ->   int:


														with self.assertRaises(snake_case__    ):
																					__snake_case		:      Optional[int]										=       pa.array(TypedSequence([1, 2, 3]   ,     try_type=Value("bool"    )   ,     type=Value("int64"    )    )    )

							def    __snake_case (							self	:  Any    )     ->   Dict:
														__snake_case		:      Union[str, Any]										=       pa.array(TypedSequence([1, 2, 3]   ,     type=Value("int32"    )    )    )
														self.assertEqual(arr.type   ,     pa.intaa()    )

							def    __snake_case (							self	:  str    )     ->   List[str]:


														with self.assertRaises((TypeError, pa.lib.ArrowInvalid)    ):
																					__snake_case		:      Tuple										=       pa.array(TypedSequence(["foo", "bar"]   ,     type=Value("int64"    )    )    )

							def    __snake_case (							self	:  List[str]    )     ->   Any:
														__snake_case		:      Optional[Any]										=       pa.array(TypedSequence([1, 2, 3]   ,     try_type=Value("int32"    )    )    )
														self.assertEqual(arr.type   ,     pa.intaa()    )

							def    __snake_case (							self	:  Optional[Any]    )     ->   List[Any]:
														__snake_case		:      Tuple										=       pa.array(TypedSequence(["foo", "bar"]   ,     try_type=Value("int64"    )    )    )
														self.assertEqual(arr.type   ,     pa.string()    )

							def    __snake_case (							self	:  Any    )     ->   Union[str, Any]:
														__snake_case		:      Optional[int]										=       pa.array(TypedSequence([[[1, 2, 3]]]   ,     type=ArrayaD((1, 3)   ,     "int64"    )    )    )
														self.assertEqual(arr.type   ,     ArrayaDExtensionType((1, 3)   ,     "int64"    )    )

							def    __snake_case (							self	:  Any    )     ->   Any:


														with self.assertRaises((TypeError, pa.lib.ArrowInvalid)    ):
																					__snake_case		:      Dict										=       pa.array(TypedSequence(["foo", "bar"]   ,     type=ArrayaD((1, 3)   ,     "int64"    )    )    )

							def    __snake_case (							self	:  str    )     ->   Optional[Any]:
														__snake_case		:      Optional[Any]										=       pa.array(TypedSequence([[[1, 2, 3]]]   ,     try_type=ArrayaD((1, 3)   ,     "int64"    )    )    )
														self.assertEqual(arr.type   ,     ArrayaDExtensionType((1, 3)   ,     "int64"    )    )

							def    __snake_case (							self	:  List[str]    )     ->   List[str]:
														__snake_case		:      Optional[int]										=       pa.array(TypedSequence(["foo", "bar"]   ,     try_type=ArrayaD((1, 3)   ,     "int64"    )    )    )
														self.assertEqual(arr.type   ,     pa.string()    )






							@require_pil
							def    __snake_case (							self	:  Optional[int]    )     ->   List[Any]:
														import PIL.Image

														__snake_case		:      List[str]										=       PIL.Image.fromarray(np.arange(10   ,     dtype=np.uinta    ).reshape(2   ,     5    )    )
														with patch(
														    "datasets.arrow_writer.cast_to_python_objects"   ,     side_effect=snake_case__    ) as mock_cast_to_python_objects:
																					__snake_case		:      str										=       pa.array(TypedSequence([{"path": None, "bytes": B"image_bytes"}, pil_image]   ,     type=Image()    )    )
																					__snake_case		:      Optional[Any]										=       mock_cast_to_python_objects.call_args_list[-1]
																					self.assertIn("optimize_list_casting"   ,     snake_case__    )
																					self.assertFalse(kwargs["optimize_list_casting"]    )







def     lowerCAmelCase_					( __lowerCamelCase							,     __lowerCamelCase     ):
							__snake_case		:      Union[str, Any]										=       pa.BufferReader(__lowerCamelCase     ) if isinstance(__lowerCamelCase							,     pa.Buffer     ) else pa.memory_map(__lowerCamelCase     )
							__snake_case		:      str										=       pa.ipc.open_stream(__lowerCamelCase     )
							__snake_case		:      pa.Table										=       f.read_all()
							assert len(pa_table.to_batches()     ) == expected_num_chunks
							assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}
							del pa_table







@pytest.mark.parametrize("writer_batch_size"							,     [None, 1, 1_0]     )
@pytest.mark.parametrize(
    "fields"							,     [None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}]     )
def     lowerCAmelCase_					( __lowerCamelCase							,     __lowerCamelCase     ):
							__snake_case		:      Union[str, Any]										=       pa.BufferOutputStream()
							__snake_case		:      List[Any]										=       pa.schema(__lowerCamelCase     ) if fields else None
							with ArrowWriter(stream=__lowerCamelCase							,     schema=__lowerCamelCase							,     writer_batch_size=__lowerCamelCase     ) as writer:
														writer.write({"col_1": "foo", "col_2": 1}     )
														writer.write({"col_1": "bar", "col_2": 2}     )
														__snake_case		:      Dict										=       writer.finalize()
							assert num_examples == 2
							assert num_bytes > 0
							if not fields:
														__snake_case		:      Any										=       {"col_1": pa.string(), "col_2": pa.intaa()}
							assert writer._schema == pa.schema(__lowerCamelCase							,     metadata=writer._schema.metadata     )
							_check_output(output.getvalue()							,     expected_num_chunks=num_examples if writer_batch_size == 1 else 1     )







def     lowerCAmelCase_					( ):
							__snake_case		:      str										=       pa.BufferOutputStream()
							__snake_case		:      List[Any]										=       Features({"labels": ClassLabel(names=["neg", "pos"]     )}     )
							with ArrowWriter(stream=__lowerCamelCase							,     features=__lowerCamelCase     ) as writer:
														writer.write({"labels": 0}     )
														writer.write({"labels": 1}     )
														__snake_case		:      Union[str, Any]										=       writer.finalize()
							assert num_examples == 2
							assert num_bytes > 0
							assert writer._schema == features.arrow_schema
							assert writer._schema.metadata == features.arrow_schema.metadata
							__snake_case		:      List[Any]										=       pa.BufferReader(output.getvalue()     )
							__snake_case		:      Optional[int]										=       pa.ipc.open_stream(__lowerCamelCase     )
							__snake_case		:      pa.Table										=       f.read_all()
							__snake_case		:      Union[str, Any]										=       pa_table.schema
							assert pa_table.num_rows == 2
							assert schema == features.arrow_schema
							assert schema.metadata == features.arrow_schema.metadata
							assert features == Features.from_arrow_schema(__lowerCamelCase     )







@pytest.mark.parametrize("writer_batch_size"							,     [None, 1, 1_0]     )
def     lowerCAmelCase_					( __lowerCamelCase     ):
							__snake_case		:      str										=       pa.BufferOutputStream()
							with ArrowWriter(
							    stream=__lowerCamelCase							,     writer_batch_size=__lowerCamelCase							,     hash_salt="split_name"							,     check_duplicates=__lowerCamelCase							,     ) as writer:
														with pytest.raises(__lowerCamelCase     ):
																					writer.write({"col_1": "foo", "col_2": 1}							,     key=[1, 2]     )
																					__snake_case		:      Optional[int]										=       writer.finalize()







@pytest.mark.parametrize("writer_batch_size"							,     [None, 2, 1_0]     )
def     lowerCAmelCase_					( __lowerCamelCase     ):
							__snake_case		:      Optional[Any]										=       pa.BufferOutputStream()
							with ArrowWriter(
							    stream=__lowerCamelCase							,     writer_batch_size=__lowerCamelCase							,     hash_salt="split_name"							,     check_duplicates=__lowerCamelCase							,     ) as writer:
														with pytest.raises(__lowerCamelCase     ):
																					writer.write({"col_1": "foo", "col_2": 1}							,     key=1_0     )
																					writer.write({"col_1": "bar", "col_2": 2}							,     key=1_0     )
																					__snake_case		:      Dict										=       writer.finalize()







@pytest.mark.parametrize("writer_batch_size"							,     [None, 2, 1_0]     )
def     lowerCAmelCase_					( __lowerCamelCase     ):
							__snake_case		:      Dict										=       pa.BufferOutputStream()
							with ArrowWriter(
							    stream=__lowerCamelCase							,     writer_batch_size=__lowerCamelCase							,     hash_salt="split_name"							,     check_duplicates=__lowerCamelCase							,     ) as writer:
														writer.write({"col_1": "foo", "col_2": 1}							,     key=1     )
														writer.write({"col_1": "bar", "col_2": 2}							,     key=2     )
														__snake_case		:      List[Any]										=       writer.finalize()
							assert num_examples == 2
							assert num_bytes > 0
							_check_output(output.getvalue()							,     expected_num_chunks=num_examples if writer_batch_size == 1 else 1     )







@pytest.mark.parametrize("writer_batch_size"							,     [None, 1, 1_0]     )
@pytest.mark.parametrize(
    "fields"							,     [None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}]     )
def     lowerCAmelCase_					( __lowerCamelCase							,     __lowerCamelCase     ):
							__snake_case		:      int										=       pa.BufferOutputStream()
							__snake_case		:      List[Any]										=       pa.schema(__lowerCamelCase     ) if fields else None
							with ArrowWriter(stream=__lowerCamelCase							,     schema=__lowerCamelCase							,     writer_batch_size=__lowerCamelCase     ) as writer:
														writer.write_batch({"col_1": ["foo", "bar"], "col_2": [1, 2]}     )
														writer.write_batch({"col_1": [], "col_2": []}     )
														__snake_case		:      Tuple										=       writer.finalize()
							assert num_examples == 2
							assert num_bytes > 0
							if not fields:
														__snake_case		:      Dict										=       {"col_1": pa.string(), "col_2": pa.intaa()}
							assert writer._schema == pa.schema(__lowerCamelCase							,     metadata=writer._schema.metadata     )
							_check_output(output.getvalue()							,     expected_num_chunks=num_examples if writer_batch_size == 1 else 1     )







@pytest.mark.parametrize("writer_batch_size"							,     [None, 1, 1_0]     )
@pytest.mark.parametrize(
    "fields"							,     [None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}]     )
def     lowerCAmelCase_					( __lowerCamelCase							,     __lowerCamelCase     ):
							__snake_case		:      Dict										=       pa.BufferOutputStream()
							__snake_case		:      List[str]										=       pa.schema(__lowerCamelCase     ) if fields else None
							with ArrowWriter(stream=__lowerCamelCase							,     schema=__lowerCamelCase							,     writer_batch_size=__lowerCamelCase     ) as writer:
														writer.write_table(pa.Table.from_pydict({"col_1": ["foo", "bar"], "col_2": [1, 2]}     )     )
														__snake_case		:      Dict										=       writer.finalize()
							assert num_examples == 2
							assert num_bytes > 0
							if not fields:
														__snake_case		:      Optional[int]										=       {"col_1": pa.string(), "col_2": pa.intaa()}
							assert writer._schema == pa.schema(__lowerCamelCase							,     metadata=writer._schema.metadata     )
							_check_output(output.getvalue()							,     expected_num_chunks=num_examples if writer_batch_size == 1 else 1     )







@pytest.mark.parametrize("writer_batch_size"							,     [None, 1, 1_0]     )
@pytest.mark.parametrize(
    "fields"							,     [None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}]     )
def     lowerCAmelCase_					( __lowerCamelCase							,     __lowerCamelCase     ):
							__snake_case		:      Optional[int]										=       pa.BufferOutputStream()
							__snake_case		:      List[str]										=       pa.schema(__lowerCamelCase     ) if fields else None
							with ArrowWriter(stream=__lowerCamelCase							,     schema=__lowerCamelCase							,     writer_batch_size=__lowerCamelCase     ) as writer:
														writer.write_row(pa.Table.from_pydict({"col_1": ["foo"], "col_2": [1]}     )     )
														writer.write_row(pa.Table.from_pydict({"col_1": ["bar"], "col_2": [2]}     )     )
														__snake_case		:      int										=       writer.finalize()
							assert num_examples == 2
							assert num_bytes > 0
							if not fields:
														__snake_case		:      List[Any]										=       {"col_1": pa.string(), "col_2": pa.intaa()}
							assert writer._schema == pa.schema(__lowerCamelCase							,     metadata=writer._schema.metadata     )
							_check_output(output.getvalue()							,     expected_num_chunks=num_examples if writer_batch_size == 1 else 1     )







def     lowerCAmelCase_					( ):
							with tempfile.TemporaryDirectory() as tmp_dir:
														__snake_case		:      Union[str, Any]										=       {"col_1": pa.string(), "col_2": pa.intaa()}
														__snake_case		:      Union[str, Any]										=       os.path.join(__lowerCamelCase							,     "test.arrow"     )
														with ArrowWriter(path=__lowerCamelCase							,     schema=pa.schema(__lowerCamelCase     )     ) as writer:
																					writer.write_batch({"col_1": ["foo", "bar"], "col_2": [1, 2]}     )
																					__snake_case		:      str										=       writer.finalize()
														assert num_examples == 2
														assert num_bytes > 0
														assert writer._schema == pa.schema(__lowerCamelCase							,     metadata=writer._schema.metadata     )
														_check_output(__lowerCamelCase							,     1     )







def     lowerCAmelCase_					( __lowerCamelCase     ):
							if pa.types.is_list(__lowerCamelCase     ):
														return get_base_dtype(arr_type.value_type     )
							else:
														return arr_type







def     lowerCAmelCase_					( __lowerCamelCase							,     __lowerCamelCase     ):
							if isinstance(lst[0]							,     __lowerCamelCase     ):
														change_first_primitive_element_in_list(lst[0]							,     __lowerCamelCase     )
							else:
														__snake_case		:      List[str]										=       value







@pytest.mark.parametrize("optimized_int_type, expected_dtype"							,     [(None, pa.intaa()), (Value("int32"     ), pa.intaa())]     )
@pytest.mark.parametrize("sequence"							,     [[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]]     )
def     lowerCAmelCase_					( __lowerCamelCase							,     __lowerCamelCase							,     __lowerCamelCase     ):
							__snake_case		:      Any										=       pa.array(TypedSequence(__lowerCamelCase							,     optimized_int_type=__lowerCamelCase     )     )
							assert get_base_dtype(arr.type     ) == expected_dtype







@pytest.mark.parametrize(
    "col, expected_dtype"							,     [
        ("attention_mask", pa.inta()),
        ("special_tokens_mask", pa.inta()),
        ("token_type_ids", pa.inta()),
        ("input_ids", pa.intaa()),
        ("other", pa.intaa()),
    ]							,     )
@pytest.mark.parametrize("sequence"							,     [[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]]     )
def     lowerCAmelCase_					( __lowerCamelCase							,     __lowerCamelCase							,     __lowerCamelCase     ):
							# in range
							__snake_case		:      Tuple										=       pa.array(OptimizedTypedSequence(__lowerCamelCase							,     col=__lowerCamelCase     )     )
							assert get_base_dtype(arr.type     ) == expected_dtype

							# not in range
							if col != "other":
														# avoids errors due to in-place modifications
														__snake_case		:      Union[str, Any]										=       copy.deepcopy(__lowerCamelCase     )
														__snake_case		:      Union[str, Any]										=       np.iinfo(expected_dtype.to_pandas_dtype()     ).max + 1
														change_first_primitive_element_in_list(__lowerCamelCase							,     __lowerCamelCase     )
														__snake_case		:      Any										=       pa.array(OptimizedTypedSequence(__lowerCamelCase							,     col=__lowerCamelCase     )     )
														assert get_base_dtype(arr.type     ) == pa.intaa()







@pytest.mark.parametrize("raise_exception"							,     [False, True]     )
def     lowerCAmelCase_					( __lowerCamelCase							,     __lowerCamelCase     ):
							__snake_case		:      List[Any]										=       str(tmp_path / "dataset-train.arrow"     )
							try:
														with ArrowWriter(path=__lowerCamelCase     ) as writer:
																					if raise_exception:
																												raise pa.lib.ArrowInvalid()
																					else:
																												writer.stream.close()
							except pa.lib.ArrowInvalid:
														pass
							finally:
														assert writer.stream.closed







def     lowerCAmelCase_					( __lowerCamelCase     ):
							__snake_case		:      int										=       "mock://dataset-train.arrow"
							with ArrowWriter(path=__lowerCamelCase							,     storage_options=mockfs.storage_options     ) as writer:
														assert isinstance(writer._fs							,     type(__lowerCamelCase     )     )
														assert writer._fs.storage_options == mockfs.storage_options
														writer.write({"col_1": "foo", "col_2": 1}     )
														writer.write({"col_1": "bar", "col_2": 2}     )
														__snake_case		:      Optional[int]										=       writer.finalize()
							assert num_examples == 2
							assert num_bytes > 0
							assert mockfs.exists(__lowerCamelCase     )







def     lowerCAmelCase_					( ):
							__snake_case		:      Optional[int]										=       pa.BufferOutputStream()
							with ParquetWriter(stream=__lowerCamelCase     ) as writer:
														writer.write({"col_1": "foo", "col_2": 1}     )
														writer.write({"col_1": "bar", "col_2": 2}     )
														__snake_case		:      Optional[Any]										=       writer.finalize()
							assert num_examples == 2
							assert num_bytes > 0
							__snake_case		:      Optional[Any]										=       pa.BufferReader(output.getvalue()     )
							__snake_case		:      pa.Table										=       pq.read_table(__lowerCamelCase     )
							assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}







@require_pil
@pytest.mark.parametrize("embed_local_files"							,     [False, True]     )
def     lowerCAmelCase_					( __lowerCamelCase							,     __lowerCamelCase     ):
							import PIL.Image

							__snake_case		:      Dict										=       str(tmp_path / "test_image_rgb.jpg"     )
							PIL.Image.fromarray(np.zeros((5, 5)							,     dtype=np.uinta     )     ).save(__lowerCamelCase							,     format="png"     )
							__snake_case		:      Union[str, Any]										=       pa.BufferOutputStream()
							with ParquetWriter(
							    stream=__lowerCamelCase							,     features=Features({"image": Image()}     )							,     embed_local_files=__lowerCamelCase     ) as writer:
														writer.write({"image": image_path}     )
														writer.finalize()
							__snake_case		:      str										=       pa.BufferReader(output.getvalue()     )
							__snake_case		:      pa.Table										=       pq.read_table(__lowerCamelCase     )
							__snake_case		:      Tuple										=       pa_table.to_pydict()
							if embed_local_files:
														assert isinstance(out["image"][0]["path"]							,     __lowerCamelCase     )
														with open(__lowerCamelCase							,     "rb"     ) as f:
																					assert out["image"][0]["bytes"] == f.read()
							else:
														assert out["image"][0]["path"] == image_path
														assert out["image"][0]["bytes"] is None







def     lowerCAmelCase_					( ):
							__snake_case		:      List[Any]										=       pa.schema([pa.field("col_1"							,     pa.string()							,     nullable=__lowerCamelCase     )]     )
							__snake_case		:      List[str]										=       pa.BufferOutputStream()
							with ArrowWriter(stream=__lowerCamelCase     ) as writer:
														writer._build_writer(inferred_schema=__lowerCamelCase     )
														assert writer._schema == pa.schema([pa.field("col_1"							,     pa.string()     )]     )





import logging

from transformers.configuration_utils import PretrainedConfig


_snake_case					:		Optional[Any]      =    logging.getLogger(__name__)

class     a		(_lowerCAmelCase				):




							"""simple docstring"""


							__UpperCAmelCase   :							Tuple					  =  "masked_bert"

							def __init__(							self	:  Optional[int]   ,     lowerCamelCase	:  Any=30522   ,     lowerCamelCase	:  Tuple=768   ,     lowerCamelCase	:  str=12   ,     lowerCamelCase	:  Dict=12   ,     lowerCamelCase	:  List[str]=3072   ,     lowerCamelCase	:  List[str]="gelu"   ,     lowerCamelCase	:  Any=0.1   ,     lowerCamelCase	:  Any=0.1   ,     lowerCamelCase	:  List[Any]=512   ,     lowerCamelCase	:  int=2   ,     lowerCamelCase	:  str=0.02   ,     lowerCamelCase	:  List[str]=1E-12   ,     lowerCamelCase	:  Any=0   ,     lowerCamelCase	:  Dict="topK"   ,     lowerCamelCase	:  List[Any]="constant"   ,     lowerCamelCase	:  Dict=0.0   ,     **lowerCamelCase	:  List[Any]   ,     )     ->   Dict:
														super().__init__(pad_token_id=lowerCamelCase   ,     **lowerCamelCase    )

														__snake_case		:      Optional[Any]										=       vocab_size
														__snake_case		:      Optional[int]										=       hidden_size
														__snake_case		:      Tuple										=       num_hidden_layers
														__snake_case		:      Union[str, Any]										=       num_attention_heads
														__snake_case		:      Optional[Any]										=       hidden_act
														__snake_case		:      str										=       intermediate_size
														__snake_case		:      Tuple										=       hidden_dropout_prob
														__snake_case		:      Dict										=       attention_probs_dropout_prob
														__snake_case		:      List[str]										=       max_position_embeddings
														__snake_case		:      Union[str, Any]										=       type_vocab_size
														__snake_case		:      Any										=       initializer_range
														__snake_case		:      str										=       layer_norm_eps
														__snake_case		:      str										=       pruning_method
														__snake_case		:      int										=       mask_init
														__snake_case		:      Any										=       mask_scale

