


import pyarrow.parquet as pq
import pytest

from datasets import Audio, Dataset, DatasetDict, Features, NamedSplit, Sequence, Value, config
from datasets.features.image import Image
from datasets.io.parquet import ParquetDatasetReader, ParquetDatasetWriter, get_writer_batch_size

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases






def 			UpperCamelCase (     __magic_name__					:							Any   ,   __magic_name__					:							Any					)       ->  Union[str, Any]:
		"""simple docstring"""




		assert isinstance(__magic_name__   ,   __magic_name__					)
		assert dataset.num_rows == 4
		assert dataset.num_columns == 3
		assert dataset.column_names == ["col_1", "col_2", "col_3"]
		for feature, expected_dtype in expected_features.items():
				assert dataset.features[feature].dtype == expected_dtype






@pytest.mark.parametrize("""keep_in_memory"""   ,   [False, True]					)
def 			UpperCamelCase (     __magic_name__					:							int   ,   __magic_name__					:							Optional[int]   ,   __magic_name__					:							int					)       ->  List[Any]:
		"""simple docstring"""




		lowercase__					      =		tmp_path / """cache"""
		lowercase__					      =		{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
		with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
				lowercase__					      =		ParquetDatasetReader(__magic_name__   ,   cache_dir=__magic_name__   ,   keep_in_memory=__magic_name__					).read()
		_check_parquet_dataset(__magic_name__   ,   __magic_name__					)






@pytest.mark.parametrize(
    """features"""   ,   [
        None,
        {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""},
        {"""col_1""": """string""", """col_2""": """string""", """col_3""": """string"""},
        {"""col_1""": """int32""", """col_2""": """int32""", """col_3""": """int32"""},
        {"""col_1""": """float32""", """col_2""": """float32""", """col_3""": """float32"""},
    ]   ,   )
def 			UpperCamelCase (     __magic_name__					:							Optional[Any]   ,   __magic_name__					:							Optional[int]   ,   __magic_name__					:							Any					)       ->  Optional[Any]:
		"""simple docstring"""




		lowercase__					      =		tmp_path / """cache"""
		lowercase__					      =		{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
		lowercase__					      =		features.copy() if features else default_expected_features
		lowercase__					      =		(
		    Features({feature: Value(__magic_name__					) for feature, dtype in features.items()}					) if features is not None else None
		)
		lowercase__					      =		ParquetDatasetReader(__magic_name__   ,   features=__magic_name__   ,   cache_dir=__magic_name__					).read()
		_check_parquet_dataset(__magic_name__   ,   __magic_name__					)






@pytest.mark.parametrize("""split"""   ,   [None, NamedSplit("""train"""					), """train""", """test"""]					)
def 			UpperCamelCase (     __magic_name__					:							Tuple   ,   __magic_name__					:							List[Any]   ,   __magic_name__					:							Dict					)       ->  str:
		"""simple docstring"""




		lowercase__					      =		tmp_path / """cache"""
		lowercase__					      =		{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
		lowercase__					      =		ParquetDatasetReader(__magic_name__   ,   cache_dir=__magic_name__   ,   split=__magic_name__					).read()
		_check_parquet_dataset(__magic_name__   ,   __magic_name__					)
		assert dataset.split == split if split else "train"






@pytest.mark.parametrize("""path_type"""   ,   [str, list]					)
def 			UpperCamelCase (     __magic_name__					:							int   ,   __magic_name__					:							Optional[int]   ,   __magic_name__					:							List[Any]					)       ->  int:
		"""simple docstring"""




		if issubclass(__magic_name__   ,   __magic_name__					):
				lowercase__					      =		parquet_path
		elif issubclass(__magic_name__   ,   __magic_name__					):
				lowercase__					      =		[parquet_path]
		lowercase__					      =		tmp_path / """cache"""
		lowercase__					      =		{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
		lowercase__					      =		ParquetDatasetReader(__magic_name__   ,   cache_dir=__magic_name__					).read()
		_check_parquet_dataset(__magic_name__   ,   __magic_name__					)






def 			UpperCamelCase (     __magic_name__					:							List[Any]   ,   __magic_name__					:							Any   ,   __magic_name__					:							int=("train",)					)       ->  int:
		"""simple docstring"""




		assert isinstance(__magic_name__   ,   __magic_name__					)
		for split in splits:
				lowercase__					      =		dataset_dict[split]
				assert dataset.num_rows == 4
				assert dataset.num_columns == 3
				assert dataset.column_names == ["col_1", "col_2", "col_3"]
				for feature, expected_dtype in expected_features.items():
						assert dataset.features[feature].dtype == expected_dtype






@pytest.mark.parametrize("""keep_in_memory"""   ,   [False, True]					)
def 			UpperCamelCase (     __magic_name__					:							List[str]   ,   __magic_name__					:							List[Any]   ,   __magic_name__					:							Any					)       ->  Union[str, Any]:
		"""simple docstring"""




		lowercase__					      =		tmp_path / """cache"""
		lowercase__					      =		{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
		with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
				lowercase__					      =		ParquetDatasetReader(
				    {"""train""": parquet_path}   ,   cache_dir=__magic_name__   ,   keep_in_memory=__magic_name__					).read()
		_check_parquet_datasetdict(__magic_name__   ,   __magic_name__					)






@pytest.mark.parametrize(
    """features"""   ,   [
        None,
        {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""},
        {"""col_1""": """string""", """col_2""": """string""", """col_3""": """string"""},
        {"""col_1""": """int32""", """col_2""": """int32""", """col_3""": """int32"""},
        {"""col_1""": """float32""", """col_2""": """float32""", """col_3""": """float32"""},
    ]   ,   )
def 			UpperCamelCase (     __magic_name__					:							Optional[int]   ,   __magic_name__					:							Dict   ,   __magic_name__					:							List[Any]					)       ->  Tuple:
		"""simple docstring"""




		lowercase__					      =		tmp_path / """cache"""
		lowercase__					      =		{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
		lowercase__					      =		features.copy() if features else default_expected_features
		lowercase__					      =		(
		    Features({feature: Value(__magic_name__					) for feature, dtype in features.items()}					) if features is not None else None
		)
		lowercase__					      =		ParquetDatasetReader({"""train""": parquet_path}   ,   features=__magic_name__   ,   cache_dir=__magic_name__					).read()
		_check_parquet_datasetdict(__magic_name__   ,   __magic_name__					)






@pytest.mark.parametrize("""split"""   ,   [None, NamedSplit("""train"""					), """train""", """test"""]					)
def 			UpperCamelCase (     __magic_name__					:							int   ,   __magic_name__					:							str   ,   __magic_name__					:							Optional[int]					)       ->  List[Any]:
		"""simple docstring"""




		if split:
				lowercase__					      =		{split: parquet_path}
		else:
				lowercase__					      =		"""train"""
				lowercase__					      =		{"""train""": parquet_path, """test""": parquet_path}
		lowercase__					      =		tmp_path / """cache"""
		lowercase__					      =		{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
		lowercase__					      =		ParquetDatasetReader(__magic_name__   ,   cache_dir=__magic_name__					).read()
		_check_parquet_datasetdict(__magic_name__   ,   __magic_name__   ,   splits=list(path.keys()					)					)
		assert all(dataset[split].split == split for split in path.keys()					)






def 			UpperCamelCase (     __magic_name__					:							Optional[Any]   ,   __magic_name__					:							List[Any]					)       ->  int:
		"""simple docstring"""




		lowercase__					      =		ParquetDatasetWriter(__magic_name__   ,   tmp_path / """foo.parquet"""					)
		assert writer.write() > 0
		lowercase__					      =		pq.ParquetFile(tmp_path / """foo.parquet"""					)
		lowercase__					      =		pf.read()
		assert dataset.data.table == output_table






def 			UpperCamelCase (     __magic_name__					:							Optional[int]   ,   __magic_name__					:							Optional[int]					)       ->  Tuple:
		"""simple docstring"""




		lowercase__					      =		str(shared_datadir / """test_image_rgb.jpg"""					)
		lowercase__					      =		{"""image""": [image_path]}
		lowercase__					      =		Features({"""image""": Image()}					)
		lowercase__					      =		Dataset.from_dict(__magic_name__   ,   features=__magic_name__					)
		lowercase__					      =		ParquetDatasetWriter(__magic_name__   ,   tmp_path / """foo.parquet"""					)
		assert writer.write() > 0

		lowercase__					      =		Dataset.from_parquet(str(tmp_path / """foo.parquet"""					)					)
		assert dataset.features == reloaded_dataset.features

		lowercase__					      =		ParquetDatasetReader(str(tmp_path / """foo.parquet"""					)   ,   streaming=__magic_name__					).read()
		assert dataset.features == reloaded_iterable_dataset.features







@pytest.mark.parametrize(
    """feature, expected"""   ,   [
        (Features({"""foo""": Value("""int32"""					)}					), None),
        (Features({"""image""": Image(), """foo""": Value("""int32"""					)}					), config.PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS),
        (Features({"""nested""": Sequence(Audio()					)}					), config.PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS),
    ]   ,   )
def 			UpperCamelCase (     __magic_name__					:							Optional[int]   ,   __magic_name__					:							List[str]					)       ->  Optional[int]:
		"""simple docstring"""




		assert get_writer_batch_size(__magic_name__					) == expected





from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow


if is_tf_available():
	import numpy as np
	import tensorflow as tf

	from transformers import TFXLMRobertaModel



@require_tf
@require_sentencepiece
@require_tokenizers
class 							A   (	unittest.TestCase							):
					'''simple docstring'''






					@slow
					def   lowerCamelCase__					(self   :						Any		)						->       Union[str, Any]:




							"""simple docstring"""


							lowercase__					      =		TFXLMRobertaModel.from_pretrained("""jplu/tf-xlm-roberta-base"""		)

							lowercase__					      =		{
							    """input_ids""": tf.convert_to_tensor([[0, 2646, 1_0269, 83, 9_9942, 2]]				, dtype=tf.intaa		),  # "My dog is cute"
							    """attention_mask""": tf.convert_to_tensor([[1, 1, 1, 1, 1, 1]]				, dtype=tf.intaa		),
							}

							lowercase__					      =		model(_UpperCAmelCase		)["""last_hidden_state"""]
							lowercase__					      =		tf.TensorShape((1, 6, 768)		)
							self.assertEqual(output.shape				, _UpperCAmelCase		)
							# compare the actual values for a slice.
							lowercase__					      =		tf.convert_to_tensor(
							    [
							        [
							            [0.0_681_762, 0.10_894_451, 0.06_772_504],
							            [-0.06_423_668, 0.02_366_615, 0.04_329_344],
							            [-0.06_057_295, 0.09_974_135, -0.00_070_584],
							        ]
							    ]				, dtype=tf.floataa				, )

							self.assertTrue(np.allclose(output[:, :3, :3].numpy()				, expected_slice.numpy()				, atol=1E-4		)		)

