





"""simple docstring"""




import io
import json

import fsspec
import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.json import JsonDatasetReader, JsonDatasetWriter

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases





def       _UpperCamelCase       (a__				:Optional[int]				,     a__				:List[str]    ):






							"""simple docstring"""





							assert isinstance(a__				,     a__    )
							assert dataset.num_rows == 4
							assert dataset.num_columns == 3
							assert dataset.column_names == ["col_1", "col_2", "col_3"]
							for feature, expected_dtype in expected_features.items():
														assert dataset.features[feature].dtype == expected_dtype





@pytest.mark.parametrize("""keep_in_memory"""				,     [False, True]    )
def       _UpperCamelCase       (a__				:Any				,     a__				:str				,     a__				:Union[str, Any]    ):






							"""simple docstring"""





							UpperCamelCase__			       =     tmp_path / """cache"""
							UpperCamelCase__			       =     {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
							with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
														UpperCamelCase__			       =     JsonDatasetReader(a__				,     cache_dir=a__				,     keep_in_memory=a__    ).read()
							_check_json_dataset(a__				,     a__    )





@pytest.mark.parametrize(
    """features"""				,     [
        None,
        {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""},
        {"""col_1""": """string""", """col_2""": """string""", """col_3""": """string"""},
        {"""col_1""": """int32""", """col_2""": """int32""", """col_3""": """int32"""},
        {"""col_1""": """float32""", """col_2""": """float32""", """col_3""": """float32"""},
    ]				,     )
def       _UpperCamelCase       (a__				:str				,     a__				:List[str]				,     a__				:Union[str, Any]    ):






							"""simple docstring"""





							UpperCamelCase__			       =     tmp_path / """cache"""
							UpperCamelCase__			       =     {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
							UpperCamelCase__			       =     features.copy() if features else default_expected_features
							UpperCamelCase__			       =     (
							    Features({feature: Value(a__    ) for feature, dtype in features.items()}    ) if features is not None else None
							)
							UpperCamelCase__			       =     JsonDatasetReader(a__				,     features=a__				,     cache_dir=a__    ).read()
							_check_json_dataset(a__				,     a__    )





@pytest.mark.parametrize(
    """features"""				,     [
        None,
        {"""col_3""": """float64""", """col_1""": """string""", """col_2""": """int64"""},
    ]				,     )
def       _UpperCamelCase       (a__				:List[Any]				,     a__				:List[Any]				,     a__				:Tuple    ):






							"""simple docstring"""





							UpperCamelCase__			       =     tmp_path / """cache"""
							UpperCamelCase__			       =     {"""col_3""": """float64""", """col_1""": """string""", """col_2""": """int64"""}
							UpperCamelCase__			       =     features.copy() if features else default_expected_features
							UpperCamelCase__			       =     (
							    Features({feature: Value(a__    ) for feature, dtype in features.items()}    ) if features is not None else None
							)
							UpperCamelCase__			       =     JsonDatasetReader(a__				,     features=a__				,     cache_dir=a__    ).read()
							assert isinstance(a__				,     a__    )
							assert dataset.num_rows == 2
							assert dataset.num_columns == 3
							assert dataset.column_names == ["col_3", "col_1", "col_2"]
							for feature, expected_dtype in expected_features.items():
														assert dataset.features[feature].dtype == expected_dtype





def       _UpperCamelCase       (a__				:Optional[int]				,     a__				:Optional[Any]    ):






							"""simple docstring"""





							UpperCamelCase__			       =     {"""col_2""": """int64""", """col_3""": """float64""", """col_1""": """string"""}
							UpperCamelCase__			       =     features.copy()
							UpperCamelCase__			       =     (
							    Features({feature: Value(a__    ) for feature, dtype in features.items()}    ) if features is not None else None
							)
							UpperCamelCase__			       =     tmp_path / """cache"""
							UpperCamelCase__			       =     JsonDatasetReader(a__				,     features=a__				,     cache_dir=a__    ).read()
							assert isinstance(a__				,     a__    )
							assert dataset.num_rows == 2
							assert dataset.num_columns == 3
							assert dataset.column_names == ["col_2", "col_3", "col_1"]
							for feature, expected_dtype in expected_features.items():
														assert dataset.features[feature].dtype == expected_dtype





@pytest.mark.parametrize("""split"""				,     [None, NamedSplit("""train"""    ), """train""", """test"""]    )
def       _UpperCamelCase       (a__				:List[Any]				,     a__				:Optional[int]				,     a__				:Union[str, Any]    ):






							"""simple docstring"""





							UpperCamelCase__			       =     tmp_path / """cache"""
							UpperCamelCase__			       =     {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
							UpperCamelCase__			       =     JsonDatasetReader(a__				,     cache_dir=a__				,     split=a__    ).read()
							_check_json_dataset(a__				,     a__    )
							assert dataset.split == split if split else "train"





@pytest.mark.parametrize("""path_type"""				,     [str, list]    )
def       _UpperCamelCase       (a__				:List[str]				,     a__				:List[Any]				,     a__				:str    ):






							"""simple docstring"""





							if issubclass(a__				,     a__    ):
														UpperCamelCase__			       =     jsonl_path
							elif issubclass(a__				,     a__    ):
														UpperCamelCase__			       =     [jsonl_path]
							UpperCamelCase__			       =     tmp_path / """cache"""
							UpperCamelCase__			       =     {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
							UpperCamelCase__			       =     JsonDatasetReader(a__				,     cache_dir=a__    ).read()
							_check_json_dataset(a__				,     a__    )





def       _UpperCamelCase       (a__				:Any				,     a__				:Optional[Any]				,     a__				:str=("train",)    ):






							"""simple docstring"""





							assert isinstance(a__				,     a__    )
							for split in splits:
														UpperCamelCase__			       =     dataset_dict[split]
														assert dataset.num_rows == 4
														assert dataset.num_columns == 3
														assert dataset.column_names == ["col_1", "col_2", "col_3"]
														for feature, expected_dtype in expected_features.items():
																					assert dataset.features[feature].dtype == expected_dtype





@pytest.mark.parametrize("""keep_in_memory"""				,     [False, True]    )
def       _UpperCamelCase       (a__				:Union[str, Any]				,     a__				:Tuple				,     a__				:Union[str, Any]    ):






							"""simple docstring"""





							UpperCamelCase__			       =     tmp_path / """cache"""
							UpperCamelCase__			       =     {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
							with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
														UpperCamelCase__			       =     JsonDatasetReader({"""train""": jsonl_path}				,     cache_dir=a__				,     keep_in_memory=a__    ).read()
							_check_json_datasetdict(a__				,     a__    )





@pytest.mark.parametrize(
    """features"""				,     [
        None,
        {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""},
        {"""col_1""": """string""", """col_2""": """string""", """col_3""": """string"""},
        {"""col_1""": """int32""", """col_2""": """int32""", """col_3""": """int32"""},
        {"""col_1""": """float32""", """col_2""": """float32""", """col_3""": """float32"""},
    ]				,     )
def       _UpperCamelCase       (a__				:int				,     a__				:Union[str, Any]				,     a__				:str    ):






							"""simple docstring"""





							UpperCamelCase__			       =     tmp_path / """cache"""
							UpperCamelCase__			       =     {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
							UpperCamelCase__			       =     features.copy() if features else default_expected_features
							UpperCamelCase__			       =     (
							    Features({feature: Value(a__    ) for feature, dtype in features.items()}    ) if features is not None else None
							)
							UpperCamelCase__			       =     JsonDatasetReader({"""train""": jsonl_path}				,     features=a__				,     cache_dir=a__    ).read()
							_check_json_datasetdict(a__				,     a__    )





@pytest.mark.parametrize("""split"""				,     [None, NamedSplit("""train"""    ), """train""", """test"""]    )
def       _UpperCamelCase       (a__				:Optional[Any]				,     a__				:Optional[int]				,     a__				:int    ):






							"""simple docstring"""





							if split:
														UpperCamelCase__			       =     {split: jsonl_path}
							else:
														UpperCamelCase__			       =     """train"""
														UpperCamelCase__			       =     {"""train""": jsonl_path, """test""": jsonl_path}
							UpperCamelCase__			       =     tmp_path / """cache"""
							UpperCamelCase__			       =     {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
							UpperCamelCase__			       =     JsonDatasetReader(a__				,     cache_dir=a__    ).read()
							_check_json_datasetdict(a__				,     a__				,     splits=list(path.keys()    )    )
							assert all(dataset[split].split == split for split in path.keys()    )





def       _UpperCamelCase       (a__				:Any    ):






							"""simple docstring"""





							return json.load(a__    )





def       _UpperCamelCase       (a__				:Tuple    ):






							"""simple docstring"""





							return [json.loads(a__    ) for line in buffer]

class     __SCREAMING_SNAKE_CASE   :






							@pytest.mark.parametrize("""lines, load_json_function"""					,	[(True, load_json_lines), (False, load_json)]					)
							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					,	__lowerCAmelCase					,	__lowerCAmelCase					):
														with io.BytesIO() as buffer:
																					JsonDatasetWriter(__lowerCAmelCase					,	__lowerCAmelCase					,	lines=__lowerCAmelCase					).write()
																					buffer.seek(0					)
																					UpperCamelCase__			       =     load_json_function(__lowerCAmelCase					)
														assert isinstance(__lowerCAmelCase					,	__lowerCAmelCase					)
														assert isinstance(exported_content[0]					,	__lowerCAmelCase					)
														assert len(__lowerCAmelCase					) == 10






							@pytest.mark.parametrize(
							    """orient, container, keys, len_at"""					,	[
							        ("""records""", list, {"""tokens""", """labels""", """answers""", """id"""}, None),
							        ("""split""", dict, {"""columns""", """data"""}, """data"""),
							        ("""index""", dict, set("""0123456789"""					), None),
							        ("""columns""", dict, {"""tokens""", """labels""", """answers""", """id"""}, """tokens"""),
							        ("""values""", list, None, None),
							        ("""table""", dict, {"""schema""", """data"""}, """data"""),
							    ]					,	)
							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					,	__lowerCAmelCase					,	__lowerCAmelCase					,	__lowerCAmelCase					,	__lowerCAmelCase					):
														with io.BytesIO() as buffer:
																					JsonDatasetWriter(__lowerCAmelCase					,	__lowerCAmelCase					,	lines=__lowerCAmelCase					,	orient=__lowerCAmelCase					).write()
																					buffer.seek(0					)
																					UpperCamelCase__			       =     load_json(__lowerCAmelCase					)
														assert isinstance(__lowerCAmelCase					,	__lowerCAmelCase					)
														if keys:
																					if container is dict:
																												assert exported_content.keys() == keys
																					else:
																												assert exported_content[0].keys() == keys
														else:
																					assert not hasattr(__lowerCAmelCase					,	"""keys"""					) and not hasattr(exported_content[0]					,	"""keys"""					)
														if len_at:
																					assert len(exported_content[len_at]					) == 10
														else:
																					assert len(__lowerCAmelCase					) == 10






							@pytest.mark.parametrize("""lines, load_json_function"""					,	[(True, load_json_lines), (False, load_json)]					)
							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					,	__lowerCAmelCase					,	__lowerCAmelCase					):
														with io.BytesIO() as buffer:
																					JsonDatasetWriter(__lowerCAmelCase					,	__lowerCAmelCase					,	lines=__lowerCAmelCase					,	num_proc=2					).write()
																					buffer.seek(0					)
																					UpperCamelCase__			       =     load_json_function(__lowerCAmelCase					)
														assert isinstance(__lowerCAmelCase					,	__lowerCAmelCase					)
														assert isinstance(exported_content[0]					,	__lowerCAmelCase					)
														assert len(__lowerCAmelCase					) == 10






							@pytest.mark.parametrize(
							    """orient, container, keys, len_at"""					,	[
							        ("""records""", list, {"""tokens""", """labels""", """answers""", """id"""}, None),
							        ("""split""", dict, {"""columns""", """data"""}, """data"""),
							        ("""index""", dict, set("""0123456789"""					), None),
							        ("""columns""", dict, {"""tokens""", """labels""", """answers""", """id"""}, """tokens"""),
							        ("""values""", list, None, None),
							        ("""table""", dict, {"""schema""", """data"""}, """data"""),
							    ]					,	)
							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					,	__lowerCAmelCase					,	__lowerCAmelCase					,	__lowerCAmelCase					,	__lowerCAmelCase					):
														with io.BytesIO() as buffer:
																					JsonDatasetWriter(__lowerCAmelCase					,	__lowerCAmelCase					,	lines=__lowerCAmelCase					,	orient=__lowerCAmelCase					,	num_proc=2					).write()
																					buffer.seek(0					)
																					UpperCamelCase__			       =     load_json(__lowerCAmelCase					)
														assert isinstance(__lowerCAmelCase					,	__lowerCAmelCase					)
														if keys:
																					if container is dict:
																												assert exported_content.keys() == keys
																					else:
																												assert exported_content[0].keys() == keys
														else:
																					assert not hasattr(__lowerCAmelCase					,	"""keys"""					) and not hasattr(exported_content[0]					,	"""keys"""					)
														if len_at:
																					assert len(exported_content[len_at]					) == 10
														else:
																					assert len(__lowerCAmelCase					) == 10






							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					):
														with pytest.raises(__lowerCAmelCase					):
																					with io.BytesIO() as buffer:
																												JsonDatasetWriter(__lowerCAmelCase					,	__lowerCAmelCase					,	num_proc=0					)







							@pytest.mark.parametrize("""compression, extension"""					,	[("""gzip""", """gz"""), ("""bz2""", """bz2"""), ("""xz""", """xz""")]					)
							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					,	__lowerCAmelCase					,	__lowerCAmelCase					,	__lowerCAmelCase					,	__lowerCAmelCase					):
														UpperCamelCase__			       =     tmp_path_factory.mktemp("""data"""					) / f"""test.json.{extension}"""
														UpperCamelCase__			       =     str(shared_datadir / f"""test_file.json.{extension}"""					)
														JsonDatasetWriter(__lowerCAmelCase					,	__lowerCAmelCase					,	compression=__lowerCAmelCase					).write()

														with fsspec.open(__lowerCAmelCase					,	"""rb"""					,	compression="""infer"""					) as f:
																					UpperCamelCase__			       =     f.read()
														with fsspec.open(__lowerCAmelCase					,	"""rb"""					,	compression="""infer"""					) as f:
																					UpperCamelCase__			       =     f.read()
														assert exported_content == original_content











import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import is_speech_available, is_vision_available
from transformers.testing_utils import require_torch


if is_vision_available():
					from transformers import TvltImageProcessor

if is_speech_available():
					from transformers import TvltFeatureExtractor

from transformers import TvltProcessor

@require_torch
class     __SCREAMING_SNAKE_CASE   (       unittest.TestCase    ):






							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     """ZinengTang/tvlt-base"""
														UpperCamelCase__			       =     tempfile.mkdtemp()






							def 			_lowerCamelCase						(  self					,	**__lowerCAmelCase					):
														return TvltImageProcessor.from_pretrained(self.checkpoint					,	**__lowerCAmelCase					)






							def 			_lowerCamelCase						(  self					,	**__lowerCAmelCase					):
														return TvltFeatureExtractor.from_pretrained(self.checkpoint					,	**__lowerCAmelCase					)






							def 			_lowerCamelCase						(  self					):
														shutil.rmtree(self.tmpdirname					)






							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     self.get_image_processor()
														UpperCamelCase__			       =     self.get_feature_extractor()

														UpperCamelCase__			       =     TvltProcessor(image_processor=__lowerCAmelCase					,	feature_extractor=__lowerCAmelCase					)
														processor.save_pretrained(self.tmpdirname					)
														UpperCamelCase__			       =     TvltProcessor.from_pretrained(self.tmpdirname					)

														self.assertIsInstance(processor.feature_extractor					,	__lowerCAmelCase					)
														self.assertIsInstance(processor.image_processor					,	__lowerCAmelCase					)






							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     self.get_image_processor()
														UpperCamelCase__			       =     self.get_feature_extractor()

														UpperCamelCase__			       =     TvltProcessor(image_processor=__lowerCAmelCase					,	feature_extractor=__lowerCAmelCase					)

														UpperCamelCase__			       =     np.ones([12000]					)

														UpperCamelCase__			       =     feature_extractor(__lowerCAmelCase					,	return_tensors="""np"""					)
														UpperCamelCase__			       =     processor(audio=__lowerCAmelCase					,	return_tensors="""np"""					)

														for key in audio_dict.keys():
																					self.assertAlmostEqual(audio_dict[key].sum()					,	input_processor[key].sum()					,	delta=1E-2					)






							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     self.get_image_processor()
														UpperCamelCase__			       =     self.get_feature_extractor()

														UpperCamelCase__			       =     TvltProcessor(image_processor=__lowerCAmelCase					,	feature_extractor=__lowerCAmelCase					)

														UpperCamelCase__			       =     np.ones([3, 224, 224]					)

														UpperCamelCase__			       =     image_processor(__lowerCAmelCase					,	return_tensors="""np"""					)
														UpperCamelCase__			       =     processor(images=__lowerCAmelCase					,	return_tensors="""np"""					)

														for key in image_dict.keys():
																					self.assertAlmostEqual(image_dict[key].sum()					,	input_processor[key].sum()					,	delta=1E-2					)






							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     self.get_image_processor()
														UpperCamelCase__			       =     self.get_feature_extractor()

														UpperCamelCase__			       =     TvltProcessor(image_processor=__lowerCAmelCase					,	feature_extractor=__lowerCAmelCase					)

														UpperCamelCase__			       =     np.ones([12000]					)
														UpperCamelCase__			       =     np.ones([3, 224, 224]					)

														UpperCamelCase__			       =     processor(audio=__lowerCAmelCase					,	images=__lowerCAmelCase					)

														self.assertListEqual(list(inputs.keys()					)					,	["""audio_values""", """audio_mask""", """pixel_values""", """pixel_mask"""]					)

														# test if it raises when no input is passed
														with pytest.raises(__lowerCAmelCase					):
																					processor()







							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     self.get_image_processor()
														UpperCamelCase__			       =     self.get_feature_extractor()

														UpperCamelCase__			       =     TvltProcessor(image_processor=__lowerCAmelCase					,	feature_extractor=__lowerCAmelCase					)

														self.assertListEqual(
														    processor.model_input_names					,	image_processor.model_input_names + feature_extractor.model_input_names					,	msg="""`processor` and `image_processor`+`feature_extractor` model input names do not match"""					,	)




