

'''simple docstring'''



import contextlib
import csv
import json
import os
import sqlitea
import tarfile
import textwrap
import zipfile

import pyarrow as pa
import pyarrow.parquet as pq
import pytest

import datasets
import datasets.config


@pytest.fixture(scope='''session'''					)
def 				lowerCamelCase__				(	):




						'''simple docstring'''






						UpperCAmelCase					    =     10
						UpperCAmelCase					    =     datasets.Features(
						    {
						        '''tokens''': datasets.Sequence(datasets.Value('''string'''					)					),
						        '''labels''': datasets.Sequence(datasets.ClassLabel(names=['''negative''', '''positive''']					)					),
						        '''answers''': datasets.Sequence(
						            {
						                '''text''': datasets.Value('''string'''					),
						                '''answer_start''': datasets.Value('''int32'''					),
						            }					),
						        '''id''': datasets.Value('''int64'''					),
						    }					)
						UpperCAmelCase					    =     datasets.Dataset.from_dict(
						    {
						        '''tokens''': [['''foo'''] * 5] * n,
						        '''labels''': [[1] * 5] * n,
						        '''answers''': [{'''answer_start''': [97], '''text''': ['''1976''']}] * 10,
						        '''id''': list(range(UpperCAmelCase__					)					),
						    }				,       features=UpperCAmelCase__				,       )
						return dataset


@pytest.fixture(scope='''session'''					)
def 				lowerCamelCase__				(	A       :      int				,       A       :      Optional[int]					):




						'''simple docstring'''






						UpperCAmelCase					    =     str(tmp_path_factory.mktemp('''data'''					) / '''file.arrow'''					)
						dataset.map(cache_file_name=UpperCAmelCase__					)
						return filename


# FILE_CONTENT + files


_lowercase      :      Dict            =					"""\
    Text data.
    Second line of data."""


@pytest.fixture(scope='''session'''					)
def 				lowerCamelCase__				(	A       :      int					):




						'''simple docstring'''






						UpperCAmelCase					    =     tmp_path_factory.mktemp('''data'''					) / '''file.txt'''
						UpperCAmelCase					    =     FILE_CONTENT
						with open(UpperCAmelCase__				,       '''w'''					) as f:
												f.write(UpperCAmelCase__					)
						return filename


@pytest.fixture(scope='''session'''					)
def 				lowerCamelCase__				(	A       :      int					):




						'''simple docstring'''






						import bza

						UpperCAmelCase					    =     tmp_path_factory.mktemp('''data'''					) / '''file.txt.bz2'''
						UpperCAmelCase					    =     bytes(UpperCAmelCase__				,       '''utf-8'''					)
						with bza.open(UpperCAmelCase__				,       '''wb'''					) as f:
												f.write(UpperCAmelCase__					)
						return path


@pytest.fixture(scope='''session'''					)
def 				lowerCamelCase__				(	A       :      List[Any]					):




						'''simple docstring'''






						import gzip

						UpperCAmelCase					    =     str(tmp_path_factory.mktemp('''data'''					) / '''file.txt.gz'''					)
						UpperCAmelCase					    =     bytes(UpperCAmelCase__				,       '''utf-8'''					)
						with gzip.open(UpperCAmelCase__				,       '''wb'''					) as f:
												f.write(UpperCAmelCase__					)
						return path


@pytest.fixture(scope='''session'''					)
def 				lowerCamelCase__				(	A       :      Optional[Any]					):




						'''simple docstring'''






						if datasets.config.LZ4_AVAILABLE:
												import lza.frame

												UpperCAmelCase					    =     tmp_path_factory.mktemp('''data'''					) / '''file.txt.lz4'''
												UpperCAmelCase					    =     bytes(UpperCAmelCase__				,       '''utf-8'''					)
												with lza.frame.open(UpperCAmelCase__				,       '''wb'''					) as f:
																		f.write(UpperCAmelCase__					)
												return path


@pytest.fixture(scope='''session'''					)
def 				lowerCamelCase__				(	A       :      str				,       A       :      Union[str, Any]					):




						'''simple docstring'''






						if datasets.config.PY7ZR_AVAILABLE:
												import pyazr

												UpperCAmelCase					    =     tmp_path_factory.mktemp('''data'''					) / '''file.txt.7z'''
												with pyazr.SevenZipFile(UpperCAmelCase__				,       '''w'''					) as archive:
																		archive.write(UpperCAmelCase__				,       arcname=os.path.basename(UpperCAmelCase__					)					)
												return path


@pytest.fixture(scope='''session'''					)
def 				lowerCamelCase__				(	A       :      Optional[int]				,       A       :      Dict					):




						'''simple docstring'''






						import tarfile

						UpperCAmelCase					    =     tmp_path_factory.mktemp('''data'''					) / '''file.txt.tar'''
						with tarfile.TarFile(UpperCAmelCase__				,       '''w'''					) as f:
												f.add(UpperCAmelCase__				,       arcname=os.path.basename(UpperCAmelCase__					)					)
						return path


@pytest.fixture(scope='''session'''					)
def 				lowerCamelCase__				(	A       :      Optional[Any]					):




						'''simple docstring'''






						import lzma

						UpperCAmelCase					    =     tmp_path_factory.mktemp('''data'''					) / '''file.txt.xz'''
						UpperCAmelCase					    =     bytes(UpperCAmelCase__				,       '''utf-8'''					)
						with lzma.open(UpperCAmelCase__				,       '''wb'''					) as f:
												f.write(UpperCAmelCase__					)
						return path


@pytest.fixture(scope='''session'''					)
def 				lowerCamelCase__				(	A       :      Any				,       A       :      str					):




						'''simple docstring'''






						import zipfile

						UpperCAmelCase					    =     tmp_path_factory.mktemp('''data'''					) / '''file.txt.zip'''
						with zipfile.ZipFile(UpperCAmelCase__				,       '''w'''					) as f:
												f.write(UpperCAmelCase__				,       arcname=os.path.basename(UpperCAmelCase__					)					)
						return path


@pytest.fixture(scope='''session'''					)
def 				lowerCamelCase__				(	A       :      List[Any]					):




						'''simple docstring'''






						if datasets.config.ZSTANDARD_AVAILABLE:
												import zstandard as zstd

												UpperCAmelCase					    =     tmp_path_factory.mktemp('''data'''					) / '''file.txt.zst'''
												UpperCAmelCase					    =     bytes(UpperCAmelCase__				,       '''utf-8'''					)
												with zstd.open(UpperCAmelCase__				,       '''wb'''					) as f:
																		f.write(UpperCAmelCase__					)
												return path


@pytest.fixture(scope='''session'''					)
def 				lowerCamelCase__				(	A       :      int					):




						'''simple docstring'''






						UpperCAmelCase					    =     tmp_path_factory.mktemp('''data'''					) / '''file.xml'''
						UpperCAmelCase					    =     textwrap.dedent(
						    '''\
    <?xml version=\"1.0\" encoding=\"UTF-8\" ?>
    <tmx version=\"1.4\">
      <header segtype=\"sentence\" srclang=\"ca\" />
      <body>
        <tu>
          <tuv xml:lang=\"ca\"><seg>Contingut 1</seg></tuv>
          <tuv xml:lang=\"en\"><seg>Content 1</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang=\"ca\"><seg>Contingut 2</seg></tuv>
          <tuv xml:lang=\"en\"><seg>Content 2</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang=\"ca\"><seg>Contingut 3</seg></tuv>
          <tuv xml:lang=\"en\"><seg>Content 3</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang=\"ca\"><seg>Contingut 4</seg></tuv>
          <tuv xml:lang=\"en\"><seg>Content 4</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang=\"ca\"><seg>Contingut 5</seg></tuv>
          <tuv xml:lang=\"en\"><seg>Content 5</seg></tuv>
        </tu>
      </body>
    </tmx>'''					)
						with open(UpperCAmelCase__				,       '''w'''					) as f:
												f.write(UpperCAmelCase__					)
						return filename


_lowercase      :      List[Any]            =					[
    {"""col_1""": """0""", """col_2""": 0, """col_3""": 0.0},
    {"""col_1""": """1""", """col_2""": 1, """col_3""": 1.0},
    {"""col_1""": """2""", """col_2""": 2, """col_3""": 2.0},
    {"""col_1""": """3""", """col_2""": 3, """col_3""": 3.0},
]
_lowercase      :      Any            =					[
    {"""col_1""": """4""", """col_2""": 4, """col_3""": 4.0},
    {"""col_1""": """5""", """col_2""": 5, """col_3""": 5.0},
]
_lowercase      :      Any            =					{
    """col_1""": ["""0""", """1""", """2""", """3"""],
    """col_2""": [0, 1, 2, 3],
    """col_3""": [0.0, 1.0, 2.0, 3.0],
}

_lowercase      :      Optional[Any]            =					[
    {"""col_3""": 0.0, """col_1""": """0""", """col_2""": 0},
    {"""col_3""": 1.0, """col_1""": """1""", """col_2""": 1},
]

_lowercase      :      Tuple            =					[
    {"""col_1""": """s0""", """col_2""": 0, """col_3""": 0.0},
    {"""col_1""": """s1""", """col_2""": 1, """col_3""": 1.0},
    {"""col_1""": """s2""", """col_2""": 2, """col_3""": 2.0},
    {"""col_1""": """s3""", """col_2""": 3, """col_3""": 3.0},
]


@pytest.fixture(scope='''session'''					)
def 				lowerCamelCase__				(	):




						'''simple docstring'''






						return DATA_DICT_OF_LISTS


@pytest.fixture(scope='''session'''					)
def 				lowerCamelCase__				(	A       :      Optional[int]					):




						'''simple docstring'''






						UpperCAmelCase					    =     datasets.Dataset.from_dict(UpperCAmelCase__					)
						UpperCAmelCase					    =     str(tmp_path_factory.mktemp('''data'''					) / '''dataset.arrow'''					)
						dataset.map(cache_file_name=UpperCAmelCase__					)
						return path


@pytest.fixture(scope='''session'''					)
def 				lowerCamelCase__				(	A       :      str					):




						'''simple docstring'''






						UpperCAmelCase					    =     str(tmp_path_factory.mktemp('''data'''					) / '''dataset.sqlite'''					)
						with contextlib.closing(sqlitea.connect(UpperCAmelCase__					)					) as con:
												UpperCAmelCase					    =     con.cursor()
												cur.execute('''CREATE TABLE dataset(col_1 text, col_2 int, col_3 real)'''					)
												for item in DATA:
																		cur.execute('''INSERT INTO dataset(col_1, col_2, col_3) VALUES (?, ?, ?)'''				,       tuple(item.values()					)					)
												con.commit()
						return path


@pytest.fixture(scope='''session'''					)
def 				lowerCamelCase__				(	A       :      Optional[Any]					):




						'''simple docstring'''






						UpperCAmelCase					    =     str(tmp_path_factory.mktemp('''data'''					) / '''dataset.csv'''					)
						with open(UpperCAmelCase__				,       '''w'''				,       newline=''''''					) as f:
												UpperCAmelCase					    =     csv.DictWriter(UpperCAmelCase__				,       fieldnames=['''col_1''', '''col_2''', '''col_3''']					)
												writer.writeheader()
												for item in DATA:
																		writer.writerow(UpperCAmelCase__					)
						return path


@pytest.fixture(scope='''session'''					)
def 				lowerCamelCase__				(	A       :      Any					):




						'''simple docstring'''






						UpperCAmelCase					    =     str(tmp_path_factory.mktemp('''data'''					) / '''dataset2.csv'''					)
						with open(UpperCAmelCase__				,       '''w'''				,       newline=''''''					) as f:
												UpperCAmelCase					    =     csv.DictWriter(UpperCAmelCase__				,       fieldnames=['''col_1''', '''col_2''', '''col_3''']					)
												writer.writeheader()
												for item in DATA:
																		writer.writerow(UpperCAmelCase__					)
						return path


@pytest.fixture(scope='''session'''					)
def 				lowerCamelCase__				(	A       :      Union[str, Any]				,       A       :      List[str]					):




						'''simple docstring'''






						import bza

						UpperCAmelCase					    =     tmp_path_factory.mktemp('''data'''					) / '''dataset.csv.bz2'''
						with open(UpperCAmelCase__				,       '''rb'''					) as f:
												UpperCAmelCase					    =     f.read()
						# data = bytes(FILE_CONTENT, "utf-8")
						with bza.open(UpperCAmelCase__				,       '''wb'''					) as f:
												f.write(UpperCAmelCase__					)
						return path


@pytest.fixture(scope='''session'''					)
def 				lowerCamelCase__				(	A       :      Union[str, Any]				,       A       :      int				,       A       :      int					):




						'''simple docstring'''






						UpperCAmelCase					    =     tmp_path_factory.mktemp('''data'''					) / '''dataset.csv.zip'''
						with zipfile.ZipFile(UpperCAmelCase__				,       '''w'''					) as f:
												f.write(UpperCAmelCase__				,       arcname=os.path.basename(UpperCAmelCase__					)					)
												f.write(UpperCAmelCase__				,       arcname=os.path.basename(UpperCAmelCase__					)					)
						return path


@pytest.fixture(scope='''session'''					)
def 				lowerCamelCase__				(	A       :      Dict				,       A       :      Any				,       A       :      List[Any]					):




						'''simple docstring'''






						UpperCAmelCase					    =     tmp_path_factory.mktemp('''data'''					) / '''dataset.csv.zip'''
						with zipfile.ZipFile(UpperCAmelCase__				,       '''w'''					) as f:
												f.write(UpperCAmelCase__				,       arcname=os.path.basename(csv_path.replace('''.csv'''				,       '''.CSV'''					)					)					)
												f.write(UpperCAmelCase__				,       arcname=os.path.basename(csva_path.replace('''.csv'''				,       '''.CSV'''					)					)					)
						return path


@pytest.fixture(scope='''session'''					)
def 				lowerCamelCase__				(	A       :      str				,       A       :      Optional[int]				,       A       :      Tuple					):




						'''simple docstring'''






						UpperCAmelCase					    =     tmp_path_factory.mktemp('''data'''					) / '''dataset_with_dir.csv.zip'''
						with zipfile.ZipFile(UpperCAmelCase__				,       '''w'''					) as f:
												f.write(UpperCAmelCase__				,       arcname=os.path.join('''main_dir'''				,       os.path.basename(UpperCAmelCase__					)					)					)
												f.write(UpperCAmelCase__				,       arcname=os.path.join('''main_dir'''				,       os.path.basename(UpperCAmelCase__					)					)					)
						return path


@pytest.fixture(scope='''session'''					)
def 				lowerCamelCase__				(	A       :      Dict					):




						'''simple docstring'''






						UpperCAmelCase					    =     str(tmp_path_factory.mktemp('''data'''					) / '''dataset.parquet'''					)
						UpperCAmelCase					    =     pa.schema(
						    {
						        '''col_1''': pa.string(),
						        '''col_2''': pa.intaa(),
						        '''col_3''': pa.floataa(),
						    }					)
						with open(UpperCAmelCase__				,       '''wb'''					) as f:
												UpperCAmelCase					    =     pq.ParquetWriter(UpperCAmelCase__				,       schema=UpperCAmelCase__					)
												UpperCAmelCase					    =     pa.Table.from_pydict({k: [DATA[i][k] for i in range(len(UpperCAmelCase__					)					)] for k in DATA[0]}				,       schema=UpperCAmelCase__					)
												writer.write_table(UpperCAmelCase__					)
												writer.close()
						return path


@pytest.fixture(scope='''session'''					)
def 				lowerCamelCase__				(	A       :      Optional[int]					):




						'''simple docstring'''






						UpperCAmelCase					    =     str(tmp_path_factory.mktemp('''data'''					) / '''dataset.json'''					)
						UpperCAmelCase					    =     {'''data''': DATA}
						with open(UpperCAmelCase__				,       '''w'''					) as f:
												json.dump(UpperCAmelCase__				,       UpperCAmelCase__					)
						return path


@pytest.fixture(scope='''session'''					)
def 				lowerCamelCase__				(	A       :      Union[str, Any]					):




						'''simple docstring'''






						UpperCAmelCase					    =     str(tmp_path_factory.mktemp('''data'''					) / '''dataset.json'''					)
						UpperCAmelCase					    =     {'''data''': DATA_DICT_OF_LISTS}
						with open(UpperCAmelCase__				,       '''w'''					) as f:
												json.dump(UpperCAmelCase__				,       UpperCAmelCase__					)
						return path


@pytest.fixture(scope='''session'''					)
def 				lowerCamelCase__				(	A       :      Any					):




						'''simple docstring'''






						UpperCAmelCase					    =     str(tmp_path_factory.mktemp('''data'''					) / '''dataset.jsonl'''					)
						with open(UpperCAmelCase__				,       '''w'''					) as f:
												for item in DATA:
																		f.write(json.dumps(UpperCAmelCase__					) + '''\n'''					)
						return path


@pytest.fixture(scope='''session'''					)
def 				lowerCamelCase__				(	A       :      int					):




						'''simple docstring'''






						UpperCAmelCase					    =     str(tmp_path_factory.mktemp('''data'''					) / '''dataset2.jsonl'''					)
						with open(UpperCAmelCase__				,       '''w'''					) as f:
												for item in DATA:
																		f.write(json.dumps(UpperCAmelCase__					) + '''\n'''					)
						return path


@pytest.fixture(scope='''session'''					)
def 				lowerCamelCase__				(	A       :      Optional[Any]					):




						'''simple docstring'''






						UpperCAmelCase					    =     str(tmp_path_factory.mktemp('''data'''					) / '''dataset_312.jsonl'''					)
						with open(UpperCAmelCase__				,       '''w'''					) as f:
												for item in DATA_312:
																		f.write(json.dumps(UpperCAmelCase__					) + '''\n'''					)
						return path


@pytest.fixture(scope='''session'''					)
def 				lowerCamelCase__				(	A       :      str					):




						'''simple docstring'''






						UpperCAmelCase					    =     str(tmp_path_factory.mktemp('''data'''					) / '''dataset-str.jsonl'''					)
						with open(UpperCAmelCase__				,       '''w'''					) as f:
												for item in DATA_STR:
																		f.write(json.dumps(UpperCAmelCase__					) + '''\n'''					)
						return path


@pytest.fixture(scope='''session'''					)
def 				lowerCamelCase__				(	A       :      str				,       A       :      Optional[int]					):




						'''simple docstring'''






						import gzip

						UpperCAmelCase					    =     str(tmp_path_factory.mktemp('''data'''					) / '''dataset.txt.gz'''					)
						with open(UpperCAmelCase__				,       '''rb'''					) as orig_file:
												with gzip.open(UpperCAmelCase__				,       '''wb'''					) as zipped_file:
																		zipped_file.writelines(UpperCAmelCase__					)
						return path


@pytest.fixture(scope='''session'''					)
def 				lowerCamelCase__				(	A       :      List[Any]				,       A       :      List[Any]					):




						'''simple docstring'''






						import gzip

						UpperCAmelCase					    =     str(tmp_path_factory.mktemp('''data'''					) / '''dataset.jsonl.gz'''					)
						with open(UpperCAmelCase__				,       '''rb'''					) as orig_file:
												with gzip.open(UpperCAmelCase__				,       '''wb'''					) as zipped_file:
																		zipped_file.writelines(UpperCAmelCase__					)
						return path


@pytest.fixture(scope='''session'''					)
def 				lowerCamelCase__				(	A       :      Tuple				,       A       :      str				,       A       :      List[str]					):




						'''simple docstring'''






						UpperCAmelCase					    =     tmp_path_factory.mktemp('''data'''					) / '''dataset.jsonl.zip'''
						with zipfile.ZipFile(UpperCAmelCase__				,       '''w'''					) as f:
												f.write(UpperCAmelCase__				,       arcname=os.path.basename(UpperCAmelCase__					)					)
												f.write(UpperCAmelCase__				,       arcname=os.path.basename(UpperCAmelCase__					)					)
						return path


@pytest.fixture(scope='''session'''					)
def 				lowerCamelCase__				(	A       :      Tuple				,       A       :      Union[str, Any]				,       A       :      int				,       A       :      Optional[Any]					):




						'''simple docstring'''






						UpperCAmelCase					    =     tmp_path_factory.mktemp('''data'''					) / '''dataset_nested.jsonl.zip'''
						with zipfile.ZipFile(UpperCAmelCase__				,       '''w'''					) as f:
												f.write(UpperCAmelCase__				,       arcname=os.path.join('''nested'''				,       os.path.basename(UpperCAmelCase__					)					)					)
						return path


@pytest.fixture(scope='''session'''					)
def 				lowerCamelCase__				(	A       :      Union[str, Any]				,       A       :      Optional[int]				,       A       :      Union[str, Any]					):




						'''simple docstring'''






						UpperCAmelCase					    =     tmp_path_factory.mktemp('''data'''					) / '''dataset_with_dir.jsonl.zip'''
						with zipfile.ZipFile(UpperCAmelCase__				,       '''w'''					) as f:
												f.write(UpperCAmelCase__				,       arcname=os.path.join('''main_dir'''				,       os.path.basename(UpperCAmelCase__					)					)					)
												f.write(UpperCAmelCase__				,       arcname=os.path.join('''main_dir'''				,       os.path.basename(UpperCAmelCase__					)					)					)
						return path


@pytest.fixture(scope='''session'''					)
def 				lowerCamelCase__				(	A       :      Union[str, Any]				,       A       :      Optional[int]				,       A       :      Union[str, Any]					):




						'''simple docstring'''






						UpperCAmelCase					    =     tmp_path_factory.mktemp('''data'''					) / '''dataset.jsonl.tar'''
						with tarfile.TarFile(UpperCAmelCase__				,       '''w'''					) as f:
												f.add(UpperCAmelCase__				,       arcname=os.path.basename(UpperCAmelCase__					)					)
												f.add(UpperCAmelCase__				,       arcname=os.path.basename(UpperCAmelCase__					)					)
						return path


@pytest.fixture(scope='''session'''					)
def 				lowerCamelCase__				(	A       :      Dict				,       A       :      Optional[Any]				,       A       :      Union[str, Any]				,       A       :      Tuple					):




						'''simple docstring'''






						UpperCAmelCase					    =     tmp_path_factory.mktemp('''data'''					) / '''dataset_nested.jsonl.tar'''
						with tarfile.TarFile(UpperCAmelCase__				,       '''w'''					) as f:
												f.add(UpperCAmelCase__				,       arcname=os.path.join('''nested'''				,       os.path.basename(UpperCAmelCase__					)					)					)
						return path


@pytest.fixture(scope='''session'''					)
def 				lowerCamelCase__				(	A       :      List[str]					):




						'''simple docstring'''






						UpperCAmelCase					    =     ['''0''', '''1''', '''2''', '''3''']
						UpperCAmelCase					    =     str(tmp_path_factory.mktemp('''data'''					) / '''dataset.txt'''					)
						with open(UpperCAmelCase__				,       '''w'''					) as f:
												for item in data:
																		f.write(item + '''\n'''					)
						return path


@pytest.fixture(scope='''session'''					)
def 				lowerCamelCase__				(	A       :      int					):




						'''simple docstring'''






						UpperCAmelCase					    =     ['''0''', '''1''', '''2''', '''3''']
						UpperCAmelCase					    =     str(tmp_path_factory.mktemp('''data'''					) / '''dataset2.txt'''					)
						with open(UpperCAmelCase__				,       '''w'''					) as f:
												for item in data:
																		f.write(item + '''\n'''					)
						return path


@pytest.fixture(scope='''session'''					)
def 				lowerCamelCase__				(	A       :      Dict					):




						'''simple docstring'''






						UpperCAmelCase					    =     ['''0''', '''1''', '''2''', '''3''']
						UpperCAmelCase					    =     tmp_path_factory.mktemp('''data'''					) / '''dataset.abc'''
						with open(UpperCAmelCase__				,       '''w'''					) as f:
												for item in data:
																		f.write(item + '''\n'''					)
						return path


@pytest.fixture(scope='''session'''					)
def 				lowerCamelCase__				(	A       :      List[str]				,       A       :      int				,       A       :      int					):




						'''simple docstring'''






						UpperCAmelCase					    =     tmp_path_factory.mktemp('''data'''					) / '''dataset.text.zip'''
						with zipfile.ZipFile(UpperCAmelCase__				,       '''w'''					) as f:
												f.write(UpperCAmelCase__				,       arcname=os.path.basename(UpperCAmelCase__					)					)
												f.write(UpperCAmelCase__				,       arcname=os.path.basename(UpperCAmelCase__					)					)
						return path


@pytest.fixture(scope='''session'''					)
def 				lowerCamelCase__				(	A       :      Tuple				,       A       :      Optional[int]				,       A       :      List[Any]					):




						'''simple docstring'''






						UpperCAmelCase					    =     tmp_path_factory.mktemp('''data'''					) / '''dataset_with_dir.text.zip'''
						with zipfile.ZipFile(UpperCAmelCase__				,       '''w'''					) as f:
												f.write(UpperCAmelCase__				,       arcname=os.path.join('''main_dir'''				,       os.path.basename(UpperCAmelCase__					)					)					)
												f.write(UpperCAmelCase__				,       arcname=os.path.join('''main_dir'''				,       os.path.basename(UpperCAmelCase__					)					)					)
						return path


@pytest.fixture(scope='''session'''					)
def 				lowerCamelCase__				(	A       :      List[Any]				,       A       :      List[Any]				,       A       :      Optional[int]					):




						'''simple docstring'''






						UpperCAmelCase					    =     tmp_path_factory.mktemp('''data'''					) / '''dataset.ext.zip'''
						with zipfile.ZipFile(UpperCAmelCase__				,       '''w'''					) as f:
												f.write(UpperCAmelCase__				,       arcname=os.path.basename('''unsupported.ext'''					)					)
												f.write(UpperCAmelCase__				,       arcname=os.path.basename('''unsupported_2.ext'''					)					)
						return path


@pytest.fixture(scope='''session'''					)
def 				lowerCamelCase__				(	A       :      Optional[int]					):




						'''simple docstring'''






						UpperCAmelCase					    =     '''\n'''.join(['''First''', '''Second\u2029with Unicode new line''', '''Third''']					)
						UpperCAmelCase					    =     str(tmp_path_factory.mktemp('''data'''					) / '''dataset_with_unicode_new_lines.txt'''					)
						with open(UpperCAmelCase__				,       '''w'''				,       encoding='''utf-8'''					) as f:
												f.write(UpperCAmelCase__					)
						return path


@pytest.fixture(scope='''session'''					)
def 				lowerCamelCase__				(	):




						'''simple docstring'''






						return os.path.join('''tests'''				,       '''features'''				,       '''data'''				,       '''test_image_rgb.jpg'''					)


@pytest.fixture(scope='''session'''					)
def 				lowerCamelCase__				(	):




						'''simple docstring'''






						return os.path.join('''tests'''				,       '''features'''				,       '''data'''				,       '''test_audio_44100.wav'''					)


@pytest.fixture(scope='''session'''					)
def 				lowerCamelCase__				(	A       :      Optional[int]				,       A       :      Optional[int]					):




						'''simple docstring'''






						UpperCAmelCase					    =     tmp_path_factory.mktemp('''data'''					) / '''dataset.img.zip'''
						with zipfile.ZipFile(UpperCAmelCase__				,       '''w'''					) as f:
												f.write(UpperCAmelCase__				,       arcname=os.path.basename(UpperCAmelCase__					)					)
												f.write(UpperCAmelCase__				,       arcname=os.path.basename(UpperCAmelCase__					).replace('''.jpg'''				,       '''2.jpg'''					)					)
						return path




@pytest.fixture(scope='''session'''					)
def 				lowerCamelCase__				(	A       :      Optional[Any]					):




						'''simple docstring'''






						UpperCAmelCase					    =     tmp_path_factory.mktemp('''data_dir'''					)

						(data_dir / "subdir").mkdir()
						with open(data_dir / '''subdir''' / '''train.txt'''				,       '''w'''					) as f:
												f.write('''foo\n''' * 10					)
						with open(data_dir / '''subdir''' / '''test.txt'''				,       '''w'''					) as f:
												f.write('''bar\n''' * 10					)
						# hidden file
						with open(data_dir / '''subdir''' / '''.test.txt'''				,       '''w'''					) as f:
												f.write('''bar\n''' * 10					)

						# hidden directory
						(data_dir / ".subdir").mkdir()
						with open(data_dir / '''.subdir''' / '''train.txt'''				,       '''w'''					) as f:
												f.write('''foo\n''' * 10					)
						with open(data_dir / '''.subdir''' / '''test.txt'''				,       '''w'''					) as f:
												f.write('''bar\n''' * 10					)

						return data_dir









'''simple docstring'''



import gc
import unittest

import numpy as np
import torch
from transformers import CLIPTextConfig, CLIPTextModelWithProjection, CLIPTokenizer

from diffusers import HeunDiscreteScheduler, PriorTransformer, ShapEPipeline
from diffusers.pipelines.shap_e import ShapERenderer
from diffusers.utils import load_numpy, slow
from diffusers.utils.testing_utils import require_torch_gpu, torch_device

from ..test_pipelines_common import PipelineTesterMixin, assert_mean_pixel_difference






class 	UpperCamelCase__(       lowerCAmelCase						,    unittest.TestCase	):
				__magic_name__    :				Any  		=	ShapEPipeline
				__magic_name__    :				Tuple  		=	["prompt"]
				__magic_name__    :				Optional[int]  		=	["prompt"]
				__magic_name__    :				Dict  		=	[
				    "num_images_per_prompt",
				    "num_inference_steps",
				    "generator",
				    "latents",
				    "guidance_scale",
				    "frame_size",
				    "output_type",
				    "return_dict",
				]
				__magic_name__    :				Optional[int]  		=	False




				@property
				def 					a__(		self      :			Optional[Any]				)->					Dict:

										"""simple docstring"""




										return 32




				@property
				def 					a__(		self      :			Dict				)->					Dict:

										"""simple docstring"""




										return 32




				@property
				def 					a__(		self      :			Optional[Any]				)->					List[str]:

										"""simple docstring"""




										return self.time_input_dim * 4




				@property
				def 					a__(		self      :			List[str]				)->					str:

										"""simple docstring"""




										return 8




				@property
				def 					a__(		self      :			int				)->					Optional[int]:

										"""simple docstring"""




										UpperCAmelCase					    =     CLIPTokenizer.from_pretrained('''hf-internal-testing/tiny-random-clip'''				)
										return tokenizer




				@property
				def 					a__(		self      :			Tuple				)->					Optional[int]:

										"""simple docstring"""




										torch.manual_seed(0				)
										UpperCAmelCase					    =     CLIPTextConfig(
										    bos_token_id=0				,   eos_token_id=2				,   hidden_size=self.text_embedder_hidden_size				,   projection_dim=self.text_embedder_hidden_size				,   intermediate_size=37				,   layer_norm_eps=1E-05				,   num_attention_heads=4				,   num_hidden_layers=5				,   pad_token_id=1				,   vocab_size=1000				,   )
										return CLIPTextModelWithProjection(lowerCAmelCase				)




				@property
				def 					a__(		self      :			str				)->					List[str]:

										"""simple docstring"""




										torch.manual_seed(0				)

										UpperCAmelCase					    =     {
										    '''num_attention_heads''': 2,
										    '''attention_head_dim''': 16,
										    '''embedding_dim''': self.time_input_dim,
										    '''num_embeddings''': 32,
										    '''embedding_proj_dim''': self.text_embedder_hidden_size,
										    '''time_embed_dim''': self.time_embed_dim,
										    '''num_layers''': 1,
										    '''clip_embed_dim''': self.time_input_dim * 2,
										    '''additional_embeddings''': 0,
										    '''time_embed_act_fn''': '''gelu''',
										    '''norm_in_type''': '''layer''',
										    '''encoder_hid_proj_type''': None,
										    '''added_emb_type''': None,
										}

										UpperCAmelCase					    =     PriorTransformer(**lowerCAmelCase				)
										return model




				@property
				def 					a__(		self      :			List[Any]				)->					List[Any]:

										"""simple docstring"""




										torch.manual_seed(0				)

										UpperCAmelCase					    =     {
										    '''param_shapes''': (
										        (self.renderer_dim, 93),
										        (self.renderer_dim, 8),
										        (self.renderer_dim, 8),
										        (self.renderer_dim, 8),
										    ),
										    '''d_latent''': self.time_input_dim,
										    '''d_hidden''': self.renderer_dim,
										    '''n_output''': 12,
										    '''background''': (
										        0.1,
										        0.1,
										        0.1,
										    ),
										}
										UpperCAmelCase					    =     ShapERenderer(**lowerCAmelCase				)
										return model




				def 					a__(		self      :			Any				)->					Tuple:

										"""simple docstring"""




										UpperCAmelCase					    =     self.dummy_prior
										UpperCAmelCase					    =     self.dummy_text_encoder
										UpperCAmelCase					    =     self.dummy_tokenizer
										UpperCAmelCase					    =     self.dummy_renderer

										UpperCAmelCase					    =     HeunDiscreteScheduler(
										    beta_schedule='''exp'''				,   num_train_timesteps=1024				,   prediction_type='''sample'''				,   use_karras_sigmas=lowerCAmelCase				,   clip_sample=lowerCAmelCase				,   clip_sample_range=1.0				,   )
										UpperCAmelCase					    =     {
										    '''prior''': prior,
										    '''text_encoder''': text_encoder,
										    '''tokenizer''': tokenizer,
										    '''renderer''': renderer,
										    '''scheduler''': scheduler,
										}

										return components




				def 					a__(		self      :			Tuple				,   lowerCAmelCase      :			Union[str, Any]				,   lowerCAmelCase      :			List[Any]=0				)->					Optional[Any]:

										"""simple docstring"""




										if str(lowerCAmelCase				).startswith('''mps'''				):
																UpperCAmelCase					    =     torch.manual_seed(lowerCAmelCase				)
										else:
																UpperCAmelCase					    =     torch.Generator(device=lowerCAmelCase				).manual_seed(lowerCAmelCase				)
										UpperCAmelCase					    =     {
										    '''prompt''': '''horse''',
										    '''generator''': generator,
										    '''num_inference_steps''': 1,
										    '''frame_size''': 32,
										    '''output_type''': '''np''',
										}
										return inputs




				def 					a__(		self      :			int				)->					Dict:

										"""simple docstring"""




										UpperCAmelCase					    =     '''cpu'''

										UpperCAmelCase					    =     self.get_dummy_components()

										UpperCAmelCase					    =     self.pipeline_class(**lowerCAmelCase				)
										UpperCAmelCase					    =     pipe.to(lowerCAmelCase				)

										pipe.set_progress_bar_config(disable=lowerCAmelCase				)

										UpperCAmelCase					    =     pipe(**self.get_dummy_inputs(lowerCAmelCase				)				)
										UpperCAmelCase					    =     output.images[0]
										UpperCAmelCase					    =     image[0, -3:, -3:, -1]

										assert image.shape == (20, 32, 32, 3)

										UpperCAmelCase					    =     np.array(
										    [
										        0.00039216,
										        0.00039216,
										        0.00039216,
										        0.00039216,
										        0.00039216,
										        0.00039216,
										        0.00039216,
										        0.00039216,
										        0.00039216,
										    ]				)

										assert np.abs(image_slice.flatten() - expected_slice				).max() < 1E-2




				def 					a__(		self      :			Union[str, Any]				)->					Optional[int]:

										"""simple docstring"""




										self._test_inference_batch_consistent(batch_sizes=[1, 2]				)




				def 					a__(		self      :			Optional[int]				)->					str:

										"""simple docstring"""




										UpperCAmelCase					    =     torch_device == '''cpu'''
										UpperCAmelCase					    =     True

										self._test_inference_batch_single_identical(
										    batch_size=2				,   test_max_difference=lowerCAmelCase				,   relax_max_difference=lowerCAmelCase				,   )





				def 					a__(		self      :			int				)->					List[str]:

										"""simple docstring"""




										UpperCAmelCase					    =     self.get_dummy_components()
										UpperCAmelCase					    =     self.pipeline_class(**lowerCAmelCase				)
										UpperCAmelCase					    =     pipe.to(lowerCAmelCase				)
										pipe.set_progress_bar_config(disable=lowerCAmelCase				)

										UpperCAmelCase					    =     1
										UpperCAmelCase					    =     2

										UpperCAmelCase					    =     self.get_dummy_inputs(lowerCAmelCase				)

										for key in inputs.keys():
																if key in self.batch_params:
																						UpperCAmelCase					    =     batch_size * [inputs[key]]

										UpperCAmelCase					    =     pipe(**lowerCAmelCase				,   num_images_per_prompt=lowerCAmelCase				)[0]

										assert images.shape[0] == batch_size * num_images_per_prompt







@slow
@require_torch_gpu
class 	UpperCamelCase__(       unittest.TestCase	):




				def 					a__(		self      :			Dict				)->					Union[str, Any]:

										"""simple docstring"""




										super().tearDown()
										gc.collect()
										torch.cuda.empty_cache()





				def 					a__(		self      :			Optional[Any]				)->					Union[str, Any]:

										"""simple docstring"""




										UpperCAmelCase					    =     load_numpy(
										    '''https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main'''
										    '''/shap_e/test_shap_e_np_out.npy'''				)
										UpperCAmelCase					    =     ShapEPipeline.from_pretrained('''openai/shap-e'''				)
										UpperCAmelCase					    =     pipe.to(lowerCAmelCase				)
										pipe.set_progress_bar_config(disable=lowerCAmelCase				)

										UpperCAmelCase					    =     torch.Generator(device=lowerCAmelCase				).manual_seed(0				)

										UpperCAmelCase					    =     pipe(
										    '''a shark'''				,   generator=lowerCAmelCase				,   guidance_scale=15.0				,   num_inference_steps=64				,   frame_size=64				,   output_type='''np'''				,   ).images[0]

										assert images.shape == (20, 64, 64, 3)

										assert_mean_pixel_difference(lowerCAmelCase				,   lowerCAmelCase				)





