


import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


lowerCAmelCase	=							datasets.utils.logging.get_logger(__name__)


@dataclass
class 	_a							(      datasets.BuilderConfig      ):
							_lowercase     :      Optional[datasets.Features]  				=			None
							_lowercase     :      str  				=			"utf-8"
							_lowercase     :      Optional[str]  				=			None
							_lowercase     :      Optional[str]  				=			None
							_lowercase     :      bool  				=			True  # deprecated
							_lowercase     :      Optional[int]  				=			None  # deprecated
							_lowercase     :      int  				=			10 << 20  # 10MB
							_lowercase     :      Optional[bool]  				=			None


class 	_a							(      datasets.ArrowBasedBuilder      ):
							_lowercase     :      Tuple  				=			JsonConfig

							def 				lowerCamelCase_			(							self:     List[Any]							)      ->       Dict:





									"""simple docstring"""







									if self.config.block_size is not None:
											logger.warning('''The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead'''							)
											lowercase__						=				self.config.block_size
									if self.config.use_threads is not True:
											logger.warning(
											    '''The JSON loader parameter `use_threads` is deprecated and doesn\'t have any effect anymore.'''							)
									if self.config.newlines_in_values is not None:
											raise ValueError('''The JSON loader parameter `newlines_in_values` is no longer supported'''							)
									return datasets.DatasetInfo(features=self.config.features							)

							def 				lowerCamelCase_			(							self:     Any   ,						UpperCamelCase_:     Tuple							)      ->       Union[str, Any]:





									"""simple docstring"""







									if not self.config.data_files:
											raise ValueError(f'At least one data file must be specified, but got data_files={self.config.data_files}'							)
									lowercase__						=				dl_manager.download_and_extract(self.config.data_files							)
									if isinstance(UpperCamelCase_   ,						(str, list, tuple)							):
											lowercase__						=				data_files
											if isinstance(UpperCamelCase_   ,						UpperCamelCase_							):
													lowercase__						=				[files]
											lowercase__						=				[dl_manager.iter_files(UpperCamelCase_							) for file in files]
											return [datasets.SplitGenerator(name=datasets.Split.TRAIN   ,						gen_kwargs={'''files''': files}							)]
									lowercase__						=				[]
									for split_name, files in data_files.items():
											if isinstance(UpperCamelCase_   ,						UpperCamelCase_							):
													lowercase__						=				[files]
											lowercase__						=				[dl_manager.iter_files(UpperCamelCase_							) for file in files]
											splits.append(datasets.SplitGenerator(name=UpperCamelCase_   ,						gen_kwargs={'''files''': files}							)							)
									return splits

							def 				lowerCamelCase_			(							self:     Optional[Any]   ,						UpperCamelCase_:     pa.Table							)      ->       pa.Table:





									"""simple docstring"""







									if self.config.features is not None:
											# adding missing columns
											for column_name in set(self.config.features							) - set(pa_table.column_names							):
													lowercase__						=				self.config.features.arrow_schema.field(UpperCamelCase_							).type
													lowercase__						=				pa_table.append_column(UpperCamelCase_   ,						pa.array([None] * len(UpperCamelCase_							)   ,						type=UpperCamelCase_							)							)
											# more expensive cast to support nested structures with keys in a different order
											# allows str <-> int/float or str to Audio for example
											lowercase__						=				table_cast(UpperCamelCase_   ,						self.config.features.arrow_schema							)
									return pa_table


							def 				lowerCamelCase_			(							self:     Optional[Any]   ,						UpperCamelCase_:     Optional[Any]							)      ->       int:





									"""simple docstring"""







									for file_idx, file in enumerate(itertools.chain.from_iterable(UpperCamelCase_							)							):
											# If the file is one json object and if we need to look at the list of items in one specific field
											if self.config.field is not None:
													with open(UpperCamelCase_   ,						encoding=self.config.encoding   ,						errors=self.config.encoding_errors							) as f:
															lowercase__						=				json.load(UpperCamelCase_							)

													# We keep only the field we are interested in
													lowercase__						=				dataset[self.config.field]

													# We accept two format: a list of dicts or a dict of lists
													if isinstance(UpperCamelCase_   ,						(list, tuple)							):
															lowercase__						=				set().union(*[row.keys() for row in dataset]							)
															lowercase__						=				{col: [row.get(UpperCamelCase_							) for row in dataset] for col in keys}
													else:
															lowercase__						=				dataset
													lowercase__						=				pa.Table.from_pydict(UpperCamelCase_							)
													yield file_idx, self._cast_table(UpperCamelCase_							)

											# If the file has one json object per line
											else:
													with open(UpperCamelCase_   ,						'''rb'''							) as f:
															lowercase__						=				0
															# Use block_size equal to the chunk size divided by 32 to leverage multithreading
															# Set a default minimum value of 16kB if the chunk size is really small
															lowercase__						=				max(self.config.chunksize // 32   ,						16 << 10							)
															lowercase__						=				(
															    self.config.encoding_errors if self.config.encoding_errors is not None else '''strict'''
															)
															while True:
																	lowercase__						=				f.read(self.config.chunksize							)
																	if not batch:
																			break
																	# Finish current line
																	try:
																			batch += f.readline()
																	except (AttributeError, io.UnsupportedOperation):
																			batch += readline(UpperCamelCase_							)
																	# PyArrow only accepts utf-8 encoded bytes
																	if self.config.encoding != "utf-8":
																			lowercase__						=				batch.decode(self.config.encoding   ,						errors=UpperCamelCase_							).encode('''utf-8'''							)
																	try:
																			while True:
																					try:
																							lowercase__						=				paj.read_json(
																							    io.BytesIO(UpperCamelCase_							)   ,						read_options=paj.ReadOptions(block_size=UpperCamelCase_							)							)
																							break
																					except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
																							if (
																							    isinstance(UpperCamelCase_   ,						pa.ArrowInvalid							)
																							    and "straddling" not in str(UpperCamelCase_							)
																							    or block_size > len(UpperCamelCase_							)
																							):
																									raise
																							else:
																									# Increase the block size in case it was too small.
																									# The block size will be reset for the next file.
																									logger.debug(
																									    f'Batch of {len(UpperCamelCase_							)} bytes couldn\'t be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}.'							)
																									block_size *= 2
																	except pa.ArrowInvalid as e:
																			try:
																					with open(
																					    UpperCamelCase_   ,						encoding=self.config.encoding   ,						errors=self.config.encoding_errors							) as f:
																							lowercase__						=				json.load(UpperCamelCase_							)
																			except json.JSONDecodeError:
																					logger.error(f'Failed to read file \'{file}\' with error {type(UpperCamelCase_							)}: {e}'							)
																					raise e
																			# If possible, parse the file as a list of json objects and exit the loop
																			if isinstance(UpperCamelCase_   ,						UpperCamelCase_							):  # list is the only sequence type supported in JSON
																					try:
																							lowercase__						=				set().union(*[row.keys() for row in dataset]							)
																							lowercase__						=				{col: [row.get(UpperCamelCase_							) for row in dataset] for col in keys}
																							lowercase__						=				pa.Table.from_pydict(UpperCamelCase_							)
																					except (pa.ArrowInvalid, AttributeError) as e:
																							logger.error(f'Failed to read file \'{file}\' with error {type(UpperCamelCase_							)}: {e}'							)
																							raise ValueError(f'Not able to read records in the JSON file at {file}.'							) from None
																					yield file_idx, self._cast_table(UpperCamelCase_							)
																					break
																			else:
																					logger.error(f'Failed to read file \'{file}\' with error {type(UpperCamelCase_							)}: {e}'							)
																					raise ValueError(
																					    f'Not able to read records in the JSON file at {file}. '
																					    f'You should probably indicate the field of the JSON file containing your records. '
																					    f'This JSON file contain the following fields: {str(list(dataset.keys()							)							)}. '
																					    f'Select the correct one and provide it as `field=\'XXX\'` to the dataset loading method. '							) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																	yield (file_idx, batch_idx), self._cast_table(UpperCamelCase_							)
																	batch_idx += 1






import gc
import unittest

import numpy as np
import torch
from transformers import CLIPTextConfig, CLIPTextModel, XLMRobertaTokenizer

from diffusers import AltDiffusionPipeline, AutoencoderKL, DDIMScheduler, PNDMScheduler, UNetaDConditionModel
from diffusers.pipelines.alt_diffusion.modeling_roberta_series import (
    RobertaSeriesConfig,
    RobertaSeriesModelWithTransformation,
)
from diffusers.utils import slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu

from ..pipeline_params import TEXT_TO_IMAGE_BATCH_PARAMS, TEXT_TO_IMAGE_IMAGE_PARAMS, TEXT_TO_IMAGE_PARAMS
from ..test_pipelines_common import PipelineKarrasSchedulerTesterMixin, PipelineLatentTesterMixin, PipelineTesterMixin


enable_full_determinism()


class 	_a							(      UpperCamelCase__					,     UpperCamelCase__					,     UpperCamelCase__					,     unittest.TestCase      ):
							_lowercase     :      int  				=			AltDiffusionPipeline
							_lowercase     :      Tuple  				=			TEXT_TO_IMAGE_PARAMS
							_lowercase     :      List[str]  				=			TEXT_TO_IMAGE_BATCH_PARAMS
							_lowercase     :      Union[str, Any]  				=			TEXT_TO_IMAGE_IMAGE_PARAMS
							_lowercase     :      int  				=			TEXT_TO_IMAGE_IMAGE_PARAMS

							def 				lowerCamelCase_			(							self:     Optional[int]							)      ->       Union[str, Any]:





									"""simple docstring"""







									torch.manual_seed(0							)
									lowercase__						=				UNetaDConditionModel(
									    block_out_channels=(32, 64)   ,						layers_per_block=2   ,						sample_size=32   ,						in_channels=4   ,						out_channels=4   ,						down_block_types=('''DownBlock2D''', '''CrossAttnDownBlock2D''')   ,						up_block_types=('''CrossAttnUpBlock2D''', '''UpBlock2D''')   ,						cross_attention_dim=32   ,						)
									lowercase__						=				DDIMScheduler(
									    beta_start=0.00085   ,						beta_end=0.012   ,						beta_schedule='''scaled_linear'''   ,						clip_sample=UpperCamelCase_   ,						set_alpha_to_one=UpperCamelCase_   ,						)
									torch.manual_seed(0							)
									lowercase__						=				AutoencoderKL(
									    block_out_channels=[32, 64]   ,						in_channels=3   ,						out_channels=3   ,						down_block_types=['''DownEncoderBlock2D''', '''DownEncoderBlock2D''']   ,						up_block_types=['''UpDecoderBlock2D''', '''UpDecoderBlock2D''']   ,						latent_channels=4   ,						)

									# TODO: address the non-deterministic text encoder (fails for save-load tests)
									# torch.manual_seed(0)
									# text_encoder_config = RobertaSeriesConfig(
									#     hidden_size=32,
									#     project_dim=32,
									#     intermediate_size=37,
									#     layer_norm_eps=1e-05,
									#     num_attention_heads=4,
									#     num_hidden_layers=5,
									#     vocab_size=5002,
									# )
									# text_encoder = RobertaSeriesModelWithTransformation(text_encoder_config)

									torch.manual_seed(0							)
									lowercase__						=				CLIPTextConfig(
									    bos_token_id=0   ,						eos_token_id=2   ,						hidden_size=32   ,						projection_dim=32   ,						intermediate_size=37   ,						layer_norm_eps=1E-0_5   ,						num_attention_heads=4   ,						num_hidden_layers=5   ,						pad_token_id=1   ,						vocab_size=5_002   ,						)
									lowercase__						=				CLIPTextModel(UpperCamelCase_							)

									lowercase__						=				XLMRobertaTokenizer.from_pretrained('''hf-internal-testing/tiny-xlm-roberta'''							)
									lowercase__						=				77

									lowercase__						=				{
									    '''unet''': unet,
									    '''scheduler''': scheduler,
									    '''vae''': vae,
									    '''text_encoder''': text_encoder,
									    '''tokenizer''': tokenizer,
									    '''safety_checker''': None,
									    '''feature_extractor''': None,
									}
									return components

							def 				lowerCamelCase_			(							self:     List[str]   ,						UpperCamelCase_:     Optional[int]   ,						UpperCamelCase_:     str=0							)      ->       Union[str, Any]:





									"""simple docstring"""







									if str(UpperCamelCase_							).startswith('''mps'''							):
											lowercase__						=				torch.manual_seed(UpperCamelCase_							)
									else:
											lowercase__						=				torch.Generator(device=UpperCamelCase_							).manual_seed(UpperCamelCase_							)
									lowercase__						=				{
									    '''prompt''': '''A painting of a squirrel eating a burger''',
									    '''generator''': generator,
									    '''num_inference_steps''': 2,
									    '''guidance_scale''': 6.0,
									    '''output_type''': '''numpy''',
									}
									return inputs

							def 				lowerCamelCase_			(							self:     Optional[Any]							)      ->       Dict:





									"""simple docstring"""







									super().test_attention_slicing_forward_pass(expected_max_diff=3E-3							)

							def 				lowerCamelCase_			(							self:     Dict							)      ->       str:





									"""simple docstring"""







									super().test_inference_batch_single_identical(expected_max_diff=3E-3							)

							def 				lowerCamelCase_			(							self:     Optional[int]							)      ->       Any:





									"""simple docstring"""







									lowercase__						=				'''cpu'''  # ensure determinism for the device-dependent torch.Generator

									lowercase__						=				self.get_dummy_components()
									torch.manual_seed(0							)
									lowercase__						=				RobertaSeriesConfig(
									    hidden_size=32   ,						project_dim=32   ,						intermediate_size=37   ,						layer_norm_eps=1E-0_5   ,						num_attention_heads=4   ,						num_hidden_layers=5   ,						vocab_size=5_002   ,						)
									# TODO: remove after fixing the non-deterministic text encoder
									lowercase__						=				RobertaSeriesModelWithTransformation(UpperCamelCase_							)
									lowercase__						=				text_encoder

									lowercase__						=				AltDiffusionPipeline(**UpperCamelCase_							)
									lowercase__						=				alt_pipe.to(UpperCamelCase_							)
									alt_pipe.set_progress_bar_config(disable=UpperCamelCase_							)

									lowercase__						=				self.get_dummy_inputs(UpperCamelCase_							)
									lowercase__						=				'''A photo of an astronaut'''
									lowercase__						=				alt_pipe(**UpperCamelCase_							)
									lowercase__						=				output.images
									lowercase__						=				image[0, -3:, -3:, -1]

									assert image.shape == (1, 64, 64, 3)
									lowercase__						=				np.array(
									    [0.5748162, 0.60447145, 0.48821217, 0.50100636, 0.5431185, 0.45763683, 0.49657696, 0.48132733, 0.47573093]							)

									assert np.abs(image_slice.flatten() - expected_slice							).max() < 1E-2


							def 				lowerCamelCase_			(							self:     Optional[Any]							)      ->       Tuple:





									"""simple docstring"""







									lowercase__						=				'''cpu'''  # ensure determinism for the device-dependent torch.Generator

									lowercase__						=				self.get_dummy_components()
									lowercase__						=				PNDMScheduler(skip_prk_steps=UpperCamelCase_							)
									torch.manual_seed(0							)
									lowercase__						=				RobertaSeriesConfig(
									    hidden_size=32   ,						project_dim=32   ,						intermediate_size=37   ,						layer_norm_eps=1E-0_5   ,						num_attention_heads=4   ,						num_hidden_layers=5   ,						vocab_size=5_002   ,						)
									# TODO: remove after fixing the non-deterministic text encoder
									lowercase__						=				RobertaSeriesModelWithTransformation(UpperCamelCase_							)
									lowercase__						=				text_encoder
									lowercase__						=				AltDiffusionPipeline(**UpperCamelCase_							)
									lowercase__						=				alt_pipe.to(UpperCamelCase_							)
									alt_pipe.set_progress_bar_config(disable=UpperCamelCase_							)

									lowercase__						=				self.get_dummy_inputs(UpperCamelCase_							)
									lowercase__						=				alt_pipe(**UpperCamelCase_							)
									lowercase__						=				output.images
									lowercase__						=				image[0, -3:, -3:, -1]

									assert image.shape == (1, 64, 64, 3)
									lowercase__						=				np.array(
									    [0.51605093, 0.5707241, 0.47365507, 0.50578886, 0.5633877, 0.4642503, 0.5182081, 0.48763484, 0.49084237]							)

									assert np.abs(image_slice.flatten() - expected_slice							).max() < 1E-2


@slow
@require_torch_gpu
class 	_a							(      unittest.TestCase      ):

							def 				lowerCamelCase_			(							self:     Dict							)      ->       Optional[int]:





									"""simple docstring"""







									super().tearDown()
									gc.collect()
									torch.cuda.empty_cache()

							def 				lowerCamelCase_			(							self:     Union[str, Any]							)      ->       Union[str, Any]:





									"""simple docstring"""







									lowercase__						=				AltDiffusionPipeline.from_pretrained('''BAAI/AltDiffusion'''   ,						safety_checker=UpperCamelCase_							)
									lowercase__						=				alt_pipe.to(UpperCamelCase_							)
									alt_pipe.set_progress_bar_config(disable=UpperCamelCase_							)

									lowercase__						=				'''A painting of a squirrel eating a burger'''
									lowercase__						=				torch.manual_seed(0							)
									lowercase__						=				alt_pipe([prompt]   ,						generator=UpperCamelCase_   ,						guidance_scale=6.0   ,						num_inference_steps=20   ,						output_type='''np'''							)

									lowercase__						=				output.images

									lowercase__						=				image[0, -3:, -3:, -1]

									assert image.shape == (1, 512, 512, 3)
									lowercase__						=				np.array([0.1010, 0.0800, 0.0794, 0.0885, 0.0843, 0.0762, 0.0769, 0.0729, 0.0586]							)

									assert np.abs(image_slice.flatten() - expected_slice							).max() < 1E-2


							def 				lowerCamelCase_			(							self:     Union[str, Any]							)      ->       Dict:





									"""simple docstring"""







									lowercase__						=				DDIMScheduler.from_pretrained('''BAAI/AltDiffusion'''   ,						subfolder='''scheduler'''							)

									lowercase__						=				AltDiffusionPipeline.from_pretrained('''BAAI/AltDiffusion'''   ,						scheduler=UpperCamelCase_   ,						safety_checker=UpperCamelCase_							)
									lowercase__						=				alt_pipe.to(UpperCamelCase_							)
									alt_pipe.set_progress_bar_config(disable=UpperCamelCase_							)

									lowercase__						=				'''A painting of a squirrel eating a burger'''
									lowercase__						=				torch.manual_seed(0							)

									lowercase__						=				alt_pipe([prompt]   ,						generator=UpperCamelCase_   ,						num_inference_steps=2   ,						output_type='''numpy'''							)
									lowercase__						=				output.images

									lowercase__						=				image[0, -3:, -3:, -1]

									assert image.shape == (1, 512, 512, 3)
									lowercase__						=				np.array([0.4019, 0.4052, 0.3810, 0.4119, 0.3916, 0.3982, 0.4651, 0.4195, 0.5323]							)

									assert np.abs(image_slice.flatten() - expected_slice							).max() < 1E-2


