


'''simple docstring'''



import os
import re
import shutil
from argparse import ArgumentParser, Namespace

from datasets.commands import BaseDatasetsCLICommand
from datasets.utils.logging import get_logger


_lowercase      :      List[Any]            =					"""<<<<<<< This should probably be modified because it mentions: """

_lowercase      :      List[str]            =					"""=======
>>>>>>>
"""

_lowercase      :      Any            =					[
    """TextEncoderConfig""",
    """ByteTextEncoder""",
    """SubwordTextEncoder""",
    """encoder_config""",
    """maybe_build_from_corpus""",
    """manual_dir""",
]

_lowercase      :      Optional[int]            =					[
    # (pattern, replacement)
    # Order is important here for some replacements
    (r"""tfds\.core""", r"""datasets"""),
    (r"""tf\.io\.gfile\.GFile""", r"""open"""),
    (r"""tf\.([\w\d]+)""", r"""datasets.Value('\1')"""),
    (r"""tfds\.features\.Text\(\)""", r"""datasets.Value('string')"""),
    (r"""tfds\.features\.Text\(""", r"""datasets.Value('string'),"""),
    (r"""features\s*=\s*tfds.features.FeaturesDict\(""", r"""features=datasets.Features("""),
    (r"""tfds\.features\.FeaturesDict\(""", r"""dict("""),
    (r"""The TensorFlow Datasets Authors""", r"""The TensorFlow Datasets Authors and the HuggingFace Datasets Authors"""),
    (r"""tfds\.""", r"""datasets."""),
    (r"""dl_manager\.manual_dir""", r"""self.config.data_dir"""),
    (r"""self\.builder_config""", r"""self.config"""),
]


def 				lowerCamelCase__				(	A       :      Namespace					):




						'''simple docstring'''






						return ConvertCommand(args.tfds_path				,       args.datasets_directory					)






class 	UpperCamelCase__(       lowerCAmelCase	):




				@staticmethod
				def 					a__(		lowerCAmelCase      :			ArgumentParser				)->					str:

										"""simple docstring"""




										UpperCAmelCase					    =     parser.add_parser(
										    '''convert'''				,   help='''Convert a TensorFlow Datasets dataset to a HuggingFace Datasets dataset.'''				,   )
										train_parser.add_argument(
										    '''--tfds_path'''				,   type=lowerCAmelCase				,   required=lowerCAmelCase				,   help='''Path to a TensorFlow Datasets folder to convert or a single tfds file to convert.'''				,   )
										train_parser.add_argument(
										    '''--datasets_directory'''				,   type=lowerCAmelCase				,   required=lowerCAmelCase				,   help='''Path to the HuggingFace Datasets folder.'''				)
										train_parser.set_defaults(func=lowerCAmelCase				)




				def __init__(		self      :			List[str]				,   lowerCAmelCase      :			str				,   lowerCAmelCase      :			str				,   *lowerCAmelCase      :			Tuple				)->					List[str]:

										"""simple docstring"""




										UpperCAmelCase					    =     get_logger('''datasets-cli/converting'''				)

										UpperCAmelCase					    =     tfds_path
										UpperCAmelCase					    =     datasets_directory





				def 					a__(		self      :			Dict				)->					int:

										"""simple docstring"""




										if os.path.isdir(self._tfds_path				):
																UpperCAmelCase					    =     os.path.abspath(self._tfds_path				)
										elif os.path.isfile(self._tfds_path				):
																UpperCAmelCase					    =     os.path.dirname(self._tfds_path				)
										else:
																raise ValueError('''--tfds_path is neither a directory nor a file. Please check path.'''				)

										UpperCAmelCase					    =     os.path.abspath(self._datasets_directory				)

										self._logger.info(F"""Converting datasets from {abs_tfds_path} to {abs_datasets_path}"""				)

										UpperCAmelCase					    =     []
										UpperCAmelCase					    =     []
										UpperCAmelCase					    =     {}

										if os.path.isdir(self._tfds_path				):
																UpperCAmelCase					    =     os.listdir(lowerCAmelCase				)
										else:
																UpperCAmelCase					    =     [os.path.basename(self._tfds_path				)]

										for f_name in file_names:
																self._logger.info(F"""Looking at file {f_name}"""				)
																UpperCAmelCase					    =     os.path.join(lowerCAmelCase				,   lowerCAmelCase				)
																UpperCAmelCase					    =     os.path.join(lowerCAmelCase				,   lowerCAmelCase				)

																if not os.path.isfile(lowerCAmelCase				) or "__init__" in f_name or "_test" in f_name or ".py" not in f_name:
																						self._logger.info('''Skipping file'''				)
																						continue

																with open(lowerCAmelCase				,   encoding='''utf-8'''				) as f:
																						UpperCAmelCase					    =     f.readlines()

																UpperCAmelCase					    =     []
																UpperCAmelCase					    =     False
																UpperCAmelCase					    =     False
																UpperCAmelCase					    =     []
																for line in lines:
																						UpperCAmelCase					    =     line

																						# Convert imports
																						if "import tensorflow.compat.v2 as tf" in out_line:
																												continue
																						elif "@tfds.core" in out_line:
																												continue
																						elif "builder=self" in out_line:
																												continue
																						elif "import tensorflow_datasets.public_api as tfds" in out_line:
																												UpperCAmelCase					    =     '''import datasets\n'''
																						elif "import tensorflow" in out_line:
																												# order is important here
																												UpperCAmelCase					    =     ''''''
																												continue
																						elif "from absl import logging" in out_line:
																												UpperCAmelCase					    =     '''from datasets import logging\n'''
																						elif "getLogger" in out_line:
																												UpperCAmelCase					    =     out_line.replace('''getLogger'''				,   '''get_logger'''				)
																						elif any(expression in out_line for expression in TO_HIGHLIGHT				):
																												UpperCAmelCase					    =     True
																												UpperCAmelCase					    =     list(filter(lambda lowerCAmelCase				: e in out_line				,   lowerCAmelCase				)				)
																												out_lines.append(HIGHLIGHT_MESSAGE_PRE + str(lowerCAmelCase				) + '''\n'''				)
																												out_lines.append(lowerCAmelCase				)
																												out_lines.append(lowerCAmelCase				)
																												continue
																						else:
																												for pattern, replacement in TO_CONVERT:
																																		UpperCAmelCase					    =     re.sub(lowerCAmelCase				,   lowerCAmelCase				,   lowerCAmelCase				)

                # Take care of saving utilities (to later move them together with main script)
																						if "tensorflow_datasets" in out_line:
																												UpperCAmelCase					    =     re.match(R'''from\stensorflow_datasets.*import\s([^\.\r\n]+)'''				,   lowerCAmelCase				)
																												tfds_imports.extend(imp.strip() for imp in match.group(1				).split(''','''				)				)
																												UpperCAmelCase					    =     '''from . import ''' + match.group(1				)

																						# Check we have not forget anything
																						if "tf." in out_line or "tfds." in out_line or "tensorflow_datasets" in out_line:
																												raise ValueError(F"""Error converting {out_line.strip()}"""				)

																						if "GeneratorBasedBuilder" in out_line or "BeamBasedBuilder" in out_line:
																												UpperCAmelCase					    =     True
																						out_lines.append(lowerCAmelCase				)

																if is_builder or "wmt" in f_name:
																						# We create a new directory for each dataset
																						UpperCAmelCase					    =     f_name.replace('''.py'''				,   ''''''				)
																						UpperCAmelCase					    =     os.path.join(lowerCAmelCase				,   lowerCAmelCase				)
																						UpperCAmelCase					    =     os.path.join(lowerCAmelCase				,   lowerCAmelCase				)
																						os.makedirs(lowerCAmelCase				,   exist_ok=lowerCAmelCase				)
																						self._logger.info(F"""Adding directory {output_dir}"""				)
																						imports_to_builder_map.update({imp: output_dir for imp in tfds_imports}				)
																else:
																						# Utilities will be moved at the end
																						utils_files.append(lowerCAmelCase				)

																if needs_manual_update:
																						with_manual_update.append(lowerCAmelCase				)

																with open(lowerCAmelCase				,   '''w'''				,   encoding='''utf-8'''				) as f:
																						f.writelines(lowerCAmelCase				)
																self._logger.info(F"""Converted in {output_file}"""				)

										for utils_file in utils_files:
																try:
																						UpperCAmelCase					    =     os.path.basename(lowerCAmelCase				)
																						UpperCAmelCase					    =     imports_to_builder_map[f_name.replace('''.py'''				,   ''''''				)]
																						self._logger.info(F"""Moving {dest_folder} to {utils_file}"""				)
																						shutil.copy(lowerCAmelCase				,   lowerCAmelCase				)
																except KeyError:
																						self._logger.error(F"""Cannot find destination folder for {utils_file}. Please copy manually."""				)

										if with_manual_update:
																for file_path in with_manual_update:
																						self._logger.warning(
																						    F"""You need to manually update file {file_path} to remove configurations using 'TextEncoderConfig'."""				)









'''simple docstring'''



import gc
import unittest

import numpy as np
import torch
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import (
    AutoencoderKL,
    DDIMScheduler,
    StableDiffusionSAGPipeline,
    UNetaDConditionModel,
)
from diffusers.utils import slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu

from ..pipeline_params import TEXT_TO_IMAGE_BATCH_PARAMS, TEXT_TO_IMAGE_IMAGE_PARAMS, TEXT_TO_IMAGE_PARAMS
from ..test_pipelines_common import PipelineLatentTesterMixin, PipelineTesterMixin


enable_full_determinism()






class 	UpperCamelCase__(       lowerCAmelCase						,    lowerCAmelCase						,    unittest.TestCase	):
				__magic_name__    :				List[str]  		=	StableDiffusionSAGPipeline
				__magic_name__    :				str  		=	TEXT_TO_IMAGE_PARAMS
				__magic_name__    :				Any  		=	TEXT_TO_IMAGE_BATCH_PARAMS
				__magic_name__    :				Optional[int]  		=	TEXT_TO_IMAGE_IMAGE_PARAMS
				__magic_name__    :				List[Any]  		=	TEXT_TO_IMAGE_IMAGE_PARAMS
				__magic_name__    :				str  		=	False




				def 					a__(		self      :			Union[str, Any]				)->					Tuple:

										"""simple docstring"""




										torch.manual_seed(0				)
										UpperCAmelCase					    =     UNetaDConditionModel(
										    block_out_channels=(32, 64)				,   layers_per_block=2				,   sample_size=32				,   in_channels=4				,   out_channels=4				,   down_block_types=('''DownBlock2D''', '''CrossAttnDownBlock2D''')				,   up_block_types=('''CrossAttnUpBlock2D''', '''UpBlock2D''')				,   cross_attention_dim=32				,   )
										UpperCAmelCase					    =     DDIMScheduler(
										    beta_start=0.00085				,   beta_end=0.012				,   beta_schedule='''scaled_linear'''				,   clip_sample=lowerCAmelCase				,   set_alpha_to_one=lowerCAmelCase				,   )
										torch.manual_seed(0				)
										UpperCAmelCase					    =     AutoencoderKL(
										    block_out_channels=[32, 64]				,   in_channels=3				,   out_channels=3				,   down_block_types=['''DownEncoderBlock2D''', '''DownEncoderBlock2D''']				,   up_block_types=['''UpDecoderBlock2D''', '''UpDecoderBlock2D''']				,   latent_channels=4				,   )
										torch.manual_seed(0				)
										UpperCAmelCase					    =     CLIPTextConfig(
										    bos_token_id=0				,   eos_token_id=2				,   hidden_size=32				,   intermediate_size=37				,   layer_norm_eps=1E-05				,   num_attention_heads=4				,   num_hidden_layers=5				,   pad_token_id=1				,   vocab_size=1000				,   )
										UpperCAmelCase					    =     CLIPTextModel(lowerCAmelCase				)
										UpperCAmelCase					    =     CLIPTokenizer.from_pretrained('''hf-internal-testing/tiny-random-clip'''				)

										UpperCAmelCase					    =     {
										    '''unet''': unet,
										    '''scheduler''': scheduler,
										    '''vae''': vae,
										    '''text_encoder''': text_encoder,
										    '''tokenizer''': tokenizer,
										    '''safety_checker''': None,
										    '''feature_extractor''': None,
										}
										return components




				def 					a__(		self      :			Optional[Any]				,   lowerCAmelCase      :			str				,   lowerCAmelCase      :			Tuple=0				)->					str:

										"""simple docstring"""




										if str(lowerCAmelCase				).startswith('''mps'''				):
																UpperCAmelCase					    =     torch.manual_seed(lowerCAmelCase				)
										else:
																UpperCAmelCase					    =     torch.Generator(device=lowerCAmelCase				).manual_seed(lowerCAmelCase				)
										UpperCAmelCase					    =     {
										    '''prompt''': '''.''',
										    '''generator''': generator,
										    '''num_inference_steps''': 2,
										    '''guidance_scale''': 1.0,
										    '''sag_scale''': 1.0,
										    '''output_type''': '''numpy''',
										}
										return inputs





				def 					a__(		self      :			Any				)->					List[str]:

										"""simple docstring"""




										super().test_inference_batch_single_identical(expected_max_diff=3E-3				)







@slow
@require_torch_gpu
class 	UpperCamelCase__(       unittest.TestCase	):




				def 					a__(		self      :			Union[str, Any]				)->					Any:

										"""simple docstring"""




										super().tearDown()
										gc.collect()
										torch.cuda.empty_cache()




				def 					a__(		self      :			Union[str, Any]				)->					Tuple:

										"""simple docstring"""




										UpperCAmelCase					    =     StableDiffusionSAGPipeline.from_pretrained('''CompVis/stable-diffusion-v1-4'''				)
										UpperCAmelCase					    =     sag_pipe.to(lowerCAmelCase				)
										sag_pipe.set_progress_bar_config(disable=lowerCAmelCase				)

										UpperCAmelCase					    =     '''.'''
										UpperCAmelCase					    =     torch.manual_seed(0				)
										UpperCAmelCase					    =     sag_pipe(
										    [prompt]				,   generator=lowerCAmelCase				,   guidance_scale=7.5				,   sag_scale=1.0				,   num_inference_steps=20				,   output_type='''np'''				)

										UpperCAmelCase					    =     output.images

										UpperCAmelCase					    =     image[0, -3:, -3:, -1]

										assert image.shape == (1, 512, 512, 3)
										UpperCAmelCase					    =     np.array([0.1568, 0.1738, 0.1695, 0.1693, 0.1507, 0.1705, 0.1547, 0.1751, 0.1949]				)

										assert np.abs(image_slice.flatten() - expected_slice				).max() < 5E-2




				def 					a__(		self      :			int				)->					Union[str, Any]:

										"""simple docstring"""




										UpperCAmelCase					    =     StableDiffusionSAGPipeline.from_pretrained('''stabilityai/stable-diffusion-2-1-base'''				)
										UpperCAmelCase					    =     sag_pipe.to(lowerCAmelCase				)
										sag_pipe.set_progress_bar_config(disable=lowerCAmelCase				)

										UpperCAmelCase					    =     '''.'''
										UpperCAmelCase					    =     torch.manual_seed(0				)
										UpperCAmelCase					    =     sag_pipe(
										    [prompt]				,   generator=lowerCAmelCase				,   guidance_scale=7.5				,   sag_scale=1.0				,   num_inference_steps=20				,   output_type='''np'''				)

										UpperCAmelCase					    =     output.images

										UpperCAmelCase					    =     image[0, -3:, -3:, -1]

										assert image.shape == (1, 512, 512, 3)
										UpperCAmelCase					    =     np.array([0.3459, 0.2876, 0.2537, 0.3002, 0.2671, 0.2160, 0.3026, 0.2262, 0.2371]				)

										assert np.abs(image_slice.flatten() - expected_slice				).max() < 5E-2





				def 					a__(		self      :			Optional[Any]				)->					Union[str, Any]:

										"""simple docstring"""




										UpperCAmelCase					    =     StableDiffusionSAGPipeline.from_pretrained('''stabilityai/stable-diffusion-2-1-base'''				)
										UpperCAmelCase					    =     sag_pipe.to(lowerCAmelCase				)
										sag_pipe.set_progress_bar_config(disable=lowerCAmelCase				)

										UpperCAmelCase					    =     '''.'''
										UpperCAmelCase					    =     torch.manual_seed(0				)
										UpperCAmelCase					    =     sag_pipe(
										    [prompt]				,   width=768				,   height=512				,   generator=lowerCAmelCase				,   guidance_scale=7.5				,   sag_scale=1.0				,   num_inference_steps=20				,   output_type='''np'''				,   )

										UpperCAmelCase					    =     output.images

										assert image.shape == (1, 512, 768, 3)





