


import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
			import torch


if TYPE_CHECKING:
			from transformers.pipelines.conversational import Conversation


lowerCAmelCase	=							logging.get_logger(__name__)
lowerCAmelCase	=							{'vocab_file': 'spiece.model'}

lowerCAmelCase	=							{
    'vocab_file': {
        'AI-Sweden/gpt-sw3-126m': 'https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-350m': 'https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-1.6b': 'https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-6.7b': 'https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-20b': 'https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model',
    }
}

lowerCAmelCase	=							{
    'AI-Sweden/gpt-sw3-126m': 2048,
    'AI-Sweden/gpt-sw3-350m': 2048,
    'AI-Sweden/gpt-sw3-1.6b': 2048,
    'AI-Sweden/gpt-sw3-6.7b': 2048,
    'AI-Sweden/gpt-sw3-20b': 2048,
}


class 	_a							(      __lowercase      ):
							_lowercase     :      str  				=			VOCAB_FILES_NAMES
							_lowercase     :      Union[str, Any]  				=			PRETRAINED_VOCAB_FILES_MAP
							_lowercase     :      Optional[int]  				=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							_lowercase     :      Optional[Any]  				=			["input_ids", "attention_mask"]

							def __init__(							self:     Optional[int]   ,						UpperCamelCase_:     Optional[int]   ,						UpperCamelCase_:     Any=False   ,						UpperCamelCase_:     List[str]=False   ,						UpperCamelCase_:     Optional[Any]=False   ,						UpperCamelCase_:     List[str]=None   ,						UpperCamelCase_:     Union[str, Any]=None   ,						UpperCamelCase_:     Optional[Any]=None   ,						UpperCamelCase_:     Any=None   ,						UpperCamelCase_:     Any = None   ,						**UpperCamelCase_:     List[Any]   ,						)      ->       None:





									"""simple docstring"""







									lowercase__						=				{} if sp_model_kwargs is None else sp_model_kwargs

									lowercase__						=				kwargs.get('''name_or_path'''							)
									if name_or_path is None:
											logger.warning(
											    '''name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,'''
											    ''' you are testing the model, this can safely be ignored'''							)
											lowercase__						=				'''None'''

									# Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
									lowercase__						=				'''<|endoftext|>''' if eos_token is None else eos_token
									lowercase__						=				'''<unk>''' if unk_token is None else unk_token
									if "gpt-sw3-7b" in name_or_path:
											lowercase__						=				unk_token if pad_token is None else pad_token
											lowercase__						=				eos_token if bos_token is None else bos_token
									else:
											lowercase__						=				'''<pad>''' if pad_token is None else pad_token
											lowercase__						=				'''<s>''' if bos_token is None else bos_token

									super().__init__(
									    do_lower_case=_a   ,						remove_space=_a   ,						keep_accents=_a   ,						bos_token=_a   ,						eos_token=_a   ,						unk_token=_a   ,						pad_token=_a   ,						sp_model_kwargs=self.sp_model_kwargs   ,						**_a   ,						)

									lowercase__						=				do_lower_case
									lowercase__						=				remove_space
									lowercase__						=				keep_accents
									lowercase__						=				vocab_file

									lowercase__						=				spm.SentencePieceProcessor(**self.sp_model_kwargs							)
									self.sp_model.Load(_a							)

									# Used for whitespace normalization in input texts
									# fmt : off
									lowercase__						=				{''' ''', ''' ''', ''' ''', ''' ''', ''' ''', '''　''', ''' ''', ''' ''', ''' ''', ''' ''', '''￼''', ''''''}
									# fmt : on

									# Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
									lowercase__						=				re.compile(
									    f'[{"".join(map(_a   ,						list(range(0   ,						9							)							) + list(range(11   ,						32							)							) + list(range(127   ,						160							)							) + [160, 173, 8_203]							)							)}]'							)

							def __getstate__(							self:     Dict							)      ->       List[str]:





									"""simple docstring"""







									lowercase__						=				self.__dict__.copy()
									lowercase__						=				None
									return state

							def __setstate__(							self:     List[str]   ,						UpperCamelCase_:     List[str]							)      ->       Dict:





									"""simple docstring"""







									lowercase__						=				d

									# for backward compatibility
									if not hasattr(self   ,						'''sp_model_kwargs'''							):
											lowercase__						=				{}

									lowercase__						=				spm.SentencePieceProcessor(**self.sp_model_kwargs							)
									self.sp_model.Load(self.vocab_file							)

							@property
							# Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
							def 				lowerCamelCase_			(							self:     str							)      ->       int:





									"""simple docstring"""







									return len(self.sp_model							)

							def 				lowerCamelCase_			(							self:     Optional[Any]   ,						UpperCamelCase_:     Optional[Any]							)      ->       str:





									"""simple docstring"""







									lowercase__						=				self.non_printing_characters_re.sub(''''''   ,						_a							)

									# Normalize whitespaces
									lowercase__						=				''''''.join([char if char not in self.whitespaces else ''' ''' for char in text]							)

									# NFC Unicode normalization
									lowercase__						=				unicodedata.normalize('''NFC'''   ,						_a							)
									return text

							def 				lowerCamelCase_			(							self:     Dict   ,						UpperCamelCase_:     Dict   ,						**UpperCamelCase_:     Tuple							)      ->       List[str]:





									"""simple docstring"""







									lowercase__						=				self.preprocess_text(_a							)
									return self.sp_model.encode(_a   ,						out_type=_a							)

							def 				lowerCamelCase_			(							self:     Any   ,						UpperCamelCase_:     Dict							)      ->       int:





									"""simple docstring"""







									return self.sp_model.PieceToId(_a							)

							def 				lowerCamelCase_			(							self:     Any   ,						UpperCamelCase_:     int							)      ->       str:





									"""simple docstring"""







									return self.sp_model.IdToPiece(_a							)

							@staticmethod
							def 				lowerCamelCase_			(							UpperCamelCase_:     Union[str, Any]							)      ->       str:





									"""simple docstring"""







									return out_string

							def 				lowerCamelCase_			(							self:     str   ,						UpperCamelCase_:     List[Any]							)      ->       str:





									"""simple docstring"""







									lowercase__						=				[]
									lowercase__						=				''''''
									lowercase__						=				False
									for token in tokens:
											# make sure that special tokens are not decoded using sentencepiece model
											if token in self.all_special_tokens:
													# TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
													if not prev_is_special:
															out_string += " "

													out_string += self.sp_model.decode(_a							) + token
													lowercase__						=				True
													lowercase__						=				[]
											else:
													current_sub_tokens.append(_a							)
													lowercase__						=				False
									out_string += self.sp_model.decode(_a							)

									return out_string

							def 				lowerCamelCase_			(							self:     Tuple							)      ->       Dict[str, int]:





									"""simple docstring"""







									lowercase__						=				{self.convert_ids_to_tokens(_a							): i for i in range(self.vocab_size							)}
									vocab.update(self.added_tokens_encoder							)
									return vocab

							def 				lowerCamelCase_			(							self:     List[str]   ,						UpperCamelCase_:     Optional[Any]   ,						UpperCamelCase_:     Any = None							)      ->       Tuple[str]:





									"""simple docstring"""







									if not os.path.isdir(_a							):
											logger.error(f'Vocabulary path ({save_directory}) should be a directory'							)
											return
									lowercase__						=				os.path.join(
									    _a   ,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']							)

									if os.path.abspath(self.vocab_file							) != os.path.abspath(_a							) and os.path.isfile(self.vocab_file							):
											copyfile(self.vocab_file   ,						_a							)
									elif not os.path.isfile(self.vocab_file							):
											with open(_a   ,						'''wb'''							) as fi:
													lowercase__						=				self.sp_model.serialized_model_proto()
													fi.write(_a							)

									return (out_vocab_file,)

							def 				lowerCamelCase_			(							self:     Optional[int]   ,						UpperCamelCase_:     Any   ,						UpperCamelCase_:     List[Any] = False							)      ->       Union[List[int], List[List[int]], "torch.Tensor"]:





									"""simple docstring"""







									if isinstance(_a   ,						_a							):
											lowercase__						=				self.preprocess_text(_a							)
											lowercase__						=				self.sp_model.encode(_a							)
									else:
											lowercase__						=				[self.preprocess_text(_a							) for t in text]
											lowercase__						=				self.sp_model.encode(_a							)

									if return_tensors is True or return_tensors == "pt":
											lowercase__						=				torch.tensor(_a							)

									return token_ids

							def 				lowerCamelCase_			(							self:     Optional[Any]   ,						UpperCamelCase_:     int							)      ->       str:





									"""simple docstring"""







									return self.sp_model.decode(_a							)


							def 				lowerCamelCase_			(							self:     str   ,						UpperCamelCase_:     Optional[int]							)      ->       List[int]:





									"""simple docstring"""







									lowercase__						=				[f'User: {text}' if is_user else f'Bot: {text}' for is_user, text in conversation.iter_texts()]
									lowercase__						=				(
									    f'{self.eos_token}{self.bos_token}' + f'{self.bos_token}'.join(_a							) + f'{self.bos_token}Bot:'
									)
									return self.encode(text=_a							)






import gc
import unittest

import numpy as np
import torch
from transformers import CLIPTextConfig, CLIPTextModel, XLMRobertaTokenizer

from diffusers import AltDiffusionPipeline, AutoencoderKL, DDIMScheduler, PNDMScheduler, UNetaDConditionModel
from diffusers.pipelines.alt_diffusion.modeling_roberta_series import (
    RobertaSeriesConfig,
    RobertaSeriesModelWithTransformation,
)
from diffusers.utils import slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu

from ..pipeline_params import TEXT_TO_IMAGE_BATCH_PARAMS, TEXT_TO_IMAGE_IMAGE_PARAMS, TEXT_TO_IMAGE_PARAMS
from ..test_pipelines_common import PipelineKarrasSchedulerTesterMixin, PipelineLatentTesterMixin, PipelineTesterMixin


enable_full_determinism()


class 	_a							(      UpperCamelCase__					,     UpperCamelCase__					,     UpperCamelCase__					,     unittest.TestCase      ):
							_lowercase     :      int  				=			AltDiffusionPipeline
							_lowercase     :      Tuple  				=			TEXT_TO_IMAGE_PARAMS
							_lowercase     :      List[str]  				=			TEXT_TO_IMAGE_BATCH_PARAMS
							_lowercase     :      Union[str, Any]  				=			TEXT_TO_IMAGE_IMAGE_PARAMS
							_lowercase     :      int  				=			TEXT_TO_IMAGE_IMAGE_PARAMS

							def 				lowerCamelCase_			(							self:     Optional[int]							)      ->       Union[str, Any]:





									"""simple docstring"""







									torch.manual_seed(0							)
									lowercase__						=				UNetaDConditionModel(
									    block_out_channels=(32, 64)   ,						layers_per_block=2   ,						sample_size=32   ,						in_channels=4   ,						out_channels=4   ,						down_block_types=('''DownBlock2D''', '''CrossAttnDownBlock2D''')   ,						up_block_types=('''CrossAttnUpBlock2D''', '''UpBlock2D''')   ,						cross_attention_dim=32   ,						)
									lowercase__						=				DDIMScheduler(
									    beta_start=0.00085   ,						beta_end=0.012   ,						beta_schedule='''scaled_linear'''   ,						clip_sample=UpperCamelCase_   ,						set_alpha_to_one=UpperCamelCase_   ,						)
									torch.manual_seed(0							)
									lowercase__						=				AutoencoderKL(
									    block_out_channels=[32, 64]   ,						in_channels=3   ,						out_channels=3   ,						down_block_types=['''DownEncoderBlock2D''', '''DownEncoderBlock2D''']   ,						up_block_types=['''UpDecoderBlock2D''', '''UpDecoderBlock2D''']   ,						latent_channels=4   ,						)

									# TODO: address the non-deterministic text encoder (fails for save-load tests)
									# torch.manual_seed(0)
									# text_encoder_config = RobertaSeriesConfig(
									#     hidden_size=32,
									#     project_dim=32,
									#     intermediate_size=37,
									#     layer_norm_eps=1e-05,
									#     num_attention_heads=4,
									#     num_hidden_layers=5,
									#     vocab_size=5002,
									# )
									# text_encoder = RobertaSeriesModelWithTransformation(text_encoder_config)

									torch.manual_seed(0							)
									lowercase__						=				CLIPTextConfig(
									    bos_token_id=0   ,						eos_token_id=2   ,						hidden_size=32   ,						projection_dim=32   ,						intermediate_size=37   ,						layer_norm_eps=1E-0_5   ,						num_attention_heads=4   ,						num_hidden_layers=5   ,						pad_token_id=1   ,						vocab_size=5_002   ,						)
									lowercase__						=				CLIPTextModel(UpperCamelCase_							)

									lowercase__						=				XLMRobertaTokenizer.from_pretrained('''hf-internal-testing/tiny-xlm-roberta'''							)
									lowercase__						=				77

									lowercase__						=				{
									    '''unet''': unet,
									    '''scheduler''': scheduler,
									    '''vae''': vae,
									    '''text_encoder''': text_encoder,
									    '''tokenizer''': tokenizer,
									    '''safety_checker''': None,
									    '''feature_extractor''': None,
									}
									return components

							def 				lowerCamelCase_			(							self:     List[str]   ,						UpperCamelCase_:     Optional[int]   ,						UpperCamelCase_:     str=0							)      ->       Union[str, Any]:





									"""simple docstring"""







									if str(UpperCamelCase_							).startswith('''mps'''							):
											lowercase__						=				torch.manual_seed(UpperCamelCase_							)
									else:
											lowercase__						=				torch.Generator(device=UpperCamelCase_							).manual_seed(UpperCamelCase_							)
									lowercase__						=				{
									    '''prompt''': '''A painting of a squirrel eating a burger''',
									    '''generator''': generator,
									    '''num_inference_steps''': 2,
									    '''guidance_scale''': 6.0,
									    '''output_type''': '''numpy''',
									}
									return inputs

							def 				lowerCamelCase_			(							self:     Optional[Any]							)      ->       Dict:





									"""simple docstring"""







									super().test_attention_slicing_forward_pass(expected_max_diff=3E-3							)

							def 				lowerCamelCase_			(							self:     Dict							)      ->       str:





									"""simple docstring"""







									super().test_inference_batch_single_identical(expected_max_diff=3E-3							)

							def 				lowerCamelCase_			(							self:     Optional[int]							)      ->       Any:





									"""simple docstring"""







									lowercase__						=				'''cpu'''  # ensure determinism for the device-dependent torch.Generator

									lowercase__						=				self.get_dummy_components()
									torch.manual_seed(0							)
									lowercase__						=				RobertaSeriesConfig(
									    hidden_size=32   ,						project_dim=32   ,						intermediate_size=37   ,						layer_norm_eps=1E-0_5   ,						num_attention_heads=4   ,						num_hidden_layers=5   ,						vocab_size=5_002   ,						)
									# TODO: remove after fixing the non-deterministic text encoder
									lowercase__						=				RobertaSeriesModelWithTransformation(UpperCamelCase_							)
									lowercase__						=				text_encoder

									lowercase__						=				AltDiffusionPipeline(**UpperCamelCase_							)
									lowercase__						=				alt_pipe.to(UpperCamelCase_							)
									alt_pipe.set_progress_bar_config(disable=UpperCamelCase_							)

									lowercase__						=				self.get_dummy_inputs(UpperCamelCase_							)
									lowercase__						=				'''A photo of an astronaut'''
									lowercase__						=				alt_pipe(**UpperCamelCase_							)
									lowercase__						=				output.images
									lowercase__						=				image[0, -3:, -3:, -1]

									assert image.shape == (1, 64, 64, 3)
									lowercase__						=				np.array(
									    [0.5748162, 0.60447145, 0.48821217, 0.50100636, 0.5431185, 0.45763683, 0.49657696, 0.48132733, 0.47573093]							)

									assert np.abs(image_slice.flatten() - expected_slice							).max() < 1E-2


							def 				lowerCamelCase_			(							self:     Optional[Any]							)      ->       Tuple:





									"""simple docstring"""







									lowercase__						=				'''cpu'''  # ensure determinism for the device-dependent torch.Generator

									lowercase__						=				self.get_dummy_components()
									lowercase__						=				PNDMScheduler(skip_prk_steps=UpperCamelCase_							)
									torch.manual_seed(0							)
									lowercase__						=				RobertaSeriesConfig(
									    hidden_size=32   ,						project_dim=32   ,						intermediate_size=37   ,						layer_norm_eps=1E-0_5   ,						num_attention_heads=4   ,						num_hidden_layers=5   ,						vocab_size=5_002   ,						)
									# TODO: remove after fixing the non-deterministic text encoder
									lowercase__						=				RobertaSeriesModelWithTransformation(UpperCamelCase_							)
									lowercase__						=				text_encoder
									lowercase__						=				AltDiffusionPipeline(**UpperCamelCase_							)
									lowercase__						=				alt_pipe.to(UpperCamelCase_							)
									alt_pipe.set_progress_bar_config(disable=UpperCamelCase_							)

									lowercase__						=				self.get_dummy_inputs(UpperCamelCase_							)
									lowercase__						=				alt_pipe(**UpperCamelCase_							)
									lowercase__						=				output.images
									lowercase__						=				image[0, -3:, -3:, -1]

									assert image.shape == (1, 64, 64, 3)
									lowercase__						=				np.array(
									    [0.51605093, 0.5707241, 0.47365507, 0.50578886, 0.5633877, 0.4642503, 0.5182081, 0.48763484, 0.49084237]							)

									assert np.abs(image_slice.flatten() - expected_slice							).max() < 1E-2


@slow
@require_torch_gpu
class 	_a							(      unittest.TestCase      ):

							def 				lowerCamelCase_			(							self:     Dict							)      ->       Optional[int]:





									"""simple docstring"""







									super().tearDown()
									gc.collect()
									torch.cuda.empty_cache()

							def 				lowerCamelCase_			(							self:     Union[str, Any]							)      ->       Union[str, Any]:





									"""simple docstring"""







									lowercase__						=				AltDiffusionPipeline.from_pretrained('''BAAI/AltDiffusion'''   ,						safety_checker=UpperCamelCase_							)
									lowercase__						=				alt_pipe.to(UpperCamelCase_							)
									alt_pipe.set_progress_bar_config(disable=UpperCamelCase_							)

									lowercase__						=				'''A painting of a squirrel eating a burger'''
									lowercase__						=				torch.manual_seed(0							)
									lowercase__						=				alt_pipe([prompt]   ,						generator=UpperCamelCase_   ,						guidance_scale=6.0   ,						num_inference_steps=20   ,						output_type='''np'''							)

									lowercase__						=				output.images

									lowercase__						=				image[0, -3:, -3:, -1]

									assert image.shape == (1, 512, 512, 3)
									lowercase__						=				np.array([0.1010, 0.0800, 0.0794, 0.0885, 0.0843, 0.0762, 0.0769, 0.0729, 0.0586]							)

									assert np.abs(image_slice.flatten() - expected_slice							).max() < 1E-2


							def 				lowerCamelCase_			(							self:     Union[str, Any]							)      ->       Dict:





									"""simple docstring"""







									lowercase__						=				DDIMScheduler.from_pretrained('''BAAI/AltDiffusion'''   ,						subfolder='''scheduler'''							)

									lowercase__						=				AltDiffusionPipeline.from_pretrained('''BAAI/AltDiffusion'''   ,						scheduler=UpperCamelCase_   ,						safety_checker=UpperCamelCase_							)
									lowercase__						=				alt_pipe.to(UpperCamelCase_							)
									alt_pipe.set_progress_bar_config(disable=UpperCamelCase_							)

									lowercase__						=				'''A painting of a squirrel eating a burger'''
									lowercase__						=				torch.manual_seed(0							)

									lowercase__						=				alt_pipe([prompt]   ,						generator=UpperCamelCase_   ,						num_inference_steps=2   ,						output_type='''numpy'''							)
									lowercase__						=				output.images

									lowercase__						=				image[0, -3:, -3:, -1]

									assert image.shape == (1, 512, 512, 3)
									lowercase__						=				np.array([0.4019, 0.4052, 0.3810, 0.4119, 0.3916, 0.3982, 0.4651, 0.4195, 0.5323]							)

									assert np.abs(image_slice.flatten() - expected_slice							).max() < 1E-2


