



import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__snake_case    :		int  =logging.get_logger(__name__)
__snake_case    :		int  ={'vocab_file': 'spiece.model'}

__snake_case    :		int  ={
    'vocab_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/spiece.model',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/spiece.model',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/spiece.model',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/spiece.model',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model',
    }
}

__snake_case    :		Union[str, Any]  ={
    'albert-base-v1': 5_1_2,
    'albert-large-v1': 5_1_2,
    'albert-xlarge-v1': 5_1_2,
    'albert-xxlarge-v1': 5_1_2,
    'albert-base-v2': 5_1_2,
    'albert-large-v2': 5_1_2,
    'albert-xlarge-v2': 5_1_2,
    'albert-xxlarge-v2': 5_1_2,
}

__snake_case    :		Tuple  ='▁'



class 	lowerCamelCase__      (							lowerCamelCase__):




		'''simple docstring'''



		snake_case_     =VOCAB_FILES_NAMES
		snake_case_     =PRETRAINED_VOCAB_FILES_MAP
		snake_case_     =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES



		def __init__(self		,__lowerCamelCase		,__lowerCamelCase=True		,__lowerCamelCase=True		,__lowerCamelCase=False		,__lowerCamelCase="[CLS]"		,__lowerCamelCase="[SEP]"		,__lowerCamelCase="<unk>"		,__lowerCamelCase="[SEP]"		,__lowerCamelCase="<pad>"		,__lowerCamelCase="[CLS]"		,__lowerCamelCase="[MASK]"		,__lowerCamelCase = None		,**__lowerCamelCase		,)					->      None:

						"""simple docstring"""






						lowerCAmelCase__					:    str												=					(
						    AddedToken(__lowerCamelCase		,lstrip=__lowerCamelCase		,rstrip=__lowerCamelCase		,normalized=__lowerCamelCase			)
						    if isinstance(__lowerCamelCase		,__lowerCamelCase			)
						    else mask_token
						)

						lowerCAmelCase__					:    Tuple												=					{} if sp_model_kwargs is None else sp_model_kwargs

						super().__init__(
						    do_lower_case=__lowerCamelCase		,remove_space=__lowerCamelCase		,keep_accents=__lowerCamelCase		,bos_token=__lowerCamelCase		,eos_token=__lowerCamelCase		,unk_token=__lowerCamelCase		,sep_token=__lowerCamelCase		,pad_token=__lowerCamelCase		,cls_token=__lowerCamelCase		,mask_token=__lowerCamelCase		,sp_model_kwargs=self.sp_model_kwargs		,**__lowerCamelCase		,)

						lowerCAmelCase__					:    Tuple												=					do_lower_case
						lowerCAmelCase__					:    int												=					remove_space
						lowerCAmelCase__					:    Union[str, Any]												=					keep_accents
						lowerCAmelCase__					:    List[Any]												=					vocab_file

						lowerCAmelCase__					:    Dict												=					spm.SentencePieceProcessor(**self.sp_model_kwargs			)
						self.sp_model.Load(__lowerCamelCase			)



		@property
		def 		lowerCAmelCase__				(self			)					->      List[str]:

						"""simple docstring"""






						return len(self.sp_model			)



		def 		lowerCAmelCase__				(self			)					->      Dict:

						"""simple docstring"""






						lowerCAmelCase__					:    List[Any]												=					{self.convert_ids_to_tokens(__lowerCamelCase			): i for i in range(self.vocab_size			)}
						vocab.update(self.added_tokens_encoder			)
						return vocab



		def __getstate__(self			)					->      int:

						"""simple docstring"""






						lowerCAmelCase__					:    Union[str, Any]												=					self.__dict__.copy()
						lowerCAmelCase__					:    List[Any]												=					None
						return state



		def __setstate__(self		,__lowerCamelCase			)					->      Dict:

						"""simple docstring"""






						lowerCAmelCase__					:    str												=					d

						# for backward compatibility
						if not hasattr(self		,'''sp_model_kwargs'''			):
										lowerCAmelCase__					:    List[str]												=					{}

						lowerCAmelCase__					:    Any												=					spm.SentencePieceProcessor(**self.sp_model_kwargs			)
						self.sp_model.Load(self.vocab_file			)



		def 		lowerCAmelCase__				(self		,__lowerCamelCase			)					->      Union[str, Any]:

						"""simple docstring"""






						if self.remove_space:
										lowerCAmelCase__					:    Dict												=					''' '''.join(inputs.strip().split()			)
						else:
										lowerCAmelCase__					:    int												=					inputs
						lowerCAmelCase__					:    Optional[Any]												=					outputs.replace('''``'''		,'''"'''			).replace('''\'\''''		,'''"'''			)

						if not self.keep_accents:
										lowerCAmelCase__					:    Union[str, Any]												=					unicodedata.normalize('''NFKD'''		,__lowerCamelCase			)
										lowerCAmelCase__					:    List[str]												=					''''''.join([c for c in outputs if not unicodedata.combining(__lowerCamelCase			)]			)
						if self.do_lower_case:
										lowerCAmelCase__					:    Any												=					outputs.lower()

						return outputs



		def 		lowerCAmelCase__				(self		,__lowerCamelCase			)					->      List[str]:

						"""simple docstring"""






						lowerCAmelCase__					:    Union[str, Any]												=					self.preprocess_text(__lowerCamelCase			)
						lowerCAmelCase__					:    Union[str, Any]												=					self.sp_model.encode(__lowerCamelCase		,out_type=__lowerCamelCase			)
						lowerCAmelCase__					:    str												=					[]
						for piece in pieces:
										if len(__lowerCamelCase			) > 1 and piece[-1] == str(''','''			) and piece[-2].isdigit():
														lowerCAmelCase__					:    Dict												=					self.sp_model.EncodeAsPieces(piece[:-1].replace(__lowerCamelCase		,''''''			)			)
														if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																		if len(cur_pieces[0]			) == 1:
																						lowerCAmelCase__					:    Union[str, Any]												=					cur_pieces[1:]
																		else:
																						lowerCAmelCase__					:    Union[str, Any]												=					cur_pieces[0][1:]
														cur_pieces.append(piece[-1]			)
														new_pieces.extend(__lowerCamelCase			)
										else:
														new_pieces.append(__lowerCamelCase			)

						return new_pieces



		def 		lowerCAmelCase__				(self		,__lowerCamelCase			)					->      Optional[int]:

						"""simple docstring"""






						return self.sp_model.PieceToId(__lowerCamelCase			)



		def 		lowerCAmelCase__				(self		,__lowerCamelCase			)					->      Dict:

						"""simple docstring"""






						return self.sp_model.IdToPiece(__lowerCamelCase			)



		def 		lowerCAmelCase__				(self		,__lowerCamelCase			)					->      Dict:

						"""simple docstring"""






						lowerCAmelCase__					:    List[Any]												=					[]
						lowerCAmelCase__					:    Optional[int]												=					''''''
						lowerCAmelCase__					:    str												=					False
						for token in tokens:
										# make sure that special tokens are not decoded using sentencepiece model
										if token in self.all_special_tokens:
														if not prev_is_special:
																		out_string += " "
														out_string += self.sp_model.decode(__lowerCamelCase			) + token
														lowerCAmelCase__					:    Tuple												=					True
														lowerCAmelCase__					:    Any												=					[]
										else:
														current_sub_tokens.append(__lowerCamelCase			)
														lowerCAmelCase__					:    List[Any]												=					False
						out_string += self.sp_model.decode(__lowerCamelCase			)
						return out_string.strip()



		def 		lowerCAmelCase__				(self		,__lowerCamelCase		,__lowerCamelCase = None			)					->      List[int]:

						"""simple docstring"""






						lowerCAmelCase__					:    Any												=					[self.sep_token_id]
						lowerCAmelCase__					:    Optional[int]												=					[self.cls_token_id]
						if token_ids_a is None:
										return cls + token_ids_a + sep
						return cls + token_ids_a + sep + token_ids_a + sep



		def 		lowerCAmelCase__				(self		,__lowerCamelCase		,__lowerCamelCase = None		,__lowerCamelCase = False			)					->      List[int]:

						"""simple docstring"""






						if already_has_special_tokens:
										return super().get_special_tokens_mask(
										    token_ids_a=__lowerCamelCase		,token_ids_a=__lowerCamelCase		,already_has_special_tokens=__lowerCamelCase			)

						if token_ids_a is not None:
										return [1] + ([0] * len(__lowerCamelCase			)) + [1] + ([0] * len(__lowerCamelCase			)) + [1]
						return [1] + ([0] * len(__lowerCamelCase			)) + [1]



		def 		lowerCAmelCase__				(self		,__lowerCamelCase		,__lowerCamelCase = None			)					->      List[int]:

						"""simple docstring"""






						lowerCAmelCase__					:    Optional[Any]												=					[self.sep_token_id]
						lowerCAmelCase__					:    Any												=					[self.cls_token_id]

						if token_ids_a is None:
										return len(cls + token_ids_a + sep			) * [0]
						return len(cls + token_ids_a + sep			) * [0] + len(token_ids_a + sep			) * [1]



		def 		lowerCAmelCase__				(self		,__lowerCamelCase		,__lowerCamelCase = None			)					->      Tuple[str]:

						"""simple docstring"""






						if not os.path.isdir(__lowerCamelCase			):
										logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""			)
										return
						lowerCAmelCase__					:    Optional[Any]												=					os.path.join(
						    __lowerCamelCase		,(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']			)

						if os.path.abspath(self.vocab_file			) != os.path.abspath(__lowerCamelCase			) and os.path.isfile(self.vocab_file			):
										copyfile(self.vocab_file		,__lowerCamelCase			)
						elif not os.path.isfile(self.vocab_file			):
										with open(__lowerCamelCase		,'''wb'''			) as fi:
														lowerCAmelCase__					:    List[str]												=					self.sp_model.serialized_model_proto()
														fi.write(__lowerCamelCase			)

						return (out_vocab_file,)






import random
import unittest

import torch

from diffusers import IFInpaintingSuperResolutionPipeline
from diffusers.utils import floats_tensor
from diffusers.utils.import_utils import is_xformers_available
from diffusers.utils.testing_utils import skip_mps, torch_device

from ..pipeline_params import (
    TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS,
    TEXT_GUIDED_IMAGE_INPAINTING_PARAMS,
)
from ..test_pipelines_common import PipelineTesterMixin
from . import IFPipelineTesterMixin



@skip_mps
class 	lowerCamelCase__      (							lowerCamelCase__				,    lowerCamelCase__				,    unittest.TestCase):




		'''simple docstring'''



		snake_case_     =IFInpaintingSuperResolutionPipeline
		snake_case_     =TEXT_GUIDED_IMAGE_INPAINTING_PARAMS - {"""width""", """height"""}
		snake_case_     =TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS.union({"""original_image"""})
		snake_case_     =PipelineTesterMixin.required_optional_params - {"""latents"""}



		def 		lowerCAmelCase__				(self			)					->      Union[str, Any]:

						"""simple docstring"""






						return self._get_superresolution_dummy_components()



		def 		lowerCAmelCase__				(self		,__lowerCamelCase		,__lowerCamelCase=0			)					->      Dict:

						"""simple docstring"""






						if str(__lowerCamelCase			).startswith('''mps'''			):
										lowerCAmelCase__					:    Union[str, Any]												=					torch.manual_seed(__lowerCamelCase			)
						else:
										lowerCAmelCase__					:    int												=					torch.Generator(device=__lowerCamelCase			).manual_seed(__lowerCamelCase			)

						lowerCAmelCase__					:    str												=					floats_tensor((1, 3, 16, 16)		,rng=random.Random(__lowerCamelCase			)			).to(__lowerCamelCase			)
						lowerCAmelCase__					:    Optional[Any]												=					floats_tensor((1, 3, 32, 32)		,rng=random.Random(__lowerCamelCase			)			).to(__lowerCamelCase			)
						lowerCAmelCase__					:    Union[str, Any]												=					floats_tensor((1, 3, 32, 32)		,rng=random.Random(__lowerCamelCase			)			).to(__lowerCamelCase			)

						lowerCAmelCase__					:    Dict												=					{
						    '''prompt''': '''A painting of a squirrel eating a burger''',
						    '''image''': image,
						    '''original_image''': original_image,
						    '''mask_image''': mask_image,
						    '''generator''': generator,
						    '''num_inference_steps''': 2,
						    '''output_type''': '''numpy''',
						}

						return inputs



		@unittest.skipIf(
		    torch_device != '''cuda''' or not is_xformers_available()		,reason='''XFormers attention is only available with CUDA and `xformers` installed'''		,)
		def 		lowerCAmelCase__				(self			)					->      int:

						"""simple docstring"""






						self._test_xformers_attention_forwardGenerator_pass(expected_max_diff=1e-3			)



		def 		lowerCAmelCase__				(self			)					->      int:

						"""simple docstring"""






						self._test_save_load_optional_components()



		@unittest.skipIf(torch_device != '''cuda'''		,reason='''float16 requires CUDA'''			)
		def 		lowerCAmelCase__				(self			)					->      str:

						"""simple docstring"""






						super().test_save_load_floataa(expected_max_diff=1e-1			)



		def 		lowerCAmelCase__				(self			)					->      Any:

						"""simple docstring"""






						self._test_attention_slicing_forward_pass(expected_max_diff=1e-2			)



		def 		lowerCAmelCase__				(self			)					->      Optional[Any]:

						"""simple docstring"""






						self._test_save_load_local()



		def 		lowerCAmelCase__				(self			)					->      Any:

						"""simple docstring"""






						self._test_inference_batch_single_identical(
						    expected_max_diff=1e-2		,)

