



import warnings

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding

class    _A	(     __UpperCAmelCase   ):
       UpperCamelCase__  :   str						 =						['''image_processor''', '''tokenizer''']
       UpperCamelCase__  :   str						 =						'''CLIPImageProcessor'''
       UpperCamelCase__  :   str						 =						('''CLIPTokenizer''', '''CLIPTokenizerFast''')



       def __init__(		self							:					str					,		__SCREAMING_SNAKE_CASE							:					Tuple=None					,		__SCREAMING_SNAKE_CASE							:					Dict=None					,		**__SCREAMING_SNAKE_CASE							:					str):


           '''simple docstring'''

           __a							=    None
           if "feature_extractor" in kwargs:
               warnings.warn(
                   '''The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`'''
                   ''' instead.'''					,		__SCREAMING_SNAKE_CASE					,		)
               __a							=    kwargs.pop('''feature_extractor''')

           __a							=    image_processor if image_processor is not None else feature_extractor
           if image_processor is None:
               raise ValueError('''You need to specify an `image_processor`.''')
           if tokenizer is None:
               raise ValueError('''You need to specify a `tokenizer`.''')

           super().__init__(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE)



       def __call__(		self							:					Any					,		__SCREAMING_SNAKE_CASE							:					Optional[int]=None					,		__SCREAMING_SNAKE_CASE							:					List[str]=None					,		__SCREAMING_SNAKE_CASE							:					Any=None					,		**__SCREAMING_SNAKE_CASE							:					List[str]):


           '''simple docstring'''

           if text is None and images is None:
               raise ValueError('''You have to specify either text or images. Both cannot be none.''')

           if text is not None:
               __a							=    self.tokenizer(__SCREAMING_SNAKE_CASE					,		return_tensors=__SCREAMING_SNAKE_CASE					,		**__SCREAMING_SNAKE_CASE)

           if images is not None:
               __a							=    self.image_processor(__SCREAMING_SNAKE_CASE					,		return_tensors=__SCREAMING_SNAKE_CASE					,		**__SCREAMING_SNAKE_CASE)

           if text is not None and images is not None:
               __a							=    image_features.pixel_values
               return encoding
           elif text is not None:
               return encoding
           else:
               return BatchEncoding(data=dict(**__SCREAMING_SNAKE_CASE)					,		tensor_type=__SCREAMING_SNAKE_CASE)



       def 		_lowerCamelCase      (		self							:					List[Any]					,		*__SCREAMING_SNAKE_CASE							:					int					,		**__SCREAMING_SNAKE_CASE							:					List[Any]):


           '''simple docstring'''

           return self.tokenizer.batch_decode(*__SCREAMING_SNAKE_CASE					,		**__SCREAMING_SNAKE_CASE)



       def 		_lowerCamelCase      (		self							:					Optional[Any]					,		*__SCREAMING_SNAKE_CASE							:					int					,		**__SCREAMING_SNAKE_CASE							:					Dict):


           '''simple docstring'''

           return self.tokenizer.decode(*__SCREAMING_SNAKE_CASE					,		**__SCREAMING_SNAKE_CASE)



       @property
       def 		_lowerCamelCase      (		self							:					int):


           '''simple docstring'''

           __a							=    self.tokenizer.model_input_names
           __a							=    self.image_processor.model_input_names
           return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names))



       @property
       def 		_lowerCamelCase      (		self							:					Dict):


           '''simple docstring'''

           warnings.warn(
               '''`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead.'''					,		__SCREAMING_SNAKE_CASE					,		)
           return self.image_processor_class





       @property
       def 		_lowerCamelCase      (		self							:					Dict):


           '''simple docstring'''

           warnings.warn(
               '''`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead.'''					,		__SCREAMING_SNAKE_CASE					,		)
           return self.image_processor










import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__snake_case						:Optional[Any]						=					logging.get_logger(__name__)


__snake_case						:List[Any]						=					'''▁'''

__snake_case						:List[Any]						=					{'''vocab_file''': '''spiece.model'''}

__snake_case						:Tuple						=					{
    '''vocab_file''': {
        '''google/reformer-crime-and-punishment''': (
            '''https://huggingface.co/google/reformer-crime-and-punishment/resolve/main/spiece.model'''
        )
    }
}

__snake_case						:List[Any]						=					{
    '''google/reformer-crime-and-punishment''': 52_4288,
}

class    _A	(     __UpperCAmelCase   ):
       UpperCamelCase__  :   int						 =						VOCAB_FILES_NAMES
       UpperCamelCase__  :   str						 =						PRETRAINED_VOCAB_FILES_MAP
       UpperCamelCase__  :   Dict						 =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       UpperCamelCase__  :   Optional[int]						 =						['''input_ids''', '''attention_mask''']



       def __init__(		self							:					Any					,		__SCREAMING_SNAKE_CASE							:					str					,		__SCREAMING_SNAKE_CASE							:					Optional[Any]="</s>"					,		__SCREAMING_SNAKE_CASE							:					List[Any]="<unk>"					,		__SCREAMING_SNAKE_CASE							:					Any=[]					,		__SCREAMING_SNAKE_CASE							:					Optional[Dict[str, Any]] = None					,		**__SCREAMING_SNAKE_CASE							:					List[Any]					,		):


           '''simple docstring'''

           __a							=    {} if sp_model_kwargs is None else sp_model_kwargs

           super().__init__(
               eos_token=__SCREAMING_SNAKE_CASE					,		unk_token=__SCREAMING_SNAKE_CASE					,		additional_special_tokens=__SCREAMING_SNAKE_CASE					,		sp_model_kwargs=self.sp_model_kwargs					,		**__SCREAMING_SNAKE_CASE					,		)

           __a							=    vocab_file
           __a							=    spm.SentencePieceProcessor(**self.sp_model_kwargs)
           self.sp_model.Load(__SCREAMING_SNAKE_CASE)



       @property
       def 		_lowerCamelCase      (		self							:					Optional[Any]):


           '''simple docstring'''

           return self.sp_model.get_piece_size()



       def 		_lowerCamelCase      (		self							:					List[str]):


           '''simple docstring'''

           __a							=    {self.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE): i for i in range(self.vocab_size)}
           vocab.update(self.added_tokens_encoder)
           return vocab



       def __getstate__(		self							:					Dict):


           '''simple docstring'''

           __a							=    self.__dict__.copy()
           __a							=    None
           return state



       def __setstate__(		self							:					Dict					,		__SCREAMING_SNAKE_CASE							:					List[Any]):


           '''simple docstring'''

           __a							=    d

           # for backward compatibility
           if not hasattr(self					,		'''sp_model_kwargs'''):
               __a							=    {}

           __a							=    spm.SentencePieceProcessor(**self.sp_model_kwargs)
           self.sp_model.Load(self.vocab_file)



       def 		_lowerCamelCase      (		self							:					Union[str, Any]					,		__SCREAMING_SNAKE_CASE							:					str):


           '''simple docstring'''

           return self.sp_model.encode(__SCREAMING_SNAKE_CASE					,		out_type=__SCREAMING_SNAKE_CASE)



       def 		_lowerCamelCase      (		self							:					Optional[Any]					,		__SCREAMING_SNAKE_CASE							:					List[Any]):


           '''simple docstring'''

           return self.sp_model.piece_to_id(__SCREAMING_SNAKE_CASE)



       def 		_lowerCamelCase      (		self							:					List[str]					,		__SCREAMING_SNAKE_CASE							:					List[Any]):


           '''simple docstring'''

           if index < self.sp_model.get_piece_size():
               __a							=    self.sp_model.IdToPiece(__SCREAMING_SNAKE_CASE)
           return token



       def 		_lowerCamelCase      (		self							:					List[Any]					,		__SCREAMING_SNAKE_CASE							:					Any):


           '''simple docstring'''

           __a							=    []
           __a							=    ''''''
           for token in tokens:
               # make sure that special tokens are not decoded using sentencepiece model
               if token in self.all_special_tokens:
                   out_string += self.sp_model.decode(__SCREAMING_SNAKE_CASE) + token
                   __a							=    []
               else:
                   current_sub_tokens.append(__SCREAMING_SNAKE_CASE)
           out_string += self.sp_model.decode(__SCREAMING_SNAKE_CASE)
           return out_string.strip()





       def 		_lowerCamelCase      (		self							:					str					,		__SCREAMING_SNAKE_CASE							:					str					,		__SCREAMING_SNAKE_CASE							:					Optional[str] = None):


           '''simple docstring'''

           if not os.path.isdir(__SCREAMING_SNAKE_CASE):
               logger.error(F'Vocabulary path ({save_directory}) should be a directory')
               return
           __a							=    os.path.join(
               __SCREAMING_SNAKE_CASE					,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''])

           if os.path.abspath(self.vocab_file) != os.path.abspath(__SCREAMING_SNAKE_CASE) and os.path.isfile(self.vocab_file):
               copyfile(self.vocab_file					,		__SCREAMING_SNAKE_CASE)
           elif not os.path.isfile(self.vocab_file):
               with open(__SCREAMING_SNAKE_CASE					,		'''wb''') as fi:
                   __a							=    self.sp_model.serialized_model_proto()
                   fi.write(__SCREAMING_SNAKE_CASE)

           return (out_vocab_file,)





