


'''simple docstring'''



import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
	from .tokenization_mbart import MBartTokenizer
else:
	_lowercase      :      str            =					None


_lowercase      :      Dict            =					logging.get_logger(__name__)


_lowercase      :      Optional[int]            =					{"""vocab_file""": """sentencepiece.bpe.model""", """tokenizer_file""": """tokenizer.json"""}

_lowercase      :      int            =					{
    """vocab_file""": {
        """facebook/mbart-large-en-ro""": (
            """https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model"""
        ),
        """facebook/mbart-large-cc25""": (
            """https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model"""
        ),
    },
    """tokenizer_file""": {
        """facebook/mbart-large-en-ro""": """https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/tokenizer.json""",
        """facebook/mbart-large-cc25""": """https://huggingface.co/facebook/mbart-large-cc25/resolve/main/tokenizer.json""",
    },
}

_lowercase      :      Union[str, Any]            =					{
    """facebook/mbart-large-en-ro""": 1024,
    """facebook/mbart-large-cc25""": 1024,
}

# fmt: off
_lowercase      :      Dict            =					["""ar_AR""", """cs_CZ""", """de_DE""", """en_XX""", """es_XX""", """et_EE""", """fi_FI""", """fr_XX""", """gu_IN""", """hi_IN""", """it_IT""", """ja_XX""", """kk_KZ""", """ko_KR""", """lt_LT""", """lv_LV""", """my_MM""", """ne_NP""", """nl_XX""", """ro_RO""", """ru_RU""", """si_LK""", """tr_TR""", """vi_VN""", """zh_CN"""]






class 	UpperCamelCase__(       lowerCAmelCase	):
				__magic_name__    :				List[Any]  		=	VOCAB_FILES_NAMES
				__magic_name__    :				Optional[Any]  		=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__magic_name__    :				Optional[Any]  		=	PRETRAINED_VOCAB_FILES_MAP
				__magic_name__    :				Optional[Any]  		=	["input_ids", "attention_mask"]
				__magic_name__    :				Any  		=	MBartTokenizer

				__magic_name__    :				List[int]  		=	[]
				__magic_name__    :				List[int]  		=	[]




				def __init__(		self      :			Union[str, Any]				,   lowerCAmelCase      :			Optional[Any]=None				,   lowerCAmelCase      :			Dict=None				,   lowerCAmelCase      :			int="<s>"				,   lowerCAmelCase      :			Optional[Any]="</s>"				,   lowerCAmelCase      :			List[Any]="</s>"				,   lowerCAmelCase      :			Tuple="<s>"				,   lowerCAmelCase      :			Dict="<unk>"				,   lowerCAmelCase      :			List[Any]="<pad>"				,   lowerCAmelCase      :			Optional[int]="<mask>"				,   lowerCAmelCase      :			List[str]=None				,   lowerCAmelCase      :			str=None				,   lowerCAmelCase      :			int=None				,   **lowerCAmelCase      :			List[str]				,   )->					Optional[Any]:

										"""simple docstring"""




										UpperCAmelCase					    =     AddedToken(lowerCAmelCase				,   lstrip=lowerCAmelCase				,   rstrip=lowerCAmelCase				) if isinstance(lowerCAmelCase				,   lowerCAmelCase				) else mask_token

										super().__init__(
										    vocab_file=lowerCAmelCase				,   tokenizer_file=lowerCAmelCase				,   bos_token=lowerCAmelCase				,   eos_token=lowerCAmelCase				,   sep_token=lowerCAmelCase				,   cls_token=lowerCAmelCase				,   unk_token=lowerCAmelCase				,   pad_token=lowerCAmelCase				,   mask_token=lowerCAmelCase				,   src_lang=lowerCAmelCase				,   tgt_lang=lowerCAmelCase				,   additional_special_tokens=lowerCAmelCase				,   **lowerCAmelCase				,   )

										UpperCAmelCase					    =     vocab_file
										UpperCAmelCase					    =     False if not self.vocab_file else True

										UpperCAmelCase					    =     FAIRSEQ_LANGUAGE_CODES.copy()

										if additional_special_tokens is not None:
																# Only add those special tokens if they are not already there.
																_additional_special_tokens.extend(
																    [t for t in additional_special_tokens if t not in _additional_special_tokens]				)

										self.add_special_tokens({'''additional_special_tokens''': _additional_special_tokens}				)
										UpperCAmelCase					    =     {
										    lang_code: self.convert_tokens_to_ids(lowerCAmelCase				) for lang_code in FAIRSEQ_LANGUAGE_CODES
										}

										UpperCAmelCase					    =     src_lang if src_lang is not None else '''en_XX'''
										UpperCAmelCase					    =     self.convert_tokens_to_ids(self._src_lang				)
										UpperCAmelCase					    =     tgt_lang
										self.set_src_lang_special_tokens(self._src_lang				)




				@property
				def 					a__(		self      :			Union[str, Any]				)->					str:

										"""simple docstring"""




										return self._src_lang




				@src_lang.setter
				def 					a__(		self      :			Tuple				,   lowerCAmelCase      :			str				)->					None:

										"""simple docstring"""




										UpperCAmelCase					    =     new_src_lang
										self.set_src_lang_special_tokens(self._src_lang				)




				def 					a__(		self      :			Any				,   lowerCAmelCase      :			List[int]				,   lowerCAmelCase      :			Optional[List[int]] = None				)->					List[int]:

										"""simple docstring"""




										if token_ids_a is None:
																return self.prefix_tokens + token_ids_a + self.suffix_tokens
										# We don't expect to process pairs, but leave the pair logic for API consistency
										return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens




				def 					a__(		self      :			List[Any]				,   lowerCAmelCase      :			List[int]				,   lowerCAmelCase      :			Optional[List[int]] = None				)->					List[int]:

										"""simple docstring"""




										UpperCAmelCase					    =     [self.sep_token_id]
										UpperCAmelCase					    =     [self.cls_token_id]

										if token_ids_a is None:
																return len(cls + token_ids_a + sep				) * [0]
										return len(cls + token_ids_a + sep + sep + token_ids_a + sep				) * [0]




				def 					a__(		self      :			Tuple				,   lowerCAmelCase      :			int				,   lowerCAmelCase      :			str				,   lowerCAmelCase      :			Optional[str]				,   lowerCAmelCase      :			Optional[str]				,   **lowerCAmelCase      :			List[Any]				)->					List[Any]:

										"""simple docstring"""




										if src_lang is None or tgt_lang is None:
																raise ValueError('''Translation requires a `src_lang` and a `tgt_lang` for this model'''				)
										UpperCAmelCase					    =     src_lang
										UpperCAmelCase					    =     self(lowerCAmelCase				,   add_special_tokens=lowerCAmelCase				,   return_tensors=lowerCAmelCase				,   **lowerCAmelCase				)
										UpperCAmelCase					    =     self.convert_tokens_to_ids(lowerCAmelCase				)
										UpperCAmelCase					    =     tgt_lang_id
										return inputs




				def 					a__(		self      :			int				,   lowerCAmelCase      :			List[str]				,   lowerCAmelCase      :			str = "en_XX"				,   lowerCAmelCase      :			Optional[List[str]] = None				,   lowerCAmelCase      :			str = "ro_RO"				,   **lowerCAmelCase      :			Any				,   )->					BatchEncoding:

										"""simple docstring"""




										UpperCAmelCase					    =     src_lang
										UpperCAmelCase					    =     tgt_lang
										return super().prepare_seqaseq_batch(lowerCAmelCase				,   lowerCAmelCase				,   **lowerCAmelCase				)




				def 					a__(		self      :			int				)->					str:

										"""simple docstring"""




										return self.set_src_lang_special_tokens(self.src_lang				)




				def 					a__(		self      :			Dict				)->					Any:

										"""simple docstring"""




										return self.set_tgt_lang_special_tokens(self.tgt_lang				)




				def 					a__(		self      :			Union[str, Any]				,   lowerCAmelCase      :			Optional[int]				)->					None:

										"""simple docstring"""




										UpperCAmelCase					    =     self.convert_tokens_to_ids(lowerCAmelCase				)
										UpperCAmelCase					    =     []
										UpperCAmelCase					    =     [self.eos_token_id, self.cur_lang_code]

										UpperCAmelCase					    =     self.convert_ids_to_tokens(self.prefix_tokens				)
										UpperCAmelCase					    =     self.convert_ids_to_tokens(self.suffix_tokens				)

										UpperCAmelCase					    =     processors.TemplateProcessing(
										    single=prefix_tokens_str + ['''$A'''] + suffix_tokens_str				,   pair=prefix_tokens_str + ['''$A''', '''$B'''] + suffix_tokens_str				,   special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str				,   self.prefix_tokens + self.suffix_tokens				)				)				,   )




				def 					a__(		self      :			str				,   lowerCAmelCase      :			str				)->					None:

										"""simple docstring"""




										UpperCAmelCase					    =     self.convert_tokens_to_ids(lowerCAmelCase				)
										UpperCAmelCase					    =     []
										UpperCAmelCase					    =     [self.eos_token_id, self.cur_lang_code]

										UpperCAmelCase					    =     self.convert_ids_to_tokens(self.prefix_tokens				)
										UpperCAmelCase					    =     self.convert_ids_to_tokens(self.suffix_tokens				)

										UpperCAmelCase					    =     processors.TemplateProcessing(
										    single=prefix_tokens_str + ['''$A'''] + suffix_tokens_str				,   pair=prefix_tokens_str + ['''$A''', '''$B'''] + suffix_tokens_str				,   special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str				,   self.prefix_tokens + self.suffix_tokens				)				)				,   )





				def 					a__(		self      :			Optional[int]				,   lowerCAmelCase      :			str				,   lowerCAmelCase      :			Optional[str] = None				)->					Tuple[str]:

										"""simple docstring"""




										if not self.can_save_slow_tokenizer:
																raise ValueError(
																    '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
																    '''tokenizer.'''				)

										if not os.path.isdir(lowerCAmelCase				):
																logger.error(F"""Vocabulary path ({save_directory}) should be a directory."""				)
																return
										UpperCAmelCase					    =     os.path.join(
										    lowerCAmelCase				,   (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']				)

										if os.path.abspath(self.vocab_file				) != os.path.abspath(lowerCAmelCase				):
																copyfile(self.vocab_file				,   lowerCAmelCase				)

										return (out_vocab_file,)









'''simple docstring'''



import warnings

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding






class 	UpperCamelCase__(       lowerCAmelCase	):
				__magic_name__    :				List[Any]  		=	["image_processor", "tokenizer"]
				__magic_name__    :				Tuple  		=	"ViTImageProcessor"
				__magic_name__    :				int  		=	("CLIPTokenizer", "CLIPTokenizerFast")




				def __init__(		self      :			List[str]				,   lowerCAmelCase      :			Tuple=None				,   lowerCAmelCase      :			List[str]=None				,   **lowerCAmelCase      :			Optional[int]				)->					Tuple:

										"""simple docstring"""




										UpperCAmelCase					    =     None
										if "feature_extractor" in kwargs:
																warnings.warn(
																    '''The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`'''
																    ''' instead.'''				,   lowerCAmelCase				,   )
																UpperCAmelCase					    =     kwargs.pop('''feature_extractor'''				)

										UpperCAmelCase					    =     image_processor if image_processor is not None else feature_extractor
										if image_processor is None:
																raise ValueError('''You need to specify an `image_processor`.'''				)
										if tokenizer is None:
																raise ValueError('''You need to specify a `tokenizer`.'''				)

										super().__init__(lowerCAmelCase				,   lowerCAmelCase				)




				def __call__(		self      :			int				,   lowerCAmelCase      :			Dict=None				,   lowerCAmelCase      :			Optional[int]=None				,   lowerCAmelCase      :			Union[str, Any]=None				,   lowerCAmelCase      :			int=None				,   **lowerCAmelCase      :			Tuple				)->					Optional[int]:

										"""simple docstring"""




										if text is None and visual_prompt is None and images is None:
																raise ValueError('''You have to specify either text, visual prompt or images.'''				)

										if text is not None and visual_prompt is not None:
																raise ValueError('''You have to specify exactly one type of prompt. Either text or visual prompt.'''				)

										if text is not None:
																UpperCAmelCase					    =     self.tokenizer(lowerCAmelCase				,   return_tensors=lowerCAmelCase				,   **lowerCAmelCase				)

										if visual_prompt is not None:
																UpperCAmelCase					    =     self.image_processor(lowerCAmelCase				,   return_tensors=lowerCAmelCase				,   **lowerCAmelCase				)

										if images is not None:
																UpperCAmelCase					    =     self.image_processor(lowerCAmelCase				,   return_tensors=lowerCAmelCase				,   **lowerCAmelCase				)

										if visual_prompt is not None and images is not None:
																UpperCAmelCase					    =     {
																    '''pixel_values''': image_features.pixel_values,
																    '''conditional_pixel_values''': prompt_features.pixel_values,
																}
																return encoding
										elif text is not None and images is not None:
																UpperCAmelCase					    =     image_features.pixel_values
																return encoding
										elif text is not None:
																return encoding
										elif visual_prompt is not None:
																UpperCAmelCase					    =     {
																    '''conditional_pixel_values''': prompt_features.pixel_values,
																}
																return encoding
										else:
																return BatchEncoding(data=dict(**lowerCAmelCase				)				,   tensor_type=lowerCAmelCase				)




				def 					a__(		self      :			Optional[int]				,   *lowerCAmelCase      :			List[str]				,   **lowerCAmelCase      :			Dict				)->					Tuple:

										"""simple docstring"""




										return self.tokenizer.batch_decode(*lowerCAmelCase				,   **lowerCAmelCase				)




				def 					a__(		self      :			List[Any]				,   *lowerCAmelCase      :			str				,   **lowerCAmelCase      :			List[Any]				)->					Optional[Any]:

										"""simple docstring"""




										return self.tokenizer.decode(*lowerCAmelCase				,   **lowerCAmelCase				)




				@property
				def 					a__(		self      :			Any				)->					Optional[int]:

										"""simple docstring"""




										warnings.warn(
										    '''`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead.'''				,   lowerCAmelCase				,   )
										return self.image_processor_class





				@property
				def 					a__(		self      :			str				)->					List[Any]:

										"""simple docstring"""




										warnings.warn(
										    '''`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead.'''				,   lowerCAmelCase				,   )
										return self.image_processor





