





"""simple docstring"""




import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
					from .tokenization_mbart import MBartTokenizer
else:
					UpperCamelCase__     =	None


UpperCamelCase__     =	logging.get_logger(__name__)


UpperCamelCase__     =	{"""vocab_file""": """sentencepiece.bpe.model""", """tokenizer_file""": """tokenizer.json"""}

UpperCamelCase__     =	{
    """vocab_file""": {
        """facebook/mbart-large-en-ro""": (
            """https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model"""
        ),
        """facebook/mbart-large-cc25""": (
            """https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model"""
        ),
    },
    """tokenizer_file""": {
        """facebook/mbart-large-en-ro""": """https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/tokenizer.json""",
        """facebook/mbart-large-cc25""": """https://huggingface.co/facebook/mbart-large-cc25/resolve/main/tokenizer.json""",
    },
}

UpperCamelCase__     =	{
    """facebook/mbart-large-en-ro""": 1_024,
    """facebook/mbart-large-cc25""": 1_024,
}

# fmt: off
UpperCamelCase__     =	["""ar_AR""", """cs_CZ""", """de_DE""", """en_XX""", """es_XX""", """et_EE""", """fi_FI""", """fr_XX""", """gu_IN""", """hi_IN""", """it_IT""", """ja_XX""", """kk_KZ""", """ko_KR""", """lt_LT""", """lv_LV""", """my_MM""", """ne_NP""", """nl_XX""", """ro_RO""", """ru_RU""", """si_LK""", """tr_TR""", """vi_VN""", """zh_CN"""]

class     __SCREAMING_SNAKE_CASE   (       lowerCamelCase__    ):
							snake_case       : Tuple         =    VOCAB_FILES_NAMES
							snake_case       : Any         =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							snake_case       : str         =    PRETRAINED_VOCAB_FILES_MAP
							snake_case       : int         =    ['input_ids', 'attention_mask']
							snake_case       : Union[str, Any]         =    MBartTokenizer

							snake_case       : List[int]         =    []
							snake_case       : List[int]         =    []






							def __init__(  self					,	__lowerCAmelCase=None					,	__lowerCAmelCase=None					,	__lowerCAmelCase="<s>"					,	__lowerCAmelCase="</s>"					,	__lowerCAmelCase="</s>"					,	__lowerCAmelCase="<s>"					,	__lowerCAmelCase="<unk>"					,	__lowerCAmelCase="<pad>"					,	__lowerCAmelCase="<mask>"					,	__lowerCAmelCase=None					,	__lowerCAmelCase=None					,	__lowerCAmelCase=None					,	**__lowerCAmelCase					,	):
														UpperCamelCase__			       =     AddedToken(__lowerCAmelCase					,	lstrip=__lowerCAmelCase					,	rstrip=__lowerCAmelCase					) if isinstance(__lowerCAmelCase					,	__lowerCAmelCase					) else mask_token

														super().__init__(
														    vocab_file=__lowerCAmelCase					,	tokenizer_file=__lowerCAmelCase					,	bos_token=__lowerCAmelCase					,	eos_token=__lowerCAmelCase					,	sep_token=__lowerCAmelCase					,	cls_token=__lowerCAmelCase					,	unk_token=__lowerCAmelCase					,	pad_token=__lowerCAmelCase					,	mask_token=__lowerCAmelCase					,	src_lang=__lowerCAmelCase					,	tgt_lang=__lowerCAmelCase					,	additional_special_tokens=__lowerCAmelCase					,	**__lowerCAmelCase					,	)

														UpperCamelCase__			       =     vocab_file
														UpperCamelCase__			       =     False if not self.vocab_file else True

														UpperCamelCase__			       =     FAIRSEQ_LANGUAGE_CODES.copy()

														if additional_special_tokens is not None:
																					# Only add those special tokens if they are not already there.
																					_additional_special_tokens.extend(
																					    [t for t in additional_special_tokens if t not in _additional_special_tokens]					)

														self.add_special_tokens({"""additional_special_tokens""": _additional_special_tokens}					)
														UpperCamelCase__			       =     {
														    lang_code: self.convert_tokens_to_ids(__lowerCAmelCase					) for lang_code in FAIRSEQ_LANGUAGE_CODES
														}

														UpperCamelCase__			       =     src_lang if src_lang is not None else """en_XX"""
														UpperCamelCase__			       =     self.convert_tokens_to_ids(self._src_lang					)
														UpperCamelCase__			       =     tgt_lang
														self.set_src_lang_special_tokens(self._src_lang					)






							@property
							def 			_lowerCamelCase						(  self					):
														return self._src_lang






							@src_lang.setter
							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					):
														UpperCamelCase__			       =     new_src_lang
														self.set_src_lang_special_tokens(self._src_lang					)






							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					,	__lowerCAmelCase = None					):







														if token_ids_a is None:
																					return self.prefix_tokens + token_ids_a + self.suffix_tokens
														# We don't expect to process pairs, but leave the pair logic for API consistency
														return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens






							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					,	__lowerCAmelCase = None					):
														UpperCamelCase__			       =     [self.sep_token_id]
														UpperCamelCase__			       =     [self.cls_token_id]

														if token_ids_a is None:
																					return len(cls + token_ids_a + sep					) * [0]
														return len(cls + token_ids_a + sep + sep + token_ids_a + sep					) * [0]






							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					,	__lowerCAmelCase					,	__lowerCAmelCase					,	__lowerCAmelCase					,	**__lowerCAmelCase					):







														if src_lang is None or tgt_lang is None:
																					raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model"""					)
														UpperCamelCase__			       =     src_lang
														UpperCamelCase__			       =     self(__lowerCAmelCase					,	add_special_tokens=__lowerCAmelCase					,	return_tensors=__lowerCAmelCase					,	**__lowerCAmelCase					)
														UpperCamelCase__			       =     self.convert_tokens_to_ids(__lowerCAmelCase					)
														UpperCamelCase__			       =     tgt_lang_id
														return inputs






							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					,	__lowerCAmelCase = "en_XX"					,	__lowerCAmelCase = None					,	__lowerCAmelCase = "ro_RO"					,	**__lowerCAmelCase					,	):
														UpperCamelCase__			       =     src_lang
														UpperCamelCase__			       =     tgt_lang
														return super().prepare_seqaseq_batch(__lowerCAmelCase					,	__lowerCAmelCase					,	**__lowerCAmelCase					)






							def 			_lowerCamelCase						(  self					):
														return self.set_src_lang_special_tokens(self.src_lang					)






							def 			_lowerCamelCase						(  self					):
														return self.set_tgt_lang_special_tokens(self.tgt_lang					)






							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					):
														UpperCamelCase__			       =     self.convert_tokens_to_ids(__lowerCAmelCase					)
														UpperCamelCase__			       =     []
														UpperCamelCase__			       =     [self.eos_token_id, self.cur_lang_code]

														UpperCamelCase__			       =     self.convert_ids_to_tokens(self.prefix_tokens					)
														UpperCamelCase__			       =     self.convert_ids_to_tokens(self.suffix_tokens					)

														UpperCamelCase__			       =     processors.TemplateProcessing(
														    single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str					,	pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str					,	special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str					,	self.prefix_tokens + self.suffix_tokens					)					)					,	)






							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					):
														UpperCamelCase__			       =     self.convert_tokens_to_ids(__lowerCAmelCase					)
														UpperCamelCase__			       =     []
														UpperCamelCase__			       =     [self.eos_token_id, self.cur_lang_code]

														UpperCamelCase__			       =     self.convert_ids_to_tokens(self.prefix_tokens					)
														UpperCamelCase__			       =     self.convert_ids_to_tokens(self.suffix_tokens					)

														UpperCamelCase__			       =     processors.TemplateProcessing(
														    single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str					,	pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str					,	special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str					,	self.prefix_tokens + self.suffix_tokens					)					)					,	)







							def 			_lowerCamelCase						(  self					,	__lowerCAmelCase					,	__lowerCAmelCase = None					):







														if not self.can_save_slow_tokenizer:
																					raise ValueError(
																					    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
																					    """tokenizer."""					)

														if not os.path.isdir(__lowerCAmelCase					):
																					logger.error(f"""Vocabulary path ({save_directory}) should be a directory."""					)
																					return
														UpperCamelCase__			       =     os.path.join(
														    __lowerCAmelCase					,	(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]					)

														if os.path.abspath(self.vocab_file					) != os.path.abspath(__lowerCAmelCase					):
																					copyfile(self.vocab_file					,	__lowerCAmelCase					)

														return (out_vocab_file,)











import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import is_speech_available, is_vision_available
from transformers.testing_utils import require_torch


if is_vision_available():
					from transformers import TvltImageProcessor

if is_speech_available():
					from transformers import TvltFeatureExtractor

from transformers import TvltProcessor

@require_torch
class     __SCREAMING_SNAKE_CASE   (       unittest.TestCase    ):






							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     """ZinengTang/tvlt-base"""
														UpperCamelCase__			       =     tempfile.mkdtemp()






							def 			_lowerCamelCase						(  self					,	**__lowerCAmelCase					):
														return TvltImageProcessor.from_pretrained(self.checkpoint					,	**__lowerCAmelCase					)






							def 			_lowerCamelCase						(  self					,	**__lowerCAmelCase					):
														return TvltFeatureExtractor.from_pretrained(self.checkpoint					,	**__lowerCAmelCase					)






							def 			_lowerCamelCase						(  self					):
														shutil.rmtree(self.tmpdirname					)






							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     self.get_image_processor()
														UpperCamelCase__			       =     self.get_feature_extractor()

														UpperCamelCase__			       =     TvltProcessor(image_processor=__lowerCAmelCase					,	feature_extractor=__lowerCAmelCase					)
														processor.save_pretrained(self.tmpdirname					)
														UpperCamelCase__			       =     TvltProcessor.from_pretrained(self.tmpdirname					)

														self.assertIsInstance(processor.feature_extractor					,	__lowerCAmelCase					)
														self.assertIsInstance(processor.image_processor					,	__lowerCAmelCase					)






							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     self.get_image_processor()
														UpperCamelCase__			       =     self.get_feature_extractor()

														UpperCamelCase__			       =     TvltProcessor(image_processor=__lowerCAmelCase					,	feature_extractor=__lowerCAmelCase					)

														UpperCamelCase__			       =     np.ones([12000]					)

														UpperCamelCase__			       =     feature_extractor(__lowerCAmelCase					,	return_tensors="""np"""					)
														UpperCamelCase__			       =     processor(audio=__lowerCAmelCase					,	return_tensors="""np"""					)

														for key in audio_dict.keys():
																					self.assertAlmostEqual(audio_dict[key].sum()					,	input_processor[key].sum()					,	delta=1E-2					)






							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     self.get_image_processor()
														UpperCamelCase__			       =     self.get_feature_extractor()

														UpperCamelCase__			       =     TvltProcessor(image_processor=__lowerCAmelCase					,	feature_extractor=__lowerCAmelCase					)

														UpperCamelCase__			       =     np.ones([3, 224, 224]					)

														UpperCamelCase__			       =     image_processor(__lowerCAmelCase					,	return_tensors="""np"""					)
														UpperCamelCase__			       =     processor(images=__lowerCAmelCase					,	return_tensors="""np"""					)

														for key in image_dict.keys():
																					self.assertAlmostEqual(image_dict[key].sum()					,	input_processor[key].sum()					,	delta=1E-2					)






							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     self.get_image_processor()
														UpperCamelCase__			       =     self.get_feature_extractor()

														UpperCamelCase__			       =     TvltProcessor(image_processor=__lowerCAmelCase					,	feature_extractor=__lowerCAmelCase					)

														UpperCamelCase__			       =     np.ones([12000]					)
														UpperCamelCase__			       =     np.ones([3, 224, 224]					)

														UpperCamelCase__			       =     processor(audio=__lowerCAmelCase					,	images=__lowerCAmelCase					)

														self.assertListEqual(list(inputs.keys()					)					,	["""audio_values""", """audio_mask""", """pixel_values""", """pixel_mask"""]					)

														# test if it raises when no input is passed
														with pytest.raises(__lowerCAmelCase					):
																					processor()







							def 			_lowerCamelCase						(  self					):
														UpperCamelCase__			       =     self.get_image_processor()
														UpperCamelCase__			       =     self.get_feature_extractor()

														UpperCamelCase__			       =     TvltProcessor(image_processor=__lowerCAmelCase					,	feature_extractor=__lowerCAmelCase					)

														self.assertListEqual(
														    processor.model_input_names					,	image_processor.model_input_names + feature_extractor.model_input_names					,	msg="""`processor` and `image_processor`+`feature_extractor` model input names do not match"""					,	)




