






import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
		from .tokenization_mbart import MBartTokenizer
else:
		__UpperCAmelCase     							=						None


__UpperCAmelCase     							=						logging.get_logger(__name__)


__UpperCAmelCase     							=						{"vocab_file": "sentencepiece.bpe.model", "tokenizer_file": "tokenizer.json"}

__UpperCAmelCase     							=						{
    "vocab_file": {
        "facebook/mbart-large-en-ro": (
            "https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model"
        ),
        "facebook/mbart-large-cc25": (
            "https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model"
        ),
    },
    "tokenizer_file": {
        "facebook/mbart-large-en-ro": "https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/tokenizer.json",
        "facebook/mbart-large-cc25": "https://huggingface.co/facebook/mbart-large-cc25/resolve/main/tokenizer.json",
    },
}

__UpperCAmelCase     							=						{
    "facebook/mbart-large-en-ro": 1024,
    "facebook/mbart-large-cc25": 1024,
}

# fmt: off
__UpperCAmelCase     							=						["ar_AR", "cs_CZ", "de_DE", "en_XX", "es_XX", "et_EE", "fi_FI", "fr_XX", "gu_IN", "hi_IN", "it_IT", "ja_XX", "kk_KZ", "ko_KR", "lt_LT", "lv_LV", "my_MM", "ne_NP", "nl_XX", "ro_RO", "ru_RU", "si_LK", "tr_TR", "vi_VN", "zh_CN"]



class   lowerCamelCase				(UpperCAmelCase__      ):







							'''simple docstring'''



							_snake_case   :   Tuple							=  VOCAB_FILES_NAMES
							_snake_case   :   int							=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							_snake_case   :   Dict							=  PRETRAINED_VOCAB_FILES_MAP
							_snake_case   :   Tuple							=  ['input_ids', 'attention_mask']
							_snake_case   :   str							=  MBartTokenizer

							_snake_case   :   List[int]							=  []
							_snake_case   :   List[int]							=  []



							def __init__(							self    ,		_UpperCamelCase=None    ,		_UpperCamelCase=None    ,		_UpperCamelCase="<s>"    ,		_UpperCamelCase="</s>"    ,		_UpperCamelCase="</s>"    ,		_UpperCamelCase="<s>"    ,		_UpperCamelCase="<unk>"    ,		_UpperCamelCase="<pad>"    ,		_UpperCamelCase="<mask>"    ,		_UpperCamelCase=None    ,		_UpperCamelCase=None    ,		_UpperCamelCase=None    ,		**_UpperCamelCase    ,		)     ->							Any:
												UpperCAmelCase_   :				Union[str, Any]						=						AddedToken(_SCREAMING_SNAKE_CASE    ,		lstrip=_SCREAMING_SNAKE_CASE    ,		rstrip=_SCREAMING_SNAKE_CASE       ) if isinstance(_SCREAMING_SNAKE_CASE    ,		_SCREAMING_SNAKE_CASE       ) else mask_token

												super().__init__(
												    vocab_file=_SCREAMING_SNAKE_CASE    ,		tokenizer_file=_SCREAMING_SNAKE_CASE    ,		bos_token=_SCREAMING_SNAKE_CASE    ,		eos_token=_SCREAMING_SNAKE_CASE    ,		sep_token=_SCREAMING_SNAKE_CASE    ,		cls_token=_SCREAMING_SNAKE_CASE    ,		unk_token=_SCREAMING_SNAKE_CASE    ,		pad_token=_SCREAMING_SNAKE_CASE    ,		mask_token=_SCREAMING_SNAKE_CASE    ,		src_lang=_SCREAMING_SNAKE_CASE    ,		tgt_lang=_SCREAMING_SNAKE_CASE    ,		additional_special_tokens=_SCREAMING_SNAKE_CASE    ,		**_SCREAMING_SNAKE_CASE    ,		)

												UpperCAmelCase_   :				int						=						vocab_file
												UpperCAmelCase_   :				Optional[int]						=						False if not self.vocab_file else True

												UpperCAmelCase_   :				List[str]						=						FAIRSEQ_LANGUAGE_CODES.copy()

												if additional_special_tokens is not None:
																	# Only add those special tokens if they are not already there.
																	_additional_special_tokens.extend(
																	    [t for t in additional_special_tokens if t not in _additional_special_tokens]       )

												self.add_special_tokens({'additional_special_tokens': _additional_special_tokens}       )
												UpperCAmelCase_   :				List[Any]						=						{
												    lang_code: self.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE       ) for lang_code in FAIRSEQ_LANGUAGE_CODES
												}

												UpperCAmelCase_   :				int						=						src_lang if src_lang is not None else """en_XX"""
												UpperCAmelCase_   :				List[Any]						=						self.convert_tokens_to_ids(self._src_lang       )
												UpperCAmelCase_   :				int						=						tgt_lang
												self.set_src_lang_special_tokens(self._src_lang       )



							@property
							def  __UpperCAmelCase      (							self       )     ->							str:
												return self._src_lang



							@src_lang.setter
							def  __UpperCAmelCase      (							self    ,		_UpperCamelCase       )     ->							None:
												UpperCAmelCase_   :				Dict						=						new_src_lang
												self.set_src_lang_special_tokens(self._src_lang       )



							def  __UpperCAmelCase      (							self    ,		_UpperCamelCase    ,		_UpperCamelCase = None       )     ->							List[int]:




												if token_ids_a is None:
																	return self.prefix_tokens + token_ids_a + self.suffix_tokens
												# We don't expect to process pairs, but leave the pair logic for API consistency
												return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens



							def  __UpperCAmelCase      (							self    ,		_UpperCamelCase    ,		_UpperCamelCase = None       )     ->							List[int]:
												UpperCAmelCase_   :				str						=						[self.sep_token_id]
												UpperCAmelCase_   :				str						=						[self.cls_token_id]

												if token_ids_a is None:
																	return len(cls + token_ids_a + sep       ) * [0]
												return len(cls + token_ids_a + sep + sep + token_ids_a + sep       ) * [0]



							def  __UpperCAmelCase      (							self    ,		_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase    ,		**_UpperCamelCase       )     ->							Optional[int]:




												if src_lang is None or tgt_lang is None:
																	raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'       )
												UpperCAmelCase_   :				List[str]						=						src_lang
												UpperCAmelCase_   :				Union[str, Any]						=						self(_SCREAMING_SNAKE_CASE    ,		add_special_tokens=_SCREAMING_SNAKE_CASE    ,		return_tensors=_SCREAMING_SNAKE_CASE    ,		**_SCREAMING_SNAKE_CASE       )
												UpperCAmelCase_   :				Dict						=						self.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE       )
												UpperCAmelCase_   :				Tuple						=						tgt_lang_id
												return inputs



							def  __UpperCAmelCase      (							self    ,		_UpperCamelCase    ,		_UpperCamelCase = "en_XX"    ,		_UpperCamelCase = None    ,		_UpperCamelCase = "ro_RO"    ,		**_UpperCamelCase    ,		)     ->							BatchEncoding:
												UpperCAmelCase_   :				int						=						src_lang
												UpperCAmelCase_   :				Dict						=						tgt_lang
												return super().prepare_seqaseq_batch(_SCREAMING_SNAKE_CASE    ,		_SCREAMING_SNAKE_CASE    ,		**_SCREAMING_SNAKE_CASE       )



							def  __UpperCAmelCase      (							self       )     ->							Union[str, Any]:
												return self.set_src_lang_special_tokens(self.src_lang       )



							def  __UpperCAmelCase      (							self       )     ->							str:
												return self.set_tgt_lang_special_tokens(self.tgt_lang       )



							def  __UpperCAmelCase      (							self    ,		_UpperCamelCase       )     ->							None:
												UpperCAmelCase_   :				Any						=						self.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE       )
												UpperCAmelCase_   :				Any						=						[]
												UpperCAmelCase_   :				Tuple						=						[self.eos_token_id, self.cur_lang_code]

												UpperCAmelCase_   :				Optional[Any]						=						self.convert_ids_to_tokens(self.prefix_tokens       )
												UpperCAmelCase_   :				List[str]						=						self.convert_ids_to_tokens(self.suffix_tokens       )

												UpperCAmelCase_   :				str						=						processors.TemplateProcessing(
												    single=prefix_tokens_str + ['$A'] + suffix_tokens_str    ,		pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str    ,		special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str    ,		self.prefix_tokens + self.suffix_tokens       )       )    ,		)



							def  __UpperCAmelCase      (							self    ,		_UpperCamelCase       )     ->							None:
												UpperCAmelCase_   :				Tuple						=						self.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE       )
												UpperCAmelCase_   :				int						=						[]
												UpperCAmelCase_   :				Optional[int]						=						[self.eos_token_id, self.cur_lang_code]

												UpperCAmelCase_   :				str						=						self.convert_ids_to_tokens(self.prefix_tokens       )
												UpperCAmelCase_   :				Optional[Any]						=						self.convert_ids_to_tokens(self.suffix_tokens       )

												UpperCAmelCase_   :				int						=						processors.TemplateProcessing(
												    single=prefix_tokens_str + ['$A'] + suffix_tokens_str    ,		pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str    ,		special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str    ,		self.prefix_tokens + self.suffix_tokens       )       )    ,		)



							def  __UpperCAmelCase      (							self    ,		_UpperCamelCase    ,		_UpperCamelCase = None       )     ->							Tuple[str]:




												if not self.can_save_slow_tokenizer:
																	raise ValueError(
																	    'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
																	    'tokenizer.'       )

												if not os.path.isdir(_SCREAMING_SNAKE_CASE       ):
																	logger.error(f"Vocabulary path ({save_directory}) should be a directory."       )
																	return
												UpperCAmelCase_   :				Any						=						os.path.join(
												    _SCREAMING_SNAKE_CASE    ,		(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']       )

												if os.path.abspath(self.vocab_file       ) != os.path.abspath(_SCREAMING_SNAKE_CASE       ):
																	copyfile(self.vocab_file    ,		_SCREAMING_SNAKE_CASE       )

												return (out_vocab_file,)














from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxSeqaSeqConfigWithPast
from ...utils import logging


__UpperCAmelCase     							=						logging.get_logger(__name__)

__UpperCAmelCase     							=						{
    'google/umt5-small': 'https://huggingface.co/google/umt5-small/resolve/main/config.json',
    # See all umt5 models at https://huggingface.co/models?filter=umt5
}



class   lowerCamelCase				(_snake_case      ):







		'''simple docstring'''



		_snake_case   :   Union[str, Any]							=  '''umt5'''
		_snake_case   :   Union[str, Any]							=  ['''past_key_values''']



		def __init__(							self    ,		_UpperCamelCase=2_5_0_1_1_2    ,		_UpperCamelCase=5_1_2    ,		_UpperCamelCase=6_4    ,		_UpperCamelCase=1_0_2_4    ,		_UpperCamelCase=8    ,		_UpperCamelCase=None    ,		_UpperCamelCase=6    ,		_UpperCamelCase=3_2    ,		_UpperCamelCase=1_2_8    ,		_UpperCamelCase=0.1    ,		_UpperCamelCase=1E-6    ,		_UpperCamelCase=1.0    ,		_UpperCamelCase="gated-gelu"    ,		_UpperCamelCase=True    ,		_UpperCamelCase=True    ,		_UpperCamelCase="T5Tokenizer"    ,		_UpperCamelCase=True    ,		_UpperCamelCase=0    ,		_UpperCamelCase=1    ,		_UpperCamelCase=0    ,		**_UpperCamelCase    ,		)     ->							List[Any]:
							super().__init__(
							    is_encoder_decoder=_UpperCamelCase    ,		tokenizer_class=_UpperCamelCase    ,		tie_word_embeddings=_UpperCamelCase    ,		pad_token_id=_UpperCamelCase    ,		eos_token_id=_UpperCamelCase    ,		decoder_start_token_id=_UpperCamelCase    ,		**_UpperCamelCase    ,		)
							UpperCAmelCase_   :				Union[str, Any]						=						vocab_size
							UpperCAmelCase_   :				List[str]						=						d_model
							UpperCAmelCase_   :				Any						=						d_kv
							UpperCAmelCase_   :				Optional[int]						=						d_ff
							UpperCAmelCase_   :				List[Any]						=						num_layers
							UpperCAmelCase_   :				Optional[Any]						=						(
							    num_decoder_layers if num_decoder_layers is not None else self.num_layers
							)  # default = symmetry
							UpperCAmelCase_   :				Optional[int]						=						num_heads
							UpperCAmelCase_   :				Optional[int]						=						relative_attention_num_buckets
							UpperCAmelCase_   :				Dict						=						relative_attention_max_distance
							UpperCAmelCase_   :				Tuple						=						dropout_rate
							UpperCAmelCase_   :				Union[str, Any]						=						layer_norm_epsilon
							UpperCAmelCase_   :				Optional[int]						=						initializer_factor
							UpperCAmelCase_   :				List[str]						=						feed_forward_proj
							UpperCAmelCase_   :				Any						=						use_cache

							UpperCAmelCase_   :				List[Any]						=						self.feed_forward_proj.split('-'       )
							UpperCAmelCase_   :				List[Any]						=						act_info[-1]
							UpperCAmelCase_   :				Union[str, Any]						=						act_info[0] == 'gated'

							if len(_UpperCamelCase       ) > 1 and act_info[0] != "gated" or len(_UpperCamelCase       ) > 2:
												raise ValueError(
												    f"`feed_forward_proj`: {feed_forward_proj} is not a valid activation function of the dense layer."
												    'Please make sure `feed_forward_proj` is of the format `gated-{ACT_FN}` or `{ACT_FN}`, e.g. '
												    '\'gated-gelu\' or \'relu\''       )

							if feed_forward_proj == "gated-gelu":
												UpperCAmelCase_   :				Optional[int]						=						'gelu_new'



		@property
		def  __UpperCAmelCase      (							self       )     ->							int:
							return self.d_model



		@property
		def  __UpperCAmelCase      (							self       )     ->							Any:
							return self.num_heads



		@property
		def  __UpperCAmelCase      (							self       )     ->							List[Any]:
							return self.num_layers





class   lowerCamelCase				(_snake_case      ):







		'''simple docstring'''



		@property
		# Copied from transformers.models.t5.configuration_t5.T5OnnxConfig.inputs
		def  __UpperCAmelCase      (							self       )     ->							Mapping[str, Mapping[int, str]]:
							UpperCAmelCase_   :				str						=						{
							    'input_ids': {0: 'batch', 1: 'encoder_sequence'},
							    'attention_mask': {0: 'batch', 1: 'encoder_sequence'},
							}
							if self.use_past:
												UpperCAmelCase_   :				Optional[int]						=						'past_encoder_sequence + sequence'
												UpperCAmelCase_   :				str						=						{0: 'batch'}
												UpperCAmelCase_   :				Optional[int]						=						{0: 'batch', 1: 'past_decoder_sequence + sequence'}
							else:
												UpperCAmelCase_   :				Optional[int]						=						{0: 'batch', 1: 'decoder_sequence'}
												UpperCAmelCase_   :				Union[str, Any]						=						{0: 'batch', 1: 'decoder_sequence'}

							if self.use_past:
												self.fill_with_past_key_values_(_UpperCamelCase    ,		direction='inputs'       )

							return common_inputs



		@property
		# Copied from transformers.models.t5.configuration_t5.T5OnnxConfig.default_onnx_opset
		def  __UpperCAmelCase      (							self       )     ->							int:
							return 1_3



		@property
		def  __UpperCAmelCase      (							self       )     ->							float:
							return 5E-4






