






import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__UpperCAmelCase     							=						logging.get_logger(__name__)


__UpperCAmelCase     							=						{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt'}

# See all BART models at https://huggingface.co/models?filter=bart
__UpperCAmelCase     							=						{
    'vocab_file': {
        'facebook/bart-base': 'https://huggingface.co/facebook/bart-base/resolve/main/vocab.json',
        'facebook/bart-large': 'https://huggingface.co/facebook/bart-large/resolve/main/vocab.json',
        'facebook/bart-large-mnli': 'https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json',
        'facebook/bart-large-cnn': 'https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json',
        'facebook/bart-large-xsum': 'https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json',
        'yjernite/bart_eli5': 'https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json',
    },
    'merges_file': {
        'facebook/bart-base': 'https://huggingface.co/facebook/bart-base/resolve/main/merges.txt',
        'facebook/bart-large': 'https://huggingface.co/facebook/bart-large/resolve/main/merges.txt',
        'facebook/bart-large-mnli': 'https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt',
        'facebook/bart-large-cnn': 'https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt',
        'facebook/bart-large-xsum': 'https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt',
        'yjernite/bart_eli5': 'https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt',
    },
}

__UpperCAmelCase     							=						{
    'facebook/bart-base': 1024,
    'facebook/bart-large': 1024,
    'facebook/bart-large-mnli': 1024,
    'facebook/bart-large-cnn': 1024,
    'facebook/bart-large-xsum': 1024,
    'yjernite/bart_eli5': 1024,
}




@lru_cache()
def 					lowercase__				(					):


					'''simple docstring'''


					UpperCAmelCase_   :				Tuple						=						(
					    list(range(ord('!'  )					,	ord('~'  ) + 1  )  ) + list(range(ord('¡'  )					,	ord('¬'  ) + 1  )  ) + list(range(ord('®'  )					,	ord('ÿ'  ) + 1  )  )
					)
					UpperCAmelCase_   :				List[str]						=						bs[:]
					UpperCAmelCase_   :				Union[str, Any]						=						0
					for b in range(2**8  ):
										if b not in bs:
															bs.append(__snake_case  )
															cs.append(2**8 + n  )
															n += 1
					UpperCAmelCase_   :				Union[str, Any]						=						[chr(__snake_case  ) for n in cs]
					return dict(zip(__snake_case					,	__snake_case  )  )




def 					lowercase__				(					__snake_case   :							Optional[Any]  ):


					'''simple docstring'''


					UpperCAmelCase_   :				List[str]						=						set()
					UpperCAmelCase_   :				Tuple						=						word[0]
					for char in word[1:]:
										pairs.add((prev_char, char)  )
										UpperCAmelCase_   :				int						=						char
					return pairs






class   lowerCamelCase				(_snake_case      ):







		'''simple docstring'''



		_snake_case   :   Any							=  VOCAB_FILES_NAMES
		_snake_case   :   Optional[Any]							=  PRETRAINED_VOCAB_FILES_MAP
		_snake_case   :   Optional[Any]							=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		_snake_case   :   Dict							=  ['''input_ids''', '''attention_mask''']



		def __init__(							self    ,		_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase="replace"    ,		_UpperCamelCase="<s>"    ,		_UpperCamelCase="</s>"    ,		_UpperCamelCase="</s>"    ,		_UpperCamelCase="<s>"    ,		_UpperCamelCase="<unk>"    ,		_UpperCamelCase="<pad>"    ,		_UpperCamelCase="<mask>"    ,		_UpperCamelCase=False    ,		**_UpperCamelCase    ,		)     ->							Optional[Any]:
							UpperCAmelCase_   :				str						=						AddedToken(_UpperCamelCase    ,		lstrip=_UpperCamelCase    ,		rstrip=_UpperCamelCase       ) if isinstance(_UpperCamelCase    ,		_UpperCamelCase       ) else bos_token
							UpperCAmelCase_   :				List[Any]						=						AddedToken(_UpperCamelCase    ,		lstrip=_UpperCamelCase    ,		rstrip=_UpperCamelCase       ) if isinstance(_UpperCamelCase    ,		_UpperCamelCase       ) else eos_token
							UpperCAmelCase_   :				Dict						=						AddedToken(_UpperCamelCase    ,		lstrip=_UpperCamelCase    ,		rstrip=_UpperCamelCase       ) if isinstance(_UpperCamelCase    ,		_UpperCamelCase       ) else sep_token
							UpperCAmelCase_   :				Optional[Any]						=						AddedToken(_UpperCamelCase    ,		lstrip=_UpperCamelCase    ,		rstrip=_UpperCamelCase       ) if isinstance(_UpperCamelCase    ,		_UpperCamelCase       ) else cls_token
							UpperCAmelCase_   :				Union[str, Any]						=						AddedToken(_UpperCamelCase    ,		lstrip=_UpperCamelCase    ,		rstrip=_UpperCamelCase       ) if isinstance(_UpperCamelCase    ,		_UpperCamelCase       ) else unk_token
							UpperCAmelCase_   :				List[str]						=						AddedToken(_UpperCamelCase    ,		lstrip=_UpperCamelCase    ,		rstrip=_UpperCamelCase       ) if isinstance(_UpperCamelCase    ,		_UpperCamelCase       ) else pad_token

							# Mask token behave like a normal word, i.e. include the space before it
							UpperCAmelCase_   :				Optional[int]						=						AddedToken(_UpperCamelCase    ,		lstrip=_UpperCamelCase    ,		rstrip=_UpperCamelCase       ) if isinstance(_UpperCamelCase    ,		_UpperCamelCase       ) else mask_token

							super().__init__(
							    errors=_UpperCamelCase    ,		bos_token=_UpperCamelCase    ,		eos_token=_UpperCamelCase    ,		unk_token=_UpperCamelCase    ,		sep_token=_UpperCamelCase    ,		cls_token=_UpperCamelCase    ,		pad_token=_UpperCamelCase    ,		mask_token=_UpperCamelCase    ,		add_prefix_space=_UpperCamelCase    ,		**_UpperCamelCase    ,		)

							with open(_UpperCamelCase    ,		encoding='utf-8'       ) as vocab_handle:
												UpperCAmelCase_   :				List[str]						=						json.load(_UpperCamelCase       )
							UpperCAmelCase_   :				List[Any]						=						{v: k for k, v in self.encoder.items()}
							UpperCAmelCase_   :				str						=						errors  # how to handle errors in decoding
							UpperCAmelCase_   :				Optional[Any]						=						bytes_to_unicode()
							UpperCAmelCase_   :				List[str]						=						{v: k for k, v in self.byte_encoder.items()}
							with open(_UpperCamelCase    ,		encoding='utf-8'       ) as merges_handle:
												UpperCAmelCase_   :				Optional[Any]						=						merges_handle.read().split('\n'       )[1:-1]
							UpperCAmelCase_   :				Dict						=						[tuple(merge.split()       ) for merge in bpe_merges]
							UpperCAmelCase_   :				int						=						dict(zip(_UpperCamelCase    ,		range(len(_UpperCamelCase       )       )       )       )
							UpperCAmelCase_   :				Union[str, Any]						=						{}
							UpperCAmelCase_   :				Tuple						=						add_prefix_space

							# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
							UpperCAmelCase_   :				int						=						re.compile(r'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'       )



		@property
		def  __UpperCAmelCase      (							self       )     ->							Tuple:
							return len(self.encoder       )



		def  __UpperCAmelCase      (							self       )     ->							List[str]:
							return dict(self.encoder    ,		**self.added_tokens_encoder       )



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase       )     ->							str:
							if token in self.cache:
												return self.cache[token]
							UpperCAmelCase_   :				Dict						=						tuple(_UpperCamelCase       )
							UpperCAmelCase_   :				Any						=						get_pairs(_UpperCamelCase       )

							if not pairs:
												return token

							while True:
												UpperCAmelCase_   :				Union[str, Any]						=						min(_UpperCamelCase    ,		key=lambda _UpperCamelCase       : self.bpe_ranks.get(_UpperCamelCase    ,		float('inf'       )       )       )
												if bigram not in self.bpe_ranks:
																	break
												UpperCAmelCase_	,  UpperCAmelCase_   :				str						=						bigram
												UpperCAmelCase_   :				Union[str, Any]						=						[]
												UpperCAmelCase_   :				List[str]						=						0
												while i < len(_UpperCamelCase       ):
																	try:
																						UpperCAmelCase_   :				Optional[Any]						=						word.index(_UpperCamelCase    ,		_UpperCamelCase       )
																	except ValueError:
																						new_word.extend(word[i:]       )
																						break
																	else:
																						new_word.extend(word[i:j]       )
																						UpperCAmelCase_   :				str						=						j

																	if word[i] == first and i < len(_UpperCamelCase       ) - 1 and word[i + 1] == second:
																						new_word.append(first + second       )
																						i += 2
																	else:
																						new_word.append(word[i]       )
																						i += 1
												UpperCAmelCase_   :				Dict						=						tuple(_UpperCamelCase       )
												UpperCAmelCase_   :				Dict						=						new_word
												if len(_UpperCamelCase       ) == 1:
																	break
												else:
																	UpperCAmelCase_   :				Optional[Any]						=						get_pairs(_UpperCamelCase       )
							UpperCAmelCase_   :				Any						=						' '.join(_UpperCamelCase       )
							UpperCAmelCase_   :				Tuple						=						word
							return word



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase       )     ->							int:
							UpperCAmelCase_   :				Any						=						[]
							for token in re.findall(self.pat    ,		_UpperCamelCase       ):
												UpperCAmelCase_   :				Any						=						''.join(
												    self.byte_encoder[b] for b in token.encode('utf-8'       )       )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
												bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_UpperCamelCase       ).split(' '       )       )
							return bpe_tokens



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase       )     ->							Dict:
							return self.encoder.get(_UpperCamelCase    ,		self.encoder.get(self.unk_token       )       )



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase       )     ->							int:
							return self.decoder.get(_UpperCamelCase       )



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase       )     ->							int:
							UpperCAmelCase_   :				List[str]						=						''.join(_UpperCamelCase       )
							UpperCAmelCase_   :				List[Any]						=						bytearray([self.byte_decoder[c] for c in text]       ).decode('utf-8'    ,		errors=self.errors       )
							return text



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase    ,		_UpperCamelCase = None       )     ->							Tuple[str]:
							if not os.path.isdir(_UpperCamelCase       ):
												logger.error(f"Vocabulary path ({save_directory}) should be a directory"       )
												return
							UpperCAmelCase_   :				int						=						os.path.join(
							    _UpperCamelCase    ,		(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']       )
							UpperCAmelCase_   :				Union[str, Any]						=						os.path.join(
							    _UpperCamelCase    ,		(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']       )

							with open(_UpperCamelCase    ,		'w'    ,		encoding='utf-8'       ) as f:
												f.write(json.dumps(self.encoder    ,		indent=2    ,		sort_keys=_UpperCamelCase    ,		ensure_ascii=_UpperCamelCase       ) + '\n'       )

							UpperCAmelCase_   :				Dict						=						0
							with open(_UpperCamelCase    ,		'w'    ,		encoding='utf-8'       ) as writer:
												writer.write('#version: 0.2\n'       )
												for bpe_tokens, token_index in sorted(self.bpe_ranks.items()    ,		key=lambda _UpperCamelCase       : kv[1]       ):
																	if index != token_index:
																						logger.warning(
																						    f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
																						    ' Please check that the tokenizer is not corrupted!'       )
																						UpperCAmelCase_   :				Any						=						token_index
																	writer.write(' '.join(_UpperCamelCase       ) + '\n'       )
																	index += 1

							return vocab_file, merge_file



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase    ,		_UpperCamelCase = None       )     ->							List[int]:
							if token_ids_a is None:
												return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
							UpperCAmelCase_   :				int						=						[self.cls_token_id]
							UpperCAmelCase_   :				Any						=						[self.sep_token_id]
							return cls + token_ids_a + sep + sep + token_ids_a + sep



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase    ,		_UpperCamelCase = None    ,		_UpperCamelCase = False       )     ->							List[int]:
							if already_has_special_tokens:
												return super().get_special_tokens_mask(
												    token_ids_a=_UpperCamelCase    ,		token_ids_a=_UpperCamelCase    ,		already_has_special_tokens=_UpperCamelCase       )

							if token_ids_a is None:
												return [1] + ([0] * len(_UpperCamelCase       )) + [1]
							return [1] + ([0] * len(_UpperCamelCase       )) + [1, 1] + ([0] * len(_UpperCamelCase       )) + [1]



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase    ,		_UpperCamelCase = None       )     ->							List[int]:
							UpperCAmelCase_   :				int						=						[self.sep_token_id]
							UpperCAmelCase_   :				Tuple						=						[self.cls_token_id]

							if token_ids_a is None:
												return len(cls + token_ids_a + sep       ) * [0]
							return len(cls + token_ids_a + sep + sep + token_ids_a + sep       ) * [0]



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase    ,		_UpperCamelCase=False    ,		**_UpperCamelCase       )     ->							Tuple:
							UpperCAmelCase_   :				Optional[Any]						=						kwargs.pop('add_prefix_space'    ,		self.add_prefix_space       )
							if (is_split_into_words or add_prefix_space) and (len(_UpperCamelCase       ) > 0 and not text[0].isspace()):
												UpperCAmelCase_   :				Union[str, Any]						=						' ' + text
							return (text, kwargs)














from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxSeqaSeqConfigWithPast
from ...utils import logging


__UpperCAmelCase     							=						logging.get_logger(__name__)

__UpperCAmelCase     							=						{
    'google/umt5-small': 'https://huggingface.co/google/umt5-small/resolve/main/config.json',
    # See all umt5 models at https://huggingface.co/models?filter=umt5
}



class   lowerCamelCase				(_snake_case      ):







		'''simple docstring'''



		_snake_case   :   Union[str, Any]							=  '''umt5'''
		_snake_case   :   Union[str, Any]							=  ['''past_key_values''']



		def __init__(							self    ,		_UpperCamelCase=2_5_0_1_1_2    ,		_UpperCamelCase=5_1_2    ,		_UpperCamelCase=6_4    ,		_UpperCamelCase=1_0_2_4    ,		_UpperCamelCase=8    ,		_UpperCamelCase=None    ,		_UpperCamelCase=6    ,		_UpperCamelCase=3_2    ,		_UpperCamelCase=1_2_8    ,		_UpperCamelCase=0.1    ,		_UpperCamelCase=1E-6    ,		_UpperCamelCase=1.0    ,		_UpperCamelCase="gated-gelu"    ,		_UpperCamelCase=True    ,		_UpperCamelCase=True    ,		_UpperCamelCase="T5Tokenizer"    ,		_UpperCamelCase=True    ,		_UpperCamelCase=0    ,		_UpperCamelCase=1    ,		_UpperCamelCase=0    ,		**_UpperCamelCase    ,		)     ->							List[Any]:
							super().__init__(
							    is_encoder_decoder=_UpperCamelCase    ,		tokenizer_class=_UpperCamelCase    ,		tie_word_embeddings=_UpperCamelCase    ,		pad_token_id=_UpperCamelCase    ,		eos_token_id=_UpperCamelCase    ,		decoder_start_token_id=_UpperCamelCase    ,		**_UpperCamelCase    ,		)
							UpperCAmelCase_   :				Union[str, Any]						=						vocab_size
							UpperCAmelCase_   :				List[str]						=						d_model
							UpperCAmelCase_   :				Any						=						d_kv
							UpperCAmelCase_   :				Optional[int]						=						d_ff
							UpperCAmelCase_   :				List[Any]						=						num_layers
							UpperCAmelCase_   :				Optional[Any]						=						(
							    num_decoder_layers if num_decoder_layers is not None else self.num_layers
							)  # default = symmetry
							UpperCAmelCase_   :				Optional[int]						=						num_heads
							UpperCAmelCase_   :				Optional[int]						=						relative_attention_num_buckets
							UpperCAmelCase_   :				Dict						=						relative_attention_max_distance
							UpperCAmelCase_   :				Tuple						=						dropout_rate
							UpperCAmelCase_   :				Union[str, Any]						=						layer_norm_epsilon
							UpperCAmelCase_   :				Optional[int]						=						initializer_factor
							UpperCAmelCase_   :				List[str]						=						feed_forward_proj
							UpperCAmelCase_   :				Any						=						use_cache

							UpperCAmelCase_   :				List[Any]						=						self.feed_forward_proj.split('-'       )
							UpperCAmelCase_   :				List[Any]						=						act_info[-1]
							UpperCAmelCase_   :				Union[str, Any]						=						act_info[0] == 'gated'

							if len(_UpperCamelCase       ) > 1 and act_info[0] != "gated" or len(_UpperCamelCase       ) > 2:
												raise ValueError(
												    f"`feed_forward_proj`: {feed_forward_proj} is not a valid activation function of the dense layer."
												    'Please make sure `feed_forward_proj` is of the format `gated-{ACT_FN}` or `{ACT_FN}`, e.g. '
												    '\'gated-gelu\' or \'relu\''       )

							if feed_forward_proj == "gated-gelu":
												UpperCAmelCase_   :				Optional[int]						=						'gelu_new'



		@property
		def  __UpperCAmelCase      (							self       )     ->							int:
							return self.d_model



		@property
		def  __UpperCAmelCase      (							self       )     ->							Any:
							return self.num_heads



		@property
		def  __UpperCAmelCase      (							self       )     ->							List[Any]:
							return self.num_layers





class   lowerCamelCase				(_snake_case      ):







		'''simple docstring'''



		@property
		# Copied from transformers.models.t5.configuration_t5.T5OnnxConfig.inputs
		def  __UpperCAmelCase      (							self       )     ->							Mapping[str, Mapping[int, str]]:
							UpperCAmelCase_   :				str						=						{
							    'input_ids': {0: 'batch', 1: 'encoder_sequence'},
							    'attention_mask': {0: 'batch', 1: 'encoder_sequence'},
							}
							if self.use_past:
												UpperCAmelCase_   :				Optional[int]						=						'past_encoder_sequence + sequence'
												UpperCAmelCase_   :				str						=						{0: 'batch'}
												UpperCAmelCase_   :				Optional[int]						=						{0: 'batch', 1: 'past_decoder_sequence + sequence'}
							else:
												UpperCAmelCase_   :				Optional[int]						=						{0: 'batch', 1: 'decoder_sequence'}
												UpperCAmelCase_   :				Union[str, Any]						=						{0: 'batch', 1: 'decoder_sequence'}

							if self.use_past:
												self.fill_with_past_key_values_(_UpperCamelCase    ,		direction='inputs'       )

							return common_inputs



		@property
		# Copied from transformers.models.t5.configuration_t5.T5OnnxConfig.default_onnx_opset
		def  __UpperCAmelCase      (							self       )     ->							int:
							return 1_3



		@property
		def  __UpperCAmelCase      (							self       )     ->							float:
							return 5E-4






