





import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bart import BartTokenizer


_SCREAMING_SNAKE_CASE 		=						logging.get_logger(__name__)


_SCREAMING_SNAKE_CASE 		=						{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

# See all BART models at https://huggingface.co/models?filter=bart
_SCREAMING_SNAKE_CASE 		=						{
    """vocab_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/vocab.json""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/vocab.json""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/merges.txt""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/merges.txt""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/tokenizer.json""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/tokenizer.json""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/tokenizer.json""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/tokenizer.json""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/tokenizer.json""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/tokenizer.json""",
    },
}

_SCREAMING_SNAKE_CASE 		=						{
    """facebook/bart-base""": 10_24,
    """facebook/bart-large""": 10_24,
    """facebook/bart-large-mnli""": 10_24,
    """facebook/bart-large-cnn""": 10_24,
    """facebook/bart-large-xsum""": 10_24,
    """yjernite/bart_eli5""": 10_24,
}
class SCREAMING_SNAKE_CASE_  (   snake_case_   ):
						__magic_name__:     List[str]	 =    VOCAB_FILES_NAMES
						__magic_name__:     Union[str, Any]	 =    PRETRAINED_VOCAB_FILES_MAP
						__magic_name__:     Union[str, Any]	 =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						__magic_name__:     Union[str, Any]	 =    ["input_ids", "attention_mask"]
						__magic_name__:     Any	 =    BartTokenizer






						def __init__( self : List[str]						,				_A : Optional[int]=None						,				_A : Optional[Any]=None						,				_A : Tuple=None						,				_A : Dict="replace"						,				_A : Any="<s>"						,				_A : Any="</s>"						,				_A : str="</s>"						,				_A : Optional[Any]="<s>"						,				_A : List[str]="<unk>"						,				_A : int="<pad>"						,				_A : Tuple="<mask>"						,				_A : List[str]=False						,				_A : str=True						,				**_A : Any						,				)							->      Dict:







							"""simple docstring"""


							super().__init__(
							    _A						,				_A						,				tokenizer_file=_A						,				errors=_A						,				bos_token=_A						,				eos_token=_A						,				sep_token=_A						,				cls_token=_A						,				unk_token=_A						,				pad_token=_A						,				mask_token=_A						,				add_prefix_space=_A						,				trim_offsets=_A						,				**_A						,				)

							snake_case_						:     Dict     =		json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()		)
							if pre_tok_state.get('add_prefix_space'						,				_A		) != add_prefix_space:
								snake_case_						:     Optional[Any]     =		getattr(_A						,				pre_tok_state.pop('type'		)		)
								snake_case_						:     int     =		add_prefix_space
								snake_case_						:     Optional[int]     =		pre_tok_class(**_A		)

							snake_case_						:     Any     =		add_prefix_space

							# the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
							snake_case_						:     List[str]     =		'post_processor'
							snake_case_						:     Union[str, Any]     =		getattr(self.backend_tokenizer						,				_A						,				_A		)
							if tokenizer_component_instance:
								snake_case_						:     List[Any]     =		json.loads(tokenizer_component_instance.__getstate__()		)

								# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
								if "sep" in state:
									snake_case_						:     Union[str, Any]     =		tuple(state['sep']		)
								if "cls" in state:
									snake_case_						:     int     =		tuple(state['cls']		)

								snake_case_						:     Optional[int]     =		False

								if state.get('add_prefix_space'						,				_A		) != add_prefix_space:
									snake_case_						:     Any     =		add_prefix_space
									snake_case_						:     Optional[Any]     =		True

								if state.get('trim_offsets'						,				_A		) != trim_offsets:
									snake_case_						:     Optional[int]     =		trim_offsets
									snake_case_						:     str     =		True

								if changes_to_apply:
									snake_case_						:     List[Any]     =		getattr(_A						,				state.pop('type'		)		)
									snake_case_						:     Optional[int]     =		component_class(**_A		)
									setattr(self.backend_tokenizer						,				_A						,				_A		)






						@property
						def UpperCAmelCase_			( self : List[str]		)							->      str:







							"""simple docstring"""


							if self._mask_token is None:
								if self.verbose:
									logger.error('Using mask_token, but it is not set yet.'		)
								return None
							return str(self._mask_token		)






						@mask_token.setter
						def UpperCAmelCase_			( self : Optional[int]						,				_A : Tuple		)							->      Tuple:







							"""simple docstring"""


							snake_case_						:     Tuple     =		AddedToken(_A						,				lstrip=_A						,				rstrip=_A		) if isinstance(_A						,				_A		) else value
							snake_case_						:     List[str]     =		value






						def UpperCAmelCase_			( self : Union[str, Any]						,				*_A : List[Any]						,				**_A : Union[str, Any]		)							->      BatchEncoding:







							"""simple docstring"""


							snake_case_						:     Union[str, Any]     =		kwargs.get('is_split_into_words'						,				_A		)

							if is_split_into_words and not self.add_prefix_space:
								raise ValueError(
								    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
								    'to use it with pretokenized inputs.'		)

							return super()._batch_encode_plus(*_A						,				**_A		)






						def UpperCAmelCase_			( self : Union[str, Any]						,				*_A : Optional[Any]						,				**_A : str		)							->      BatchEncoding:







							"""simple docstring"""


							snake_case_						:     List[Any]     =		kwargs.get('is_split_into_words'						,				_A		)

							if is_split_into_words and not self.add_prefix_space:
								raise ValueError(
								    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
								    'to use it with pretokenized inputs.'		)

							return super()._encode_plus(*_A						,				**_A		)






						def UpperCAmelCase_			( self : Optional[Any]						,				_A : str						,				_A : Optional[str] = None		)							->      Tuple[str]:







							"""simple docstring"""


							snake_case_						:     List[str]     =		self._tokenizer.model.save(_A						,				name=_A		)
							return tuple(_A		)






						def UpperCAmelCase_			( self : List[Any]						,				_A : int						,				_A : str=None		)							->      Optional[Any]:







							"""simple docstring"""


							snake_case_						:     int     =		[self.bos_token_id] + token_ids_a + [self.eos_token_id]
							if token_ids_a is None:
								return output

							return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]






						def UpperCAmelCase_			( self : Any						,				_A : List[int]						,				_A : Optional[List[int]] = None		)							->      List[int]:







							"""simple docstring"""


							snake_case_						:     List[Any]     =		[self.sep_token_id]
							snake_case_						:     Tuple     =		[self.cls_token_id]

							if token_ids_a is None:
								return len(cls + token_ids_a + sep		) * [0]
							return len(cls + token_ids_a + sep + sep + token_ids_a + sep		) * [0]







from __future__ import annotations

import unittest

from transformers import AutoTokenizer, MBartConfig, is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
						import tensorflow as tf

						from transformers import TFAutoModelForSeqaSeqLM, TFMBartForConditionalGeneration, TFMBartModel
@require_tf
class SCREAMING_SNAKE_CASE_  :
						__magic_name__:     int	 =    MBartConfig
						__magic_name__:     str	 =    {}
						__magic_name__:     Union[str, Any]	 =    "gelu"






						def __init__( self : List[str]						,				_A : Optional[int]						,				_A : List[Any]=13						,				_A : List[Any]=7						,				_A : Dict=True						,				_A : Tuple=False						,				_A : Optional[Any]=99						,				_A : Dict=32						,				_A : str=2						,				_A : str=4						,				_A : Tuple=37						,				_A : Tuple=0.1						,				_A : Union[str, Any]=0.1						,				_A : Optional[int]=20						,				_A : Dict=2						,				_A : List[str]=1						,				_A : Union[str, Any]=0						,				)							->      List[Any]:







							"""simple docstring"""


							snake_case_						:     str     =		parent
							snake_case_						:     List[str]     =		batch_size
							snake_case_						:     List[str]     =		seq_length
							snake_case_						:     Union[str, Any]     =		is_training
							snake_case_						:     Optional[int]     =		use_labels
							snake_case_						:     Dict     =		vocab_size
							snake_case_						:     Union[str, Any]     =		hidden_size
							snake_case_						:     str     =		num_hidden_layers
							snake_case_						:     Optional[Any]     =		num_attention_heads
							snake_case_						:     Union[str, Any]     =		intermediate_size
							snake_case_						:     Any     =		hidden_dropout_prob
							snake_case_						:     Any     =		attention_probs_dropout_prob
							snake_case_						:     Optional[Any]     =		max_position_embeddings
							snake_case_						:     Optional[Any]     =		eos_token_id
							snake_case_						:     Tuple     =		pad_token_id
							snake_case_						:     int     =		bos_token_id






						def UpperCAmelCase_			( self : List[str]		)							->      Tuple:







							"""simple docstring"""


							snake_case_						:     List[Any]     =		ids_tensor([self.batch_size, self.seq_length - 1]						,				self.vocab_size		)
							snake_case_						:     int     =		tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size		)						,				1		)
							snake_case_						:     Dict     =		tf.concat([input_ids, eos_tensor]						,				axis=1		)

							snake_case_						:     Any     =		ids_tensor([self.batch_size, self.seq_length]						,				self.vocab_size		)

							snake_case_						:     Optional[Any]     =		self.config_cls(
							    vocab_size=self.vocab_size						,				d_model=self.hidden_size						,				encoder_layers=self.num_hidden_layers						,				decoder_layers=self.num_hidden_layers						,				encoder_attention_heads=self.num_attention_heads						,				decoder_attention_heads=self.num_attention_heads						,				encoder_ffn_dim=self.intermediate_size						,				decoder_ffn_dim=self.intermediate_size						,				dropout=self.hidden_dropout_prob						,				attention_dropout=self.attention_probs_dropout_prob						,				max_position_embeddings=self.max_position_embeddings						,				eos_token_ids=[2]						,				bos_token_id=self.bos_token_id						,				pad_token_id=self.pad_token_id						,				decoder_start_token_id=self.pad_token_id						,				**self.config_updates						,				)
							snake_case_						:     Union[str, Any]     =		prepare_mbart_inputs_dict(_A						,				_A						,				_A		)
							return config, inputs_dict






						def UpperCAmelCase_			( self : Optional[Any]						,				_A : Optional[Any]						,				_A : int		)							->      str:







							"""simple docstring"""


							snake_case_						:     Dict     =		TFMBartModel(config=_A		).get_decoder()
							snake_case_						:     Any     =		inputs_dict['input_ids']

							snake_case_						:     List[Any]     =		input_ids[:1, :]
							snake_case_						:     Dict     =		inputs_dict['attention_mask'][:1, :]
							snake_case_						:     Tuple     =		inputs_dict['head_mask']
							snake_case_						:     List[Any]     =		1

							# first forward pass
							snake_case_						:     Any     =		model(_A						,				attention_mask=_A						,				head_mask=_A						,				use_cache=_A		)

							snake_case_  ,snake_case_						:     str     =		outputs.to_tuple()
							snake_case_						:     int     =		past_key_values[1]


def 							SCREAMING_SNAKE_CASE__  (      __a				,	__a				,	__a				,	__a=None				,	__a=None				,	__a=None				,	__a=None				,	__a=None				,	):
	if attention_mask is None:
		snake_case_						:     Optional[int]     =		tf.cast(tf.math.not_equal(__a				,	config.pad_token_id				)				,	tf.inta				)
	if decoder_attention_mask is None:
		snake_case_						:     str     =		tf.concat(
		    [
		        tf.ones(decoder_input_ids[:, :1].shape				,	dtype=tf.inta				),
		        tf.cast(tf.math.not_equal(decoder_input_ids[:, 1:]				,	config.pad_token_id				)				,	tf.inta				),
		    ]				,	axis=-1				,	)
	if head_mask is None:
		snake_case_						:     str     =		tf.ones((config.encoder_layers, config.encoder_attention_heads)				)
	if decoder_head_mask is None:
		snake_case_						:     int     =		tf.ones((config.decoder_layers, config.decoder_attention_heads)				)
	if cross_attn_head_mask is None:
		snake_case_						:     Dict     =		tf.ones((config.decoder_layers, config.decoder_attention_heads)				)
	return {
	    "input_ids": input_ids,
	    "decoder_input_ids": decoder_input_ids,
	    "attention_mask": attention_mask,
	    "decoder_attention_mask": decoder_attention_mask,
	    "head_mask": head_mask,
	    "decoder_head_mask": decoder_head_mask,
	    "cross_attn_head_mask": cross_attn_head_mask,
	}
@require_tf
class SCREAMING_SNAKE_CASE_  (   snake_case_    ,			snake_case_    ,			unittest.TestCase   ):
						__magic_name__:     Tuple	 =    (TFMBartForConditionalGeneration, TFMBartModel) if is_tf_available() else ()
						__magic_name__:     int	 =    (TFMBartForConditionalGeneration,) if is_tf_available() else ()
						__magic_name__:     Union[str, Any]	 =    (
						    {
						        "conversational": TFMBartForConditionalGeneration,
						        "feature-extraction": TFMBartModel,
						        "summarization": TFMBartForConditionalGeneration,
						        "text2text-generation": TFMBartForConditionalGeneration,
						        "translation": TFMBartForConditionalGeneration,
						    }
						    if is_tf_available()
						    else {}
						)
						__magic_name__:     Tuple	 =    True
						__magic_name__:     Tuple	 =    False
						__magic_name__:     Any	 =    False






						def UpperCAmelCase_			( self : Any						,				_A : Union[str, Any]						,				_A : List[Any]						,				_A : str						,				_A : int						,				_A : Dict		)							->      Union[str, Any]:







							"""simple docstring"""


							if pipeline_test_casse_name != "FeatureExtractionPipelineTests":
								# Exception encountered when calling layer '...'
								return True

							return False






						def UpperCAmelCase_			( self : Dict		)							->      Tuple:







							"""simple docstring"""


							snake_case_						:     Optional[Any]     =		TFMBartModelTester(self		)
							snake_case_						:     List[Any]     =		ConfigTester(self						,				config_class=_A		)






						def UpperCAmelCase_			( self : Optional[Any]		)							->      str:







							"""simple docstring"""


							self.config_tester.run_common_tests()






						def UpperCAmelCase_			( self : Union[str, Any]		)							->      Tuple:







							"""simple docstring"""


							snake_case_						:     Dict     =		self.model_tester.prepare_config_and_inputs_for_common()
							self.model_tester.check_decoder_model_past_large_inputs(*_A		)
@require_sentencepiece
@require_tokenizers
@require_tf
class SCREAMING_SNAKE_CASE_  (   unittest.TestCase   ):
						__magic_name__:     Optional[int]	 =    [
						    " UN Chief Says There Is No Military Solution in Syria",
						]
						__magic_name__:     Union[str, Any]	 =    [
						    "Şeful ONU declară că nu există o soluţie militară în Siria",
						]
						__magic_name__:     List[Any]	 =    "facebook/mbart-large-en-ro"






						@cached_property
						def UpperCAmelCase_			( self : str		)							->      List[Any]:







							"""simple docstring"""


							return AutoTokenizer.from_pretrained(self.model_name		)






						@cached_property
						def UpperCAmelCase_			( self : List[Any]		)							->      Any:







							"""simple docstring"""


							snake_case_						:     List[str]     =		TFAutoModelForSeqaSeqLM.from_pretrained(self.model_name		)
							return model






						def UpperCAmelCase_			( self : Optional[int]						,				**_A : str		)							->      int:







							"""simple docstring"""


							snake_case_						:     List[str]     =		self.translate_src_text(**_A		)
							self.assertListEqual(self.expected_text						,				_A		)






						def UpperCAmelCase_			( self : Union[str, Any]						,				**_A : Dict		)							->      int:







							"""simple docstring"""


							snake_case_						:     Optional[Any]     =		self.tokenizer(self.src_text						,				**_A						,				return_tensors='tf'		)
							snake_case_						:     int     =		self.model.generate(
							    model_inputs.input_ids						,				attention_mask=model_inputs.attention_mask						,				num_beams=2		)
							snake_case_						:     Any     =		self.tokenizer.batch_decode(_A						,				skip_special_tokens=_A		)
							return generated_words






						@slow
						def UpperCAmelCase_			( self : str		)							->      List[str]:







							"""simple docstring"""


							self._assert_generated_batch_equal_expected()

