






import argparse
import collections
import json
import os
import re
import string
import sys

import numpy as np


__UpperCAmelCase     							=						re.compile(R'\b(a|an|the)\b', re.UNICODE)

__UpperCAmelCase     							=						None




def 					lowercase__				(					):


					'''simple docstring'''


					UpperCAmelCase_   :				Optional[int]						=						argparse.ArgumentParser('Official evaluation script for SQuAD version 2.0.'  )
					parser.add_argument('data_file'					,	metavar='data.json'					,	help='Input data JSON file.'  )
					parser.add_argument('pred_file'					,	metavar='pred.json'					,	help='Model predictions.'  )
					parser.add_argument(
					    '--out-file'					,	'-o'					,	metavar='eval.json'					,	help='Write accuracy metrics to file (default is stdout).'  )
					parser.add_argument(
					    '--na-prob-file'					,	'-n'					,	metavar='na_prob.json'					,	help='Model estimates of probability of no answer.'  )
					parser.add_argument(
					    '--na-prob-thresh'					,	'-t'					,	type=__snake_case					,	default=1.0					,	help='Predict "" if no-answer probability exceeds this (default = 1.0).'					,	)
					parser.add_argument(
					    '--out-image-dir'					,	'-p'					,	metavar='out_images'					,	default=__snake_case					,	help='Save precision-recall curves to directory.'  )
					parser.add_argument('--verbose'					,	'-v'					,	action='store_true'  )
					if len(sys.argv  ) == 1:
										parser.print_help()
										sys.exit(1  )
					return parser.parse_args()




def 					lowercase__				(					__snake_case   :							int  ):


					'''simple docstring'''


					UpperCAmelCase_   :				List[str]						=						{}
					for article in dataset:
										for p in article["paragraphs"]:
															for qa in p["qas"]:
																				UpperCAmelCase_   :				Dict						=						bool(qa['answers']['text']  )
					return qid_to_has_ans




def 					lowercase__				(					__snake_case   :							Tuple  ):


					'''simple docstring'''


					def remove_articles(__snake_case   :							Any  ):
										return ARTICLES_REGEX.sub(' '					,	__snake_case  )

					def white_space_fix(__snake_case   :							str  ):
										return " ".join(text.split()  )

					def remove_punc(__snake_case   :							Optional[int]  ):
										UpperCAmelCase_   :				Optional[Any]						=						set(string.punctuation  )
										return "".join(ch for ch in text if ch not in exclude  )

					def lower(__snake_case   :							str  ):
										return text.lower()

					return white_space_fix(remove_articles(remove_punc(lower(__snake_case  )  )  )  )




def 					lowercase__				(					__snake_case   :							Optional[int]  ):


					'''simple docstring'''


					if not s:
										return []
					return normalize_answer(__snake_case  ).split()




def 					lowercase__				(					__snake_case   :							Any					,	__snake_case   :							Optional[int]  ):


					'''simple docstring'''


					return int(normalize_answer(__snake_case  ) == normalize_answer(__snake_case  )  )




def 					lowercase__				(					__snake_case   :							Optional[int]					,	__snake_case   :							Any  ):


					'''simple docstring'''


					UpperCAmelCase_   :				str						=						get_tokens(__snake_case  )
					UpperCAmelCase_   :				Tuple						=						get_tokens(__snake_case  )
					UpperCAmelCase_   :				List[Any]						=						collections.Counter(__snake_case  ) & collections.Counter(__snake_case  )
					UpperCAmelCase_   :				str						=						sum(common.values()  )
					if len(__snake_case  ) == 0 or len(__snake_case  ) == 0:
										# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
										return int(gold_toks == pred_toks  )
					if num_same == 0:
										return 0
					UpperCAmelCase_   :				Dict						=						1.0 * num_same / len(__snake_case  )
					UpperCAmelCase_   :				Any						=						1.0 * num_same / len(__snake_case  )
					UpperCAmelCase_   :				Optional[Any]						=						(2 * precision * recall) / (precision + recall)
					return fa




def 					lowercase__				(					__snake_case   :							int					,	__snake_case   :							Dict  ):


					'''simple docstring'''


					UpperCAmelCase_   :				Union[str, Any]						=						{}
					UpperCAmelCase_   :				List[str]						=						{}
					for article in dataset:
										for p in article["paragraphs"]:
															for qa in p["qas"]:
																				UpperCAmelCase_   :				List[Any]						=						qa['id']
																				UpperCAmelCase_   :				int						=						[t for t in qa['answers']['text'] if normalize_answer(__snake_case  )]
																				if not gold_answers:
																									# For unanswerable questions, only correct answer is empty string
																									UpperCAmelCase_   :				Any						=						['']
																				if qid not in preds:
																									print(F"Missing prediction for {qid}"  )
																									continue
																				UpperCAmelCase_   :				Optional[int]						=						preds[qid]
																				# Take max over all gold answers
																				UpperCAmelCase_   :				Union[str, Any]						=						max(compute_exact(__snake_case					,	__snake_case  ) for a in gold_answers  )
																				UpperCAmelCase_   :				Tuple						=						max(compute_fa(__snake_case					,	__snake_case  ) for a in gold_answers  )
					return exact_scores, fa_scores




def 					lowercase__				(					__snake_case   :							List[str]					,	__snake_case   :							Union[str, Any]					,	__snake_case   :							Any					,	__snake_case   :							Any  ):


					'''simple docstring'''


					UpperCAmelCase_   :				str						=						{}
					for qid, s in scores.items():
										UpperCAmelCase_   :				Optional[Any]						=						na_probs[qid] > na_prob_thresh
										if pred_na:
															UpperCAmelCase_   :				int						=						float(not qid_to_has_ans[qid]  )
										else:
															UpperCAmelCase_   :				str						=						s
					return new_scores




def 					lowercase__				(					__snake_case   :							Any					,	__snake_case   :							List[Any]					,	__snake_case   :							Optional[int]=None  ):


					'''simple docstring'''


					if not qid_list:
										UpperCAmelCase_   :				Any						=						len(__snake_case  )
										return collections.OrderedDict(
										    [
										        ('exact', 100.0 * sum(exact_scores.values()  ) / total),
										        ('f1', 100.0 * sum(fa_scores.values()  ) / total),
										        ('total', total),
										    ]  )
					else:
										UpperCAmelCase_   :				str						=						len(__snake_case  )
										return collections.OrderedDict(
										    [
										        ('exact', 100.0 * sum(exact_scores[k] for k in qid_list  ) / total),
										        ('f1', 100.0 * sum(fa_scores[k] for k in qid_list  ) / total),
										        ('total', total),
										    ]  )




def 					lowercase__				(					__snake_case   :							Optional[Any]					,	__snake_case   :							List[str]					,	__snake_case   :							Dict  ):


					'''simple docstring'''


					for k in new_eval:
										UpperCAmelCase_   :				Tuple						=						new_eval[k]




def 					lowercase__				(					__snake_case   :							Union[str, Any]					,	__snake_case   :							Any					,	__snake_case   :							List[str]					,	__snake_case   :							str  ):


					'''simple docstring'''


					plt.step(__snake_case					,	__snake_case					,	color='b'					,	alpha=0.2					,	where='post'  )
					plt.fill_between(__snake_case					,	__snake_case					,	step='post'					,	alpha=0.2					,	color='b'  )
					plt.xlabel('Recall'  )
					plt.ylabel('Precision'  )
					plt.xlim([0.0, 1.05]  )
					plt.ylim([0.0, 1.05]  )
					plt.title(__snake_case  )
					plt.savefig(__snake_case  )
					plt.clf()




def 					lowercase__				(					__snake_case   :							List[Any]					,	__snake_case   :							Optional[int]					,	__snake_case   :							int					,	__snake_case   :							Dict					,	__snake_case   :							Any=None					,	__snake_case   :							Optional[int]=None  ):


					'''simple docstring'''


					UpperCAmelCase_   :				Optional[Any]						=						sorted(__snake_case					,	key=lambda __snake_case  : na_probs[k]  )
					UpperCAmelCase_   :				int						=						0.0
					UpperCAmelCase_   :				Optional[int]						=						1.0
					UpperCAmelCase_   :				List[str]						=						0.0
					UpperCAmelCase_   :				Union[str, Any]						=						[1.0]
					UpperCAmelCase_   :				Optional[Any]						=						[0.0]
					UpperCAmelCase_   :				str						=						0.0
					for i, qid in enumerate(__snake_case  ):
										if qid_to_has_ans[qid]:
															true_pos += scores[qid]
										UpperCAmelCase_   :				Optional[Any]						=						true_pos / float(i + 1  )
										UpperCAmelCase_   :				Optional[Any]						=						true_pos / float(__snake_case  )
										if i == len(__snake_case  ) - 1 or na_probs[qid] != na_probs[qid_list[i + 1]]:
															# i.e., if we can put a threshold after this point
															avg_prec += cur_p * (cur_r - recalls[-1])
															precisions.append(__snake_case  )
															recalls.append(__snake_case  )
					if out_image:
										plot_pr_curve(__snake_case					,	__snake_case					,	__snake_case					,	__snake_case  )
					return {"ap": 100.0 * avg_prec}




def 					lowercase__				(					__snake_case   :							Optional[int]					,	__snake_case   :							Optional[Any]					,	__snake_case   :							str					,	__snake_case   :							Tuple					,	__snake_case   :							Union[str, Any]					,	__snake_case   :							Dict  ):


					'''simple docstring'''


					if out_image_dir and not os.path.exists(__snake_case  ):
										os.makedirs(__snake_case  )
					UpperCAmelCase_   :				Dict						=						sum(1 for v in qid_to_has_ans.values() if v  )
					if num_true_pos == 0:
										return
					UpperCAmelCase_   :				Union[str, Any]						=						make_precision_recall_eval(
					    __snake_case					,	__snake_case					,	__snake_case					,	__snake_case					,	out_image=os.path.join(__snake_case					,	'pr_exact.png'  )					,	title='Precision-Recall curve for Exact Match score'					,	)
					UpperCAmelCase_   :				Tuple						=						make_precision_recall_eval(
					    __snake_case					,	__snake_case					,	__snake_case					,	__snake_case					,	out_image=os.path.join(__snake_case					,	'pr_f1.png'  )					,	title='Precision-Recall curve for F1 score'					,	)
					UpperCAmelCase_   :				Optional[int]						=						{k: float(__snake_case  ) for k, v in qid_to_has_ans.items()}
					UpperCAmelCase_   :				Union[str, Any]						=						make_precision_recall_eval(
					    __snake_case					,	__snake_case					,	__snake_case					,	__snake_case					,	out_image=os.path.join(__snake_case					,	'pr_oracle.png'  )					,	title='Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)'					,	)
					merge_eval(__snake_case					,	__snake_case					,	'pr_exact'  )
					merge_eval(__snake_case					,	__snake_case					,	'pr_f1'  )
					merge_eval(__snake_case					,	__snake_case					,	'pr_oracle'  )




def 					lowercase__				(					__snake_case   :							Optional[Any]					,	__snake_case   :							Union[str, Any]					,	__snake_case   :							Tuple					,	__snake_case   :							Any  ):


					'''simple docstring'''


					if not qid_list:
										return
					UpperCAmelCase_   :				str						=						[na_probs[k] for k in qid_list]
					UpperCAmelCase_   :				Optional[Any]						=						np.ones_like(__snake_case  ) / float(len(__snake_case  )  )
					plt.hist(__snake_case					,	weights=__snake_case					,	bins=20					,	range=(0.0, 1.0)  )
					plt.xlabel('Model probability of no-answer'  )
					plt.ylabel('Proportion of dataset'  )
					plt.title(F"Histogram of no-answer probability: {name}"  )
					plt.savefig(os.path.join(__snake_case					,	F"na_prob_hist_{name}.png"  )  )
					plt.clf()




def 					lowercase__				(					__snake_case   :							List[str]					,	__snake_case   :							int					,	__snake_case   :							Dict					,	__snake_case   :							Union[str, Any]  ):


					'''simple docstring'''


					UpperCAmelCase_   :				List[str]						=						sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]  )
					UpperCAmelCase_   :				List[Any]						=						num_no_ans
					UpperCAmelCase_   :				Optional[Any]						=						cur_score
					UpperCAmelCase_   :				List[Any]						=						0.0
					UpperCAmelCase_   :				Dict						=						sorted(__snake_case					,	key=lambda __snake_case  : na_probs[k]  )
					for i, qid in enumerate(__snake_case  ):
										if qid not in scores:
															continue
										if qid_to_has_ans[qid]:
															UpperCAmelCase_   :				Union[str, Any]						=						scores[qid]
										else:
															if preds[qid]:
																				UpperCAmelCase_   :				int						=						-1
															else:
																				UpperCAmelCase_   :				str						=						0
										cur_score += diff
										if cur_score > best_score:
															UpperCAmelCase_   :				List[str]						=						cur_score
															UpperCAmelCase_   :				Union[str, Any]						=						na_probs[qid]
					return 100.0 * best_score / len(__snake_case  ), best_thresh




def 					lowercase__				(					__snake_case   :							Optional[Any]					,	__snake_case   :							Dict					,	__snake_case   :							Dict					,	__snake_case   :							Optional[Any]					,	__snake_case   :							int					,	__snake_case   :							Optional[Any]  ):


					'''simple docstring'''


					UpperCAmelCase_	,  UpperCAmelCase_   :				Dict						=						find_best_thresh(__snake_case					,	__snake_case					,	__snake_case					,	__snake_case  )
					UpperCAmelCase_	,  UpperCAmelCase_   :				List[str]						=						find_best_thresh(__snake_case					,	__snake_case					,	__snake_case					,	__snake_case  )
					UpperCAmelCase_   :				Optional[int]						=						best_exact
					UpperCAmelCase_   :				Union[str, Any]						=						exact_thresh
					UpperCAmelCase_   :				Dict						=						best_fa
					UpperCAmelCase_   :				Tuple						=						fa_thresh




def 					lowercase__				(					):


					'''simple docstring'''


					with open(OPTS.data_file  ) as f:
										UpperCAmelCase_   :				Optional[Any]						=						json.load(__snake_case  )
										UpperCAmelCase_   :				Any						=						dataset_json['data']
					with open(OPTS.pred_file  ) as f:
										UpperCAmelCase_   :				int						=						json.load(__snake_case  )
					if OPTS.na_prob_file:
										with open(OPTS.na_prob_file  ) as f:
															UpperCAmelCase_   :				int						=						json.load(__snake_case  )
					else:
										UpperCAmelCase_   :				List[str]						=						{k: 0.0 for k in preds}
					UpperCAmelCase_   :				List[Any]						=						make_qid_to_has_ans(__snake_case  )  # maps qid to True/False
					UpperCAmelCase_   :				Tuple						=						[k for k, v in qid_to_has_ans.items() if v]
					UpperCAmelCase_   :				str						=						[k for k, v in qid_to_has_ans.items() if not v]
					UpperCAmelCase_	,  UpperCAmelCase_   :				str						=						get_raw_scores(__snake_case					,	__snake_case  )
					UpperCAmelCase_   :				Any						=						apply_no_ans_threshold(__snake_case					,	__snake_case					,	__snake_case					,	OPTS.na_prob_thresh  )
					UpperCAmelCase_   :				Any						=						apply_no_ans_threshold(__snake_case					,	__snake_case					,	__snake_case					,	OPTS.na_prob_thresh  )
					UpperCAmelCase_   :				int						=						make_eval_dict(__snake_case					,	__snake_case  )
					if has_ans_qids:
										UpperCAmelCase_   :				Any						=						make_eval_dict(__snake_case					,	__snake_case					,	qid_list=__snake_case  )
										merge_eval(__snake_case					,	__snake_case					,	'HasAns'  )
					if no_ans_qids:
										UpperCAmelCase_   :				int						=						make_eval_dict(__snake_case					,	__snake_case					,	qid_list=__snake_case  )
										merge_eval(__snake_case					,	__snake_case					,	'NoAns'  )
					if OPTS.na_prob_file:
										find_all_best_thresh(__snake_case					,	__snake_case					,	__snake_case					,	__snake_case					,	__snake_case					,	__snake_case  )
					if OPTS.na_prob_file and OPTS.out_image_dir:
										run_precision_recall_analysis(__snake_case					,	__snake_case					,	__snake_case					,	__snake_case					,	__snake_case					,	OPTS.out_image_dir  )
										histogram_na_prob(__snake_case					,	__snake_case					,	OPTS.out_image_dir					,	'hasAns'  )
										histogram_na_prob(__snake_case					,	__snake_case					,	OPTS.out_image_dir					,	'noAns'  )
					if OPTS.out_file:
										with open(OPTS.out_file					,	'w'  ) as f:
															json.dump(__snake_case					,	__snake_case  )
					else:
										print(json.dumps(__snake_case					,	indent=2  )  )


if __name__ == "__main__":
		__UpperCAmelCase     							=						parse_args()
		if OPTS.out_image_dir:
				import matplotlib

				matplotlib.use('Agg')
				import matplotlib.pyplot as plt
		main()














import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


__UpperCAmelCase     							=						logging.get_logger(__name__)

__UpperCAmelCase     							=						'▁'

__UpperCAmelCase     							=						{'vocab_file': 'sentencepiece.bpe.model'}

__UpperCAmelCase     							=						{
    'vocab_file': {
        'facebook/mbart-large-en-ro': (
            'https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model'
        ),
        'facebook/mbart-large-cc25': (
            'https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model'
        ),
    }
}

__UpperCAmelCase     							=						{
    'facebook/mbart-large-en-ro': 1024,
    'facebook/mbart-large-cc25': 1024,
}

# fmt: off
__UpperCAmelCase     							=						['ar_AR', 'cs_CZ', 'de_DE', 'en_XX', 'es_XX', 'et_EE', 'fi_FI', 'fr_XX', 'gu_IN', 'hi_IN', 'it_IT', 'ja_XX', 'kk_KZ', 'ko_KR', 'lt_LT', 'lv_LV', 'my_MM', 'ne_NP', 'nl_XX', 'ro_RO', 'ru_RU', 'si_LK', 'tr_TR', 'vi_VN', 'zh_CN']



class   lowerCamelCase				(_snake_case      ):







		'''simple docstring'''



		_snake_case   :   List[Any]							=  VOCAB_FILES_NAMES
		_snake_case   :   Union[str, Any]							=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		_snake_case   :   Tuple							=  PRETRAINED_VOCAB_FILES_MAP
		_snake_case   :   Any							=  ['''input_ids''', '''attention_mask''']

		_snake_case   :   List[int]							=  []
		_snake_case   :   List[int]							=  []



		def __init__(							self    ,		_UpperCamelCase    ,		_UpperCamelCase="<s>"    ,		_UpperCamelCase="</s>"    ,		_UpperCamelCase="</s>"    ,		_UpperCamelCase="<s>"    ,		_UpperCamelCase="<unk>"    ,		_UpperCamelCase="<pad>"    ,		_UpperCamelCase="<mask>"    ,		_UpperCamelCase=None    ,		_UpperCamelCase=None    ,		_UpperCamelCase=None    ,		_UpperCamelCase = None    ,		_UpperCamelCase=None    ,		**_UpperCamelCase    ,		)     ->							List[str]:
							# Mask token behave like a normal word, i.e. include the space before it
							UpperCAmelCase_   :				List[Any]						=						AddedToken(_UpperCamelCase    ,		lstrip=_UpperCamelCase    ,		rstrip=_UpperCamelCase       ) if isinstance(_UpperCamelCase    ,		_UpperCamelCase       ) else mask_token

							UpperCAmelCase_   :				Optional[int]						=						{} if sp_model_kwargs is None else sp_model_kwargs

							super().__init__(
							    bos_token=_UpperCamelCase    ,		eos_token=_UpperCamelCase    ,		unk_token=_UpperCamelCase    ,		sep_token=_UpperCamelCase    ,		cls_token=_UpperCamelCase    ,		pad_token=_UpperCamelCase    ,		mask_token=_UpperCamelCase    ,		tokenizer_file=_UpperCamelCase    ,		src_lang=_UpperCamelCase    ,		tgt_lang=_UpperCamelCase    ,		additional_special_tokens=_UpperCamelCase    ,		sp_model_kwargs=self.sp_model_kwargs    ,		**_UpperCamelCase    ,		)

							UpperCAmelCase_   :				Any						=						spm.SentencePieceProcessor(**self.sp_model_kwargs       )
							self.sp_model.Load(str(_UpperCamelCase       )       )
							UpperCAmelCase_   :				int						=						vocab_file

							# Original fairseq vocab and spm vocab must be "aligned":
							# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
							# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
							# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
							# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

							# Mimic fairseq token-to-id alignment for the first 4 token
							UpperCAmelCase_   :				List[str]						=						{'<s>': 0, '<pad>': 1, '</s>': 2, '<unk>': 3}

							# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
							UpperCAmelCase_   :				str						=						1

							UpperCAmelCase_   :				Optional[int]						=						len(self.sp_model       )
							UpperCAmelCase_   :				str						=						{
							    code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(_UpperCamelCase       )
							}
							UpperCAmelCase_   :				int						=						{v: k for k, v in self.lang_code_to_id.items()}
							UpperCAmelCase_   :				List[Any]						=						len(self.sp_model       ) + len(self.lang_code_to_id       ) + self.fairseq_offset

							self.fairseq_tokens_to_ids.update(self.lang_code_to_id       )
							UpperCAmelCase_   :				Tuple						=						{v: k for k, v in self.fairseq_tokens_to_ids.items()}
							UpperCAmelCase_   :				int						=						list(self.lang_code_to_id.keys()       )

							if additional_special_tokens is not None:
												# Only add those special tokens if they are not already there.
												self._additional_special_tokens.extend(
												    [t for t in additional_special_tokens if t not in self._additional_special_tokens]       )

							UpperCAmelCase_   :				Any						=						src_lang if src_lang is not None else 'en_XX'
							UpperCAmelCase_   :				Any						=						self.lang_code_to_id[self._src_lang]
							UpperCAmelCase_   :				Dict						=						tgt_lang
							self.set_src_lang_special_tokens(self._src_lang       )



		def __getstate__(							self       )     ->							Dict:
							UpperCAmelCase_   :				Optional[int]						=						self.__dict__.copy()
							UpperCAmelCase_   :				str						=						None
							UpperCAmelCase_   :				str						=						self.sp_model.serialized_model_proto()
							return state



		def __setstate__(							self    ,		_UpperCamelCase       )     ->							Dict:
							UpperCAmelCase_   :				Union[str, Any]						=						d

							# for backward compatibility
							if not hasattr(self    ,		'sp_model_kwargs'       ):
												UpperCAmelCase_   :				Any						=						{}

							UpperCAmelCase_   :				Any						=						spm.SentencePieceProcessor(**self.sp_model_kwargs       )
							self.sp_model.LoadFromSerializedProto(self.sp_model_proto       )



		@property
		def  __UpperCAmelCase      (							self       )     ->							Tuple:
							return len(self.sp_model       ) + len(self.lang_code_to_id       ) + self.fairseq_offset + 1  # Plus 1 for the mask token



		@property
		def  __UpperCAmelCase      (							self       )     ->							str:
							return self._src_lang



		@src_lang.setter
		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase       )     ->							None:
							UpperCAmelCase_   :				Tuple						=						new_src_lang
							self.set_src_lang_special_tokens(self._src_lang       )



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase    ,		_UpperCamelCase = None    ,		_UpperCamelCase = False       )     ->							List[int]:

							if already_has_special_tokens:
												return super().get_special_tokens_mask(
												    token_ids_a=_UpperCamelCase    ,		token_ids_a=_UpperCamelCase    ,		already_has_special_tokens=_UpperCamelCase       )

							UpperCAmelCase_   :				Tuple						=						[1] * len(self.prefix_tokens       )
							UpperCAmelCase_   :				Dict						=						[1] * len(self.suffix_tokens       )
							if token_ids_a is None:
												return prefix_ones + ([0] * len(_UpperCamelCase       )) + suffix_ones
							return prefix_ones + ([0] * len(_UpperCamelCase       )) + ([0] * len(_UpperCamelCase       )) + suffix_ones



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase    ,		_UpperCamelCase = None       )     ->							List[int]:
							if token_ids_a is None:
												return self.prefix_tokens + token_ids_a + self.suffix_tokens
							# We don't expect to process pairs, but leave the pair logic for API consistency
							return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase    ,		_UpperCamelCase = None       )     ->							List[int]:
							UpperCAmelCase_   :				int						=						[self.sep_token_id]
							UpperCAmelCase_   :				Optional[int]						=						[self.cls_token_id]

							if token_ids_a is None:
												return len(cls + token_ids_a + sep       ) * [0]
							return len(cls + token_ids_a + sep + sep + token_ids_a + sep       ) * [0]



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase    ,		**_UpperCamelCase       )     ->							int:
							if src_lang is None or tgt_lang is None:
												raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'       )
							UpperCAmelCase_   :				Optional[int]						=						src_lang
							UpperCAmelCase_   :				Dict						=						self(_UpperCamelCase    ,		add_special_tokens=_UpperCamelCase    ,		return_tensors=_UpperCamelCase    ,		**_UpperCamelCase       )
							UpperCAmelCase_   :				int						=						self.convert_tokens_to_ids(_UpperCamelCase       )
							UpperCAmelCase_   :				Any						=						tgt_lang_id
							return inputs



		def  __UpperCAmelCase      (							self       )     ->							str:
							UpperCAmelCase_   :				str						=						{self.convert_ids_to_tokens(_UpperCamelCase       ): i for i in range(self.vocab_size       )}
							vocab.update(self.added_tokens_encoder       )
							return vocab



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase       )     ->							List[str]:
							return self.sp_model.encode(_UpperCamelCase    ,		out_type=_UpperCamelCase       )



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase       )     ->							Tuple:
							if token in self.fairseq_tokens_to_ids:
												return self.fairseq_tokens_to_ids[token]
							UpperCAmelCase_   :				Dict						=						self.sp_model.PieceToId(_UpperCamelCase       )

							# Need to return unknown token if the SP model returned 0
							return spm_id + self.fairseq_offset if spm_id else self.unk_token_id



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase       )     ->							str:
							if index in self.fairseq_ids_to_tokens:
												return self.fairseq_ids_to_tokens[index]
							return self.sp_model.IdToPiece(index - self.fairseq_offset       )



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase       )     ->							Optional[int]:
							UpperCAmelCase_   :				Optional[int]						=						''.join(_UpperCamelCase       ).replace(_UpperCamelCase    ,		' '       ).strip()
							return out_string



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase    ,		_UpperCamelCase = None       )     ->							Tuple[str]:
							if not os.path.isdir(_UpperCamelCase       ):
												logger.error(f"Vocabulary path ({save_directory}) should be a directory"       )
												return
							UpperCAmelCase_   :				List[Any]						=						os.path.join(
							    _UpperCamelCase    ,		(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']       )

							if os.path.abspath(self.vocab_file       ) != os.path.abspath(_UpperCamelCase       ) and os.path.isfile(self.vocab_file       ):
												copyfile(self.vocab_file    ,		_UpperCamelCase       )
							elif not os.path.isfile(self.vocab_file       ):
												with open(_UpperCamelCase    ,		'wb'       ) as fi:
																	UpperCAmelCase_   :				Optional[Any]						=						self.sp_model.serialized_model_proto()
																	fi.write(_UpperCamelCase       )

							return (out_vocab_file,)



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase    ,		_UpperCamelCase = "en_XX"    ,		_UpperCamelCase = None    ,		_UpperCamelCase = "ro_RO"    ,		**_UpperCamelCase    ,		)     ->							BatchEncoding:
							UpperCAmelCase_   :				Union[str, Any]						=						src_lang
							UpperCAmelCase_   :				Dict						=						tgt_lang
							return super().prepare_seqaseq_batch(_UpperCamelCase    ,		_UpperCamelCase    ,		**_UpperCamelCase       )



		def  __UpperCAmelCase      (							self       )     ->							Union[str, Any]:
							return self.set_src_lang_special_tokens(self.src_lang       )



		def  __UpperCAmelCase      (							self       )     ->							Optional[Any]:
							return self.set_tgt_lang_special_tokens(self.tgt_lang       )



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase       )     ->							None:
							UpperCAmelCase_   :				Any						=						self.lang_code_to_id[src_lang]
							UpperCAmelCase_   :				Optional[Any]						=						[]
							UpperCAmelCase_   :				str						=						[self.eos_token_id, self.cur_lang_code]



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase       )     ->							None:
							UpperCAmelCase_   :				Any						=						self.lang_code_to_id[lang]
							UpperCAmelCase_   :				Dict						=						[]
							UpperCAmelCase_   :				int						=						[self.eos_token_id, self.cur_lang_code]






