






"""simple docstring"""




from abc import ABC, abstractmethod
from typing import List, Optional



class   lowerCamelCase				(_snake_case      ):







		'''simple docstring'''



		def __init__(							self       )     ->							str:
							# test for the above condition
							self.test()



		def  __UpperCAmelCase      (							self       )     ->							Tuple:
							UpperCAmelCase_   :				Tuple						=						0
							UpperCAmelCase_   :				List[Any]						=						False
							while not completed:
												if counter == 1:
																	self.reset()
												UpperCAmelCase_   :				Optional[int]						=						self.advance()
												if not self.does_advance(_UpperCamelCase       ):
																	raise Exception(
																	    'Custom Constraint is not defined correctly. self.does_advance(self.advance()) must be true.'       )

												UpperCAmelCase_   :				Optional[Any]						=						self.update(_UpperCamelCase       )
												counter += 1

												if counter > 1_0_0_0_0:
																	raise Exception('update() does not fulfill the constraint.'       )

							if self.remaining() != 0:
												raise Exception('Custom Constraint is not defined correctly.'       )



		@abstractmethod
		def  __UpperCAmelCase      (							self       )     ->							int:
							raise NotImplementedError(
							    f"{self.__class__} is an abstract class. Only classes inheriting this class can be called."       )



		@abstractmethod
		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase       )     ->							Optional[Any]:
							raise NotImplementedError(
							    f"{self.__class__} is an abstract class. Only classes inheriting this class can be called."       )



		@abstractmethod
		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase       )     ->							Union[str, Any]:
							raise NotImplementedError(
							    f"{self.__class__} is an abstract class. Only classes inheriting this class can be called."       )



		@abstractmethod
		def  __UpperCAmelCase      (							self       )     ->							List[Any]:
							raise NotImplementedError(
							    f"{self.__class__} is an abstract class. Only classes inheriting this class can be called."       )



		@abstractmethod
		def  __UpperCAmelCase      (							self       )     ->							Tuple:
							raise NotImplementedError(
							    f"{self.__class__} is an abstract class. Only classes inheriting this class can be called."       )



		@abstractmethod
		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase=False       )     ->							List[Any]:
							raise NotImplementedError(
							    f"{self.__class__} is an abstract class. Only classes inheriting this class can be called."       )





class   lowerCamelCase				(_snake_case      ):







		'''simple docstring'''



		def __init__(							self    ,		_UpperCamelCase       )     ->							Union[str, Any]:
							super(_UpperCamelCase    ,		self       ).__init__()

							if not isinstance(_UpperCamelCase    ,		_UpperCamelCase       ) or len(_UpperCamelCase       ) == 0:
												raise ValueError(f"`token_ids` has to be a non-empty list, but is {token_ids}."       )
							if any((not isinstance(_UpperCamelCase    ,		_UpperCamelCase       ) or token_id < 0) for token_id in token_ids       ):
												raise ValueError(f"Each list in `token_ids` has to be a list of positive integers, but is {token_ids}."       )

							UpperCAmelCase_   :				List[Any]						=						token_ids

							UpperCAmelCase_   :				Dict						=						len(self.token_ids       )
							UpperCAmelCase_   :				str						=						-1  # the index of the currently fulfilled step
							UpperCAmelCase_   :				int						=						False



		def  __UpperCAmelCase      (							self       )     ->							List[Any]:
							if self.completed:
												return None
							return self.token_ids[self.fulfilled_idx + 1]



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase       )     ->							Optional[int]:
							if not isinstance(_UpperCamelCase    ,		_UpperCamelCase       ):
												raise ValueError(f"`token_id` has to be an `int`, but is {token_id} of type {type(_UpperCamelCase       )}"       )

							if self.completed:
												return False

							return token_id == self.token_ids[self.fulfilled_idx + 1]



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase       )     ->							Optional[int]:
							if not isinstance(_UpperCamelCase    ,		_UpperCamelCase       ):
												raise ValueError(f"`token_id` has to be an `int`, but is {token_id} of type {type(_UpperCamelCase       )}"       )

							UpperCAmelCase_   :				Union[str, Any]						=						False
							UpperCAmelCase_   :				Tuple						=						False
							UpperCAmelCase_   :				Any						=						False

							if self.does_advance(_UpperCamelCase       ):
												self.fulfilled_idx += 1
												UpperCAmelCase_   :				Any						=						True
												if self.fulfilled_idx == (self.seqlen - 1):
																	UpperCAmelCase_   :				Optional[Any]						=						True
												UpperCAmelCase_   :				List[str]						=						completed
							else:
												# failed to make progress.
												UpperCAmelCase_   :				Optional[Any]						=						True
												self.reset()
							return stepped, completed, reset



		def  __UpperCAmelCase      (							self       )     ->							Dict:
							UpperCAmelCase_   :				str						=						False
							UpperCAmelCase_   :				Any						=						0



		def  __UpperCAmelCase      (							self       )     ->							Any:
							return self.seqlen - (self.fulfilled_idx + 1)



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase=False       )     ->							Any:
							UpperCAmelCase_   :				str						=						PhrasalConstraint(self.token_ids       )

							if stateful:
												UpperCAmelCase_   :				Tuple						=						self.seqlen
												UpperCAmelCase_   :				Any						=						self.fulfilled_idx
												UpperCAmelCase_   :				List[str]						=						self.completed

							return new_constraint





class   lowerCamelCase				:







		'''simple docstring'''



		def __init__(							self    ,		_UpperCamelCase    ,		_UpperCamelCase=True       )     ->							Union[str, Any]:
							UpperCAmelCase_   :				List[Any]						=						max([len(_UpperCamelCase       ) for one in nested_token_ids]       )

							UpperCAmelCase_   :				Dict						=						{}
							for token_ids in nested_token_ids:
												UpperCAmelCase_   :				Tuple						=						root
												for tidx, token_id in enumerate(_UpperCamelCase       ):
																	if token_id not in level:
																						UpperCAmelCase_   :				str						=						{}

																	UpperCAmelCase_   :				Optional[Any]						=						level[token_id]

							if no_subsets and self.has_subsets(_UpperCamelCase    ,		_UpperCamelCase       ):
												raise ValueError(
												    'Each list in `nested_token_ids` can\'t be a complete subset of another list, but is'
												    f" {nested_token_ids}."       )

							UpperCAmelCase_   :				str						=						root



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase       )     ->							Tuple:
							UpperCAmelCase_   :				Dict						=						self.trie

							for current_token in current_seq:
												UpperCAmelCase_   :				int						=						start[current_token]

							UpperCAmelCase_   :				List[Any]						=						list(start.keys()       )

							return next_tokens



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase       )     ->							List[Any]:
							UpperCAmelCase_   :				Optional[int]						=						self.next_tokens(_UpperCamelCase       )

							return len(_UpperCamelCase       ) == 0



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase       )     ->							int:
							UpperCAmelCase_   :				str						=						list(root.values()       )
							if len(_UpperCamelCase       ) == 0:
												return 1
							else:
												return sum([self.count_leaves(_UpperCamelCase       ) for nn in next_nodes]       )



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase    ,		_UpperCamelCase       )     ->							List[Any]:
							UpperCAmelCase_   :				Tuple						=						self.count_leaves(_UpperCamelCase       )
							return len(_UpperCamelCase       ) != leaf_count





class   lowerCamelCase				(_snake_case      ):







		'''simple docstring'''



		def __init__(							self    ,		_UpperCamelCase       )     ->							List[Any]:
							super(_UpperCamelCase    ,		self       ).__init__()

							if not isinstance(_UpperCamelCase    ,		_UpperCamelCase       ) or len(_UpperCamelCase       ) == 0:
												raise ValueError(f"`nested_token_ids` has to be a non-empty list, but is {nested_token_ids}."       )
							if any(not isinstance(_UpperCamelCase    ,		_UpperCamelCase       ) for token_ids in nested_token_ids       ):
												raise ValueError(f"`nested_token_ids` has to be a list of lists, but is {nested_token_ids}."       )
							if any(
							    any((not isinstance(_UpperCamelCase    ,		_UpperCamelCase       ) or token_id < 0) for token_id in token_ids       )
							    for token_ids in nested_token_ids       ):
												raise ValueError(
												    f"Each list in `nested_token_ids` has to be a list of positive integers, but is {nested_token_ids}."       )

							UpperCAmelCase_   :				Optional[Any]						=						DisjunctiveTrie(_UpperCamelCase       )
							UpperCAmelCase_   :				Any						=						nested_token_ids

							UpperCAmelCase_   :				List[Any]						=						self.trie.max_height
							UpperCAmelCase_   :				str						=						[]
							UpperCAmelCase_   :				Tuple						=						False



		def  __UpperCAmelCase      (							self       )     ->							int:
							UpperCAmelCase_   :				Union[str, Any]						=						self.trie.next_tokens(self.current_seq       )

							if len(_UpperCamelCase       ) == 0:
												return None
							else:
												return token_list



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase       )     ->							Union[str, Any]:
							if not isinstance(_UpperCamelCase    ,		_UpperCamelCase       ):
												raise ValueError(f"`token_id` is supposed to be type `int`, but is {token_id} of type {type(_UpperCamelCase       )}"       )

							UpperCAmelCase_   :				Dict						=						self.trie.next_tokens(self.current_seq       )

							return token_id in next_tokens



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase       )     ->							Union[str, Any]:
							if not isinstance(_UpperCamelCase    ,		_UpperCamelCase       ):
												raise ValueError(f"`token_id` is supposed to be type `int`, but is {token_id} of type {type(_UpperCamelCase       )}"       )

							UpperCAmelCase_   :				str						=						False
							UpperCAmelCase_   :				Union[str, Any]						=						False
							UpperCAmelCase_   :				int						=						False

							if self.does_advance(_UpperCamelCase       ):
												self.current_seq.append(_UpperCamelCase       )
												UpperCAmelCase_   :				Any						=						True
							else:
												UpperCAmelCase_   :				List[str]						=						True
												self.reset()

							UpperCAmelCase_   :				Optional[Any]						=						self.trie.reached_leaf(self.current_seq       )
							UpperCAmelCase_   :				Tuple						=						completed

							return stepped, completed, reset



		def  __UpperCAmelCase      (							self       )     ->							Tuple:
							UpperCAmelCase_   :				Optional[Any]						=						False
							UpperCAmelCase_   :				Optional[Any]						=						[]



		def  __UpperCAmelCase      (							self       )     ->							Tuple:
							if self.completed:
												# since this can be completed without reaching max height
												return 0
							else:
												return self.seqlen - len(self.current_seq       )



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase=False       )     ->							List[Any]:
							UpperCAmelCase_   :				int						=						DisjunctiveConstraint(self.token_ids       )

							if stateful:
												UpperCAmelCase_   :				int						=						self.seqlen
												UpperCAmelCase_   :				str						=						self.current_seq
												UpperCAmelCase_   :				List[Any]						=						self.completed

							return new_constraint





class   lowerCamelCase				:







		'''simple docstring'''



		def __init__(							self    ,		_UpperCamelCase       )     ->							int:
							UpperCAmelCase_   :				Tuple						=						constraints

							# max # of steps required to fulfill a given constraint
							UpperCAmelCase_   :				Any						=						max([c.seqlen for c in constraints]       )
							UpperCAmelCase_   :				Tuple						=						len(_UpperCamelCase       )
							UpperCAmelCase_   :				List[Any]						=						False

							self.init_state()



		def  __UpperCAmelCase      (							self       )     ->							int:
							UpperCAmelCase_   :				Dict						=						[]
							UpperCAmelCase_   :				List[str]						=						None
							UpperCAmelCase_   :				Any						=						[constraint.copy(stateful=_UpperCamelCase       ) for constraint in self.constraints]



		def  __UpperCAmelCase      (							self       )     ->							int:
							UpperCAmelCase_   :				List[Any]						=						0
							if self.inprogress_constraint:
												# extra points for having a constraint mid-fulfilled
												add += self.max_seqlen - self.inprogress_constraint.remaining()

							return (len(self.complete_constraints       ) * self.max_seqlen) + add



		def  __UpperCAmelCase      (							self       )     ->							Tuple:
							UpperCAmelCase_   :				List[Any]						=						[]
							if self.inprogress_constraint is None:
												for constraint in self.pending_constraints:  # "pending" == "unfulfilled yet"
																	UpperCAmelCase_   :				Optional[Any]						=						constraint.advance()
																	if isinstance(_UpperCamelCase    ,		_UpperCamelCase       ):
																						token_list.append(_UpperCamelCase       )
																	elif isinstance(_UpperCamelCase    ,		_UpperCamelCase       ):
																						token_list.extend(_UpperCamelCase       )
							else:
												UpperCAmelCase_   :				Union[str, Any]						=						self.inprogress_constraint.advance()
												if isinstance(_UpperCamelCase    ,		_UpperCamelCase       ):
																	token_list.append(_UpperCamelCase       )
												elif isinstance(_UpperCamelCase    ,		_UpperCamelCase       ):
																	token_list.extend(_UpperCamelCase       )

							if len(_UpperCamelCase       ) == 0:
												return None
							else:
												return token_list



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase       )     ->							Any:
							self.init_state()

							if token_ids is not None:
												for token in token_ids:
																	# completes or steps **one** constraint
																	UpperCAmelCase_   :				Optional[Any]						=						self.add(_UpperCamelCase       )

																	# the entire list of constraints are fulfilled
																	if self.completed:
																						break



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase       )     ->							Any:
							if not isinstance(_UpperCamelCase    ,		_UpperCamelCase       ):
												raise ValueError(f"`token_id` should be an `int`, but is `{token_id}`."       )

							UpperCAmelCase_   :				List[str]						=						False, False

							if self.completed:
												UpperCAmelCase_   :				Tuple						=						True
												UpperCAmelCase_   :				Optional[Any]						=						False
												return complete, stepped

							if self.inprogress_constraint is not None:
												# In the middle of fulfilling a constraint. If the `token_id` *does* makes an incremental progress to current
												# job, simply update the state

												UpperCAmelCase_   :				Optional[int]						=						self.inprogress_constraint.update(_UpperCamelCase       )
												if reset:
																	# 1. If the next token breaks the progress, then we must restart.
																	#     e.g. constraint = "I love pies" and sequence so far is "I love" but `token_id` == "books".

																	#     But that doesn't mean we self.init_state(), since we only reset the state for this particular
																	#     constraint, not the full list of constraints.

																	self.pending_constraints.append(self.inprogress_constraint.copy(stateful=_UpperCamelCase       )       )
																	UpperCAmelCase_   :				Tuple						=						None

												if complete:
																	# 2. If the next token completes the constraint, move it to completed list, set
																	#     inprogress to None. If there are no pending constraints either, then this full list of constraints
																	#     is complete.

																	self.complete_constraints.append(self.inprogress_constraint       )
																	UpperCAmelCase_   :				Dict						=						None

																	if len(self.pending_constraints       ) == 0:
																						# we're done!
																						UpperCAmelCase_   :				str						=						True

							else:
												# Not in the middle of fulfilling a constraint. So does this `token_id` helps us step towards any of our list
												# of constraints?

												for cidx, pending_constraint in enumerate(self.pending_constraints       ):
																	if pending_constraint.does_advance(_UpperCamelCase       ):
																						UpperCAmelCase_   :				str						=						pending_constraint.update(_UpperCamelCase       )

																						if not stepped:
																											raise Exception(
																											    '`constraint.update(token_id)` is not yielding incremental progress, '
																											    'even though `constraint.does_advance(token_id)` is true.'       )

																						if complete:
																											self.complete_constraints.append(_UpperCamelCase       )
																											UpperCAmelCase_   :				Optional[Any]						=						None

																						if not complete and stepped:
																											UpperCAmelCase_   :				str						=						pending_constraint

																						if complete or stepped:
																											# If we made any progress at all, then it's at least not a "pending constraint".

																											UpperCAmelCase_   :				Optional[int]						=						(
																											    self.pending_constraints[:cidx] + self.pending_constraints[cidx + 1 :]
																											)

																											if len(self.pending_constraints       ) == 0 and self.inprogress_constraint is None:
																																# If there's no longer any pending after this and no inprogress either, then we must be
																																# complete.

																																UpperCAmelCase_   :				str						=						True

																											break  # prevent accidentally stepping through multiple constraints with just one token.

							return complete, stepped



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase=True       )     ->							Dict:
							UpperCAmelCase_   :				Union[str, Any]						=						ConstraintListState(self.constraints       )  # we actually never though self.constraints objects
							# throughout this process. So it's at initialization state.

							if stateful:
												UpperCAmelCase_   :				List[Any]						=						[
												    constraint.copy(stateful=_UpperCamelCase       ) for constraint in self.complete_constraints
												]
												if self.inprogress_constraint is not None:
																	UpperCAmelCase_   :				List[Any]						=						self.inprogress_constraint.copy(stateful=_UpperCamelCase       )
												UpperCAmelCase_   :				Union[str, Any]						=						[constraint.copy() for constraint in self.pending_constraints]

							return new_state














import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


__UpperCAmelCase     							=						logging.get_logger(__name__)

__UpperCAmelCase     							=						'▁'

__UpperCAmelCase     							=						{'vocab_file': 'sentencepiece.bpe.model'}

__UpperCAmelCase     							=						{
    'vocab_file': {
        'facebook/mbart-large-en-ro': (
            'https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model'
        ),
        'facebook/mbart-large-cc25': (
            'https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model'
        ),
    }
}

__UpperCAmelCase     							=						{
    'facebook/mbart-large-en-ro': 1024,
    'facebook/mbart-large-cc25': 1024,
}

# fmt: off
__UpperCAmelCase     							=						['ar_AR', 'cs_CZ', 'de_DE', 'en_XX', 'es_XX', 'et_EE', 'fi_FI', 'fr_XX', 'gu_IN', 'hi_IN', 'it_IT', 'ja_XX', 'kk_KZ', 'ko_KR', 'lt_LT', 'lv_LV', 'my_MM', 'ne_NP', 'nl_XX', 'ro_RO', 'ru_RU', 'si_LK', 'tr_TR', 'vi_VN', 'zh_CN']



class   lowerCamelCase				(_snake_case      ):







		'''simple docstring'''



		_snake_case   :   List[Any]							=  VOCAB_FILES_NAMES
		_snake_case   :   Union[str, Any]							=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		_snake_case   :   Tuple							=  PRETRAINED_VOCAB_FILES_MAP
		_snake_case   :   Any							=  ['''input_ids''', '''attention_mask''']

		_snake_case   :   List[int]							=  []
		_snake_case   :   List[int]							=  []



		def __init__(							self    ,		_UpperCamelCase    ,		_UpperCamelCase="<s>"    ,		_UpperCamelCase="</s>"    ,		_UpperCamelCase="</s>"    ,		_UpperCamelCase="<s>"    ,		_UpperCamelCase="<unk>"    ,		_UpperCamelCase="<pad>"    ,		_UpperCamelCase="<mask>"    ,		_UpperCamelCase=None    ,		_UpperCamelCase=None    ,		_UpperCamelCase=None    ,		_UpperCamelCase = None    ,		_UpperCamelCase=None    ,		**_UpperCamelCase    ,		)     ->							List[str]:
							# Mask token behave like a normal word, i.e. include the space before it
							UpperCAmelCase_   :				List[Any]						=						AddedToken(_UpperCamelCase    ,		lstrip=_UpperCamelCase    ,		rstrip=_UpperCamelCase       ) if isinstance(_UpperCamelCase    ,		_UpperCamelCase       ) else mask_token

							UpperCAmelCase_   :				Optional[int]						=						{} if sp_model_kwargs is None else sp_model_kwargs

							super().__init__(
							    bos_token=_UpperCamelCase    ,		eos_token=_UpperCamelCase    ,		unk_token=_UpperCamelCase    ,		sep_token=_UpperCamelCase    ,		cls_token=_UpperCamelCase    ,		pad_token=_UpperCamelCase    ,		mask_token=_UpperCamelCase    ,		tokenizer_file=_UpperCamelCase    ,		src_lang=_UpperCamelCase    ,		tgt_lang=_UpperCamelCase    ,		additional_special_tokens=_UpperCamelCase    ,		sp_model_kwargs=self.sp_model_kwargs    ,		**_UpperCamelCase    ,		)

							UpperCAmelCase_   :				Any						=						spm.SentencePieceProcessor(**self.sp_model_kwargs       )
							self.sp_model.Load(str(_UpperCamelCase       )       )
							UpperCAmelCase_   :				int						=						vocab_file

							# Original fairseq vocab and spm vocab must be "aligned":
							# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
							# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
							# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
							# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

							# Mimic fairseq token-to-id alignment for the first 4 token
							UpperCAmelCase_   :				List[str]						=						{'<s>': 0, '<pad>': 1, '</s>': 2, '<unk>': 3}

							# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
							UpperCAmelCase_   :				str						=						1

							UpperCAmelCase_   :				Optional[int]						=						len(self.sp_model       )
							UpperCAmelCase_   :				str						=						{
							    code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(_UpperCamelCase       )
							}
							UpperCAmelCase_   :				int						=						{v: k for k, v in self.lang_code_to_id.items()}
							UpperCAmelCase_   :				List[Any]						=						len(self.sp_model       ) + len(self.lang_code_to_id       ) + self.fairseq_offset

							self.fairseq_tokens_to_ids.update(self.lang_code_to_id       )
							UpperCAmelCase_   :				Tuple						=						{v: k for k, v in self.fairseq_tokens_to_ids.items()}
							UpperCAmelCase_   :				int						=						list(self.lang_code_to_id.keys()       )

							if additional_special_tokens is not None:
												# Only add those special tokens if they are not already there.
												self._additional_special_tokens.extend(
												    [t for t in additional_special_tokens if t not in self._additional_special_tokens]       )

							UpperCAmelCase_   :				Any						=						src_lang if src_lang is not None else 'en_XX'
							UpperCAmelCase_   :				Any						=						self.lang_code_to_id[self._src_lang]
							UpperCAmelCase_   :				Dict						=						tgt_lang
							self.set_src_lang_special_tokens(self._src_lang       )



		def __getstate__(							self       )     ->							Dict:
							UpperCAmelCase_   :				Optional[int]						=						self.__dict__.copy()
							UpperCAmelCase_   :				str						=						None
							UpperCAmelCase_   :				str						=						self.sp_model.serialized_model_proto()
							return state



		def __setstate__(							self    ,		_UpperCamelCase       )     ->							Dict:
							UpperCAmelCase_   :				Union[str, Any]						=						d

							# for backward compatibility
							if not hasattr(self    ,		'sp_model_kwargs'       ):
												UpperCAmelCase_   :				Any						=						{}

							UpperCAmelCase_   :				Any						=						spm.SentencePieceProcessor(**self.sp_model_kwargs       )
							self.sp_model.LoadFromSerializedProto(self.sp_model_proto       )



		@property
		def  __UpperCAmelCase      (							self       )     ->							Tuple:
							return len(self.sp_model       ) + len(self.lang_code_to_id       ) + self.fairseq_offset + 1  # Plus 1 for the mask token



		@property
		def  __UpperCAmelCase      (							self       )     ->							str:
							return self._src_lang



		@src_lang.setter
		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase       )     ->							None:
							UpperCAmelCase_   :				Tuple						=						new_src_lang
							self.set_src_lang_special_tokens(self._src_lang       )



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase    ,		_UpperCamelCase = None    ,		_UpperCamelCase = False       )     ->							List[int]:

							if already_has_special_tokens:
												return super().get_special_tokens_mask(
												    token_ids_a=_UpperCamelCase    ,		token_ids_a=_UpperCamelCase    ,		already_has_special_tokens=_UpperCamelCase       )

							UpperCAmelCase_   :				Tuple						=						[1] * len(self.prefix_tokens       )
							UpperCAmelCase_   :				Dict						=						[1] * len(self.suffix_tokens       )
							if token_ids_a is None:
												return prefix_ones + ([0] * len(_UpperCamelCase       )) + suffix_ones
							return prefix_ones + ([0] * len(_UpperCamelCase       )) + ([0] * len(_UpperCamelCase       )) + suffix_ones



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase    ,		_UpperCamelCase = None       )     ->							List[int]:
							if token_ids_a is None:
												return self.prefix_tokens + token_ids_a + self.suffix_tokens
							# We don't expect to process pairs, but leave the pair logic for API consistency
							return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase    ,		_UpperCamelCase = None       )     ->							List[int]:
							UpperCAmelCase_   :				int						=						[self.sep_token_id]
							UpperCAmelCase_   :				Optional[int]						=						[self.cls_token_id]

							if token_ids_a is None:
												return len(cls + token_ids_a + sep       ) * [0]
							return len(cls + token_ids_a + sep + sep + token_ids_a + sep       ) * [0]



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase    ,		_UpperCamelCase    ,		**_UpperCamelCase       )     ->							int:
							if src_lang is None or tgt_lang is None:
												raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'       )
							UpperCAmelCase_   :				Optional[int]						=						src_lang
							UpperCAmelCase_   :				Dict						=						self(_UpperCamelCase    ,		add_special_tokens=_UpperCamelCase    ,		return_tensors=_UpperCamelCase    ,		**_UpperCamelCase       )
							UpperCAmelCase_   :				int						=						self.convert_tokens_to_ids(_UpperCamelCase       )
							UpperCAmelCase_   :				Any						=						tgt_lang_id
							return inputs



		def  __UpperCAmelCase      (							self       )     ->							str:
							UpperCAmelCase_   :				str						=						{self.convert_ids_to_tokens(_UpperCamelCase       ): i for i in range(self.vocab_size       )}
							vocab.update(self.added_tokens_encoder       )
							return vocab



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase       )     ->							List[str]:
							return self.sp_model.encode(_UpperCamelCase    ,		out_type=_UpperCamelCase       )



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase       )     ->							Tuple:
							if token in self.fairseq_tokens_to_ids:
												return self.fairseq_tokens_to_ids[token]
							UpperCAmelCase_   :				Dict						=						self.sp_model.PieceToId(_UpperCamelCase       )

							# Need to return unknown token if the SP model returned 0
							return spm_id + self.fairseq_offset if spm_id else self.unk_token_id



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase       )     ->							str:
							if index in self.fairseq_ids_to_tokens:
												return self.fairseq_ids_to_tokens[index]
							return self.sp_model.IdToPiece(index - self.fairseq_offset       )



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase       )     ->							Optional[int]:
							UpperCAmelCase_   :				Optional[int]						=						''.join(_UpperCamelCase       ).replace(_UpperCamelCase    ,		' '       ).strip()
							return out_string



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase    ,		_UpperCamelCase = None       )     ->							Tuple[str]:
							if not os.path.isdir(_UpperCamelCase       ):
												logger.error(f"Vocabulary path ({save_directory}) should be a directory"       )
												return
							UpperCAmelCase_   :				List[Any]						=						os.path.join(
							    _UpperCamelCase    ,		(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']       )

							if os.path.abspath(self.vocab_file       ) != os.path.abspath(_UpperCamelCase       ) and os.path.isfile(self.vocab_file       ):
												copyfile(self.vocab_file    ,		_UpperCamelCase       )
							elif not os.path.isfile(self.vocab_file       ):
												with open(_UpperCamelCase    ,		'wb'       ) as fi:
																	UpperCAmelCase_   :				Optional[Any]						=						self.sp_model.serialized_model_proto()
																	fi.write(_UpperCamelCase       )

							return (out_vocab_file,)



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase    ,		_UpperCamelCase = "en_XX"    ,		_UpperCamelCase = None    ,		_UpperCamelCase = "ro_RO"    ,		**_UpperCamelCase    ,		)     ->							BatchEncoding:
							UpperCAmelCase_   :				Union[str, Any]						=						src_lang
							UpperCAmelCase_   :				Dict						=						tgt_lang
							return super().prepare_seqaseq_batch(_UpperCamelCase    ,		_UpperCamelCase    ,		**_UpperCamelCase       )



		def  __UpperCAmelCase      (							self       )     ->							Union[str, Any]:
							return self.set_src_lang_special_tokens(self.src_lang       )



		def  __UpperCAmelCase      (							self       )     ->							Optional[Any]:
							return self.set_tgt_lang_special_tokens(self.tgt_lang       )



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase       )     ->							None:
							UpperCAmelCase_   :				Any						=						self.lang_code_to_id[src_lang]
							UpperCAmelCase_   :				Optional[Any]						=						[]
							UpperCAmelCase_   :				str						=						[self.eos_token_id, self.cur_lang_code]



		def  __UpperCAmelCase      (							self    ,		_UpperCamelCase       )     ->							None:
							UpperCAmelCase_   :				Any						=						self.lang_code_to_id[lang]
							UpperCAmelCase_   :				Dict						=						[]
							UpperCAmelCase_   :				int						=						[self.eos_token_id, self.cur_lang_code]






