





import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


_SCREAMING_SNAKE_CASE 		=						logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE 		=						"""▁"""

_SCREAMING_SNAKE_CASE 		=						{"""vocab_file""": """sentencepiece.bpe.model"""}

_SCREAMING_SNAKE_CASE 		=						{
    """vocab_file""": {
        """facebook/mbart-large-en-ro""": (
            """https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model"""
        ),
        """facebook/mbart-large-cc25""": (
            """https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model"""
        ),
    }
}

_SCREAMING_SNAKE_CASE 		=						{
    """facebook/mbart-large-en-ro""": 10_24,
    """facebook/mbart-large-cc25""": 10_24,
}

# fmt: off
_SCREAMING_SNAKE_CASE 		=						["""ar_AR""", """cs_CZ""", """de_DE""", """en_XX""", """es_XX""", """et_EE""", """fi_FI""", """fr_XX""", """gu_IN""", """hi_IN""", """it_IT""", """ja_XX""", """kk_KZ""", """ko_KR""", """lt_LT""", """lv_LV""", """my_MM""", """ne_NP""", """nl_XX""", """ro_RO""", """ru_RU""", """si_LK""", """tr_TR""", """vi_VN""", """zh_CN"""]
class SCREAMING_SNAKE_CASE_  (   snake_case_   ):
						__magic_name__:     List[str]	 =    VOCAB_FILES_NAMES
						__magic_name__:     Any	 =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						__magic_name__:     Any	 =    PRETRAINED_VOCAB_FILES_MAP
						__magic_name__:     Union[str, Any]	 =    ["input_ids", "attention_mask"]

						__magic_name__:     List[int]	 =    []
						__magic_name__:     List[int]	 =    []






						def __init__( self : Dict						,				_A : Optional[int]						,				_A : List[str]="<s>"						,				_A : Optional[int]="</s>"						,				_A : List[str]="</s>"						,				_A : Any="<s>"						,				_A : Optional[int]="<unk>"						,				_A : Union[str, Any]="<pad>"						,				_A : Any="<mask>"						,				_A : str=None						,				_A : Union[str, Any]=None						,				_A : str=None						,				_A : Optional[Dict[str, Any]] = None						,				_A : Dict=None						,				**_A : Union[str, Any]						,				)							->      Any:







							"""simple docstring"""


							snake_case_						:     Tuple     =		AddedToken(_A						,				lstrip=_A						,				rstrip=_A		) if isinstance(_A						,				_A		) else mask_token

							snake_case_						:     Any     =		{} if sp_model_kwargs is None else sp_model_kwargs

							super().__init__(
							    bos_token=_A						,				eos_token=_A						,				unk_token=_A						,				sep_token=_A						,				cls_token=_A						,				pad_token=_A						,				mask_token=_A						,				tokenizer_file=_A						,				src_lang=_A						,				tgt_lang=_A						,				additional_special_tokens=_A						,				sp_model_kwargs=self.sp_model_kwargs						,				**_A						,				)

							snake_case_						:     Optional[Any]     =		spm.SentencePieceProcessor(**self.sp_model_kwargs		)
							self.sp_model.Load(str(_A		)		)
							snake_case_						:     int     =		vocab_file

							# Original fairseq vocab and spm vocab must be "aligned":
							# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
							# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
							# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
							# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

							# Mimic fairseq token-to-id alignment for the first 4 token
							snake_case_						:     Any     =		{'<s>': 0, '<pad>': 1, '</s>': 2, '<unk>': 3}

							# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
							snake_case_						:     Optional[int]     =		1

							snake_case_						:     Optional[Any]     =		len(self.sp_model		)
							snake_case_						:     str     =		{
							    code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(_A		)
							}
							snake_case_						:     List[Any]     =		{v: k for k, v in self.lang_code_to_id.items()}
							snake_case_						:     str     =		len(self.sp_model		) + len(self.lang_code_to_id		) + self.fairseq_offset

							self.fairseq_tokens_to_ids.update(self.lang_code_to_id		)
							snake_case_						:     int     =		{v: k for k, v in self.fairseq_tokens_to_ids.items()}
							snake_case_						:     List[Any]     =		list(self.lang_code_to_id.keys()		)

							if additional_special_tokens is not None:
								# Only add those special tokens if they are not already there.
								self._additional_special_tokens.extend(
								    [t for t in additional_special_tokens if t not in self._additional_special_tokens]		)

							snake_case_						:     List[str]     =		src_lang if src_lang is not None else 'en_XX'
							snake_case_						:     Optional[Any]     =		self.lang_code_to_id[self._src_lang]
							snake_case_						:     Optional[Any]     =		tgt_lang
							self.set_src_lang_special_tokens(self._src_lang		)






						def __getstate__( self : int		)							->      Tuple:







							"""simple docstring"""


							snake_case_						:     Dict     =		self.__dict__.copy()
							snake_case_						:     List[Any]     =		None
							snake_case_						:     List[Any]     =		self.sp_model.serialized_model_proto()
							return state






						def __setstate__( self : Optional[Any]						,				_A : str		)							->      int:







							"""simple docstring"""


							snake_case_						:     Optional[int]     =		d

							# for backward compatibility
							if not hasattr(self						,				'sp_model_kwargs'		):
								snake_case_						:     List[Any]     =		{}

							snake_case_						:     List[str]     =		spm.SentencePieceProcessor(**self.sp_model_kwargs		)
							self.sp_model.LoadFromSerializedProto(self.sp_model_proto		)






						@property
						def UpperCAmelCase_			( self : List[Any]		)							->      Union[str, Any]:







							"""simple docstring"""


							return len(self.sp_model		) + len(self.lang_code_to_id		) + self.fairseq_offset + 1  # Plus 1 for the mask token






						@property
						def UpperCAmelCase_			( self : Optional[int]		)							->      str:







							"""simple docstring"""


							return self._src_lang






						@src_lang.setter
						def UpperCAmelCase_			( self : Tuple						,				_A : str		)							->      None:







							"""simple docstring"""


							snake_case_						:     Any     =		new_src_lang
							self.set_src_lang_special_tokens(self._src_lang		)






						def UpperCAmelCase_			( self : Dict						,				_A : List[int]						,				_A : Optional[List[int]] = None						,				_A : bool = False		)							->      List[int]:







							"""simple docstring"""


							if already_has_special_tokens:
								return super().get_special_tokens_mask(
								    token_ids_a=_A						,				token_ids_a=_A						,				already_has_special_tokens=_A		)

							snake_case_						:     Union[str, Any]     =		[1] * len(self.prefix_tokens		)
							snake_case_						:     Optional[Any]     =		[1] * len(self.suffix_tokens		)
							if token_ids_a is None:
								return prefix_ones + ([0] * len(_A		)) + suffix_ones
							return prefix_ones + ([0] * len(_A		)) + ([0] * len(_A		)) + suffix_ones






						def UpperCAmelCase_			( self : List[Any]						,				_A : List[int]						,				_A : Optional[List[int]] = None		)							->      List[int]:







							"""simple docstring"""


							if token_ids_a is None:
								return self.prefix_tokens + token_ids_a + self.suffix_tokens
							# We don't expect to process pairs, but leave the pair logic for API consistency
							return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens






						def UpperCAmelCase_			( self : List[Any]						,				_A : List[int]						,				_A : Optional[List[int]] = None		)							->      List[int]:







							"""simple docstring"""


							snake_case_						:     Tuple     =		[self.sep_token_id]
							snake_case_						:     Union[str, Any]     =		[self.cls_token_id]

							if token_ids_a is None:
								return len(cls + token_ids_a + sep		) * [0]
							return len(cls + token_ids_a + sep + sep + token_ids_a + sep		) * [0]






						def UpperCAmelCase_			( self : List[str]						,				_A : Dict						,				_A : str						,				_A : Optional[str]						,				_A : Optional[str]						,				**_A : Optional[int]		)							->      int:







							"""simple docstring"""


							if src_lang is None or tgt_lang is None:
								raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'		)
							snake_case_						:     List[str]     =		src_lang
							snake_case_						:     Optional[Any]     =		self(_A						,				add_special_tokens=_A						,				return_tensors=_A						,				**_A		)
							snake_case_						:     Tuple     =		self.convert_tokens_to_ids(_A		)
							snake_case_						:     Optional[Any]     =		tgt_lang_id
							return inputs






						def UpperCAmelCase_			( self : Dict		)							->      Optional[Any]:







							"""simple docstring"""


							snake_case_						:     Optional[Any]     =		{self.convert_ids_to_tokens(_A		): i for i in range(self.vocab_size		)}
							vocab.update(self.added_tokens_encoder		)
							return vocab






						def UpperCAmelCase_			( self : Optional[int]						,				_A : str		)							->      List[str]:







							"""simple docstring"""


							return self.sp_model.encode(_A						,				out_type=_A		)






						def UpperCAmelCase_			( self : Optional[Any]						,				_A : List[str]		)							->      Tuple:







							"""simple docstring"""


							if token in self.fairseq_tokens_to_ids:
								return self.fairseq_tokens_to_ids[token]
							snake_case_						:     Dict     =		self.sp_model.PieceToId(_A		)

							# Need to return unknown token if the SP model returned 0
							return spm_id + self.fairseq_offset if spm_id else self.unk_token_id






						def UpperCAmelCase_			( self : Union[str, Any]						,				_A : List[str]		)							->      List[Any]:







							"""simple docstring"""


							if index in self.fairseq_ids_to_tokens:
								return self.fairseq_ids_to_tokens[index]
							return self.sp_model.IdToPiece(index - self.fairseq_offset		)






						def UpperCAmelCase_			( self : List[str]						,				_A : Union[str, Any]		)							->      Union[str, Any]:







							"""simple docstring"""


							snake_case_						:     int     =		''.join(_A		).replace(_A						,				' '		).strip()
							return out_string






						def UpperCAmelCase_			( self : Tuple						,				_A : str						,				_A : Optional[str] = None		)							->      Tuple[str]:







							"""simple docstring"""


							if not os.path.isdir(_A		):
								logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""		)
								return
							snake_case_						:     List[Any]     =		os.path.join(
							    _A						,				(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']		)

							if os.path.abspath(self.vocab_file		) != os.path.abspath(_A		) and os.path.isfile(self.vocab_file		):
								copyfile(self.vocab_file						,				_A		)
							elif not os.path.isfile(self.vocab_file		):
								with open(_A						,				'wb'		) as fi:
									snake_case_						:     List[str]     =		self.sp_model.serialized_model_proto()
									fi.write(_A		)

							return (out_vocab_file,)






						def UpperCAmelCase_			( self : Optional[int]						,				_A : List[str]						,				_A : str = "en_XX"						,				_A : Optional[List[str]] = None						,				_A : str = "ro_RO"						,				**_A : str						,				)							->      BatchEncoding:







							"""simple docstring"""


							snake_case_						:     int     =		src_lang
							snake_case_						:     Any     =		tgt_lang
							return super().prepare_seqaseq_batch(_A						,				_A						,				**_A		)






						def UpperCAmelCase_			( self : Optional[Any]		)							->      str:







							"""simple docstring"""


							return self.set_src_lang_special_tokens(self.src_lang		)






						def UpperCAmelCase_			( self : Optional[Any]		)							->      Any:







							"""simple docstring"""


							return self.set_tgt_lang_special_tokens(self.tgt_lang		)






						def UpperCAmelCase_			( self : List[str]						,				_A : Optional[Any]		)							->      None:







							"""simple docstring"""


							snake_case_						:     int     =		self.lang_code_to_id[src_lang]
							snake_case_						:     List[str]     =		[]
							snake_case_						:     int     =		[self.eos_token_id, self.cur_lang_code]






						def UpperCAmelCase_			( self : str						,				_A : str		)							->      None:







							"""simple docstring"""


							snake_case_						:     int     =		self.lang_code_to_id[lang]
							snake_case_						:     Tuple     =		[]
							snake_case_						:     Union[str, Any]     =		[self.eos_token_id, self.cur_lang_code]








import warnings

from ..trainer import Trainer
from ..utils import logging


_SCREAMING_SNAKE_CASE 		=						logging.get_logger(__name__)
class SCREAMING_SNAKE_CASE_  (   snake_case_   ):






						def __init__( self : Any						,				_A : str=None						,				**_A : Union[str, Any]		)							->      Any:







							"""simple docstring"""


							warnings.warn(
							    '`SageMakerTrainer` is deprecated and will be removed in v5 of Transformers. You can use `Trainer` '
							    'instead.'						,				_A						,				)
							super().__init__(args=_A						,				**_A		)

