"""simple docstring"""







import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


_a					:					Union[str, Any]=			logging.get_logger(__name__)

_a					:					List[Any]=			"▁"

_a					:					List[str]=			{"vocab_file": "sentencepiece.bpe.model"}

_a					:					Union[str, Any]=			{
    "vocab_file": {
        "facebook/mbart-large-50-one-to-many-mmt": (
            "https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model"
        ),
    }
}

_a					:					List[str]=			{
    "facebook/mbart-large-50-one-to-many-mmt": 1_024,
}

# fmt: off
_a					:					List[Any]=			["ar_AR", "cs_CZ", "de_DE", "en_XX", "es_XX", "et_EE", "fi_FI", "fr_XX", "gu_IN", "hi_IN", "it_IT", "ja_XX", "kk_KZ", "ko_KR", "lt_LT", "lv_LV", "my_MM", "ne_NP", "nl_XX", "ro_RO", "ru_RU", "si_LK", "tr_TR", "vi_VN", "zh_CN", "af_ZA", "az_AZ", "bn_IN", "fa_IR", "he_IL", "hr_HR", "id_ID", "ka_GE", "km_KH", "mk_MK", "ml_IN", "mn_MN", "mr_IN", "pl_PL", "ps_AF", "pt_XX", "sv_SE", "sw_KE", "ta_IN", "te_IN", "th_TH", "tl_XX", "uk_UA", "ur_PK", "xh_ZA", "gl_ES", "sl_SI"]

class 		UpperCamelCase      (   lowercase						):
				UpperCAmelCase    :				str   =		VOCAB_FILES_NAMES
				UpperCAmelCase    :				List[Any]   =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				UpperCAmelCase    :				int   =		PRETRAINED_VOCAB_FILES_MAP
				UpperCAmelCase    :				int   =		["""input_ids""", """attention_mask"""]

				UpperCAmelCase    :				List[int]   =		[]
				UpperCAmelCase    :				List[int]   =		[]



				def __init__(self    :				int     ,  _A    :				Union[str, Any]     ,  _A    :				Any=None     ,  _A    :				Dict=None     ,  _A    :				List[Any]="</s>"     ,  _A    :				Optional[int]="</s>"     ,  _A    :				Dict="<s>"     ,  _A    :				Any="<unk>"     ,  _A    :				List[str]="<pad>"     ,  _A    :				Optional[Any]="<mask>"     ,  _A    :				Optional[Dict[str, Any]] = None     ,  **_A    :				int     ,  )   ->     None:
					# Mask token behave like a normal word, i.e. include the space before it
					__snake_case      :							List[Any]						=			AddedToken(_A     ,  lstrip=_A     ,  rstrip=_A) if isinstance(_A     ,  _A) else mask_token

					__snake_case      :							Tuple						=			{} if sp_model_kwargs is None else sp_model_kwargs

					__snake_case      :							Optional[int]						=			kwargs.get('additional_special_tokens'     ,  [])
					kwargs["additional_special_tokens"] += [
					    code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
					]

					super().__init__(
					    src_lang=_A     ,  tgt_lang=_A     ,  eos_token=_A     ,  unk_token=_A     ,  sep_token=_A     ,  cls_token=_A     ,  pad_token=_A     ,  mask_token=_A     ,  sp_model_kwargs=self.sp_model_kwargs     ,  **_A     ,  )

					__snake_case      :							Optional[int]						=			spm.SentencePieceProcessor(**self.sp_model_kwargs)
					self.sp_model.Load(str(_A))
					__snake_case      :							Any						=			vocab_file

					# Original fairseq vocab and spm vocab must be "aligned":
					# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
					# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
					# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
					# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

					# Mimic fairseq token-to-id alignment for the first 4 token
					__snake_case      :							Optional[Any]						=			{'<s>': 0, '<pad>': 1, '</s>': 2, '<unk>': 3}

					# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
					__snake_case      :							Dict						=			1

					__snake_case      :							Union[str, Any]						=			len(self.sp_model)
					__snake_case      :							Tuple						=			{
					    code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(_A)
					}
					__snake_case      :							Any						=			{v: k for k, v in self.lang_code_to_id.items()}
					__snake_case      :							Tuple						=			len(self.sp_model) + len(self.lang_code_to_id) + self.fairseq_offset

					self.fairseq_tokens_to_ids.update(self.lang_code_to_id)
					__snake_case      :							Union[str, Any]						=			{v: k for k, v in self.fairseq_tokens_to_ids.items()}

					__snake_case      :							List[Any]						=			src_lang if src_lang is not None else 'en_XX'
					__snake_case      :							int						=			self.lang_code_to_id[self._src_lang]
					__snake_case      :							List[str]						=			tgt_lang
					self.set_src_lang_special_tokens(self._src_lang)



				@property
				def  _lowercase					(self    :				List[Any])   ->     int:
					return len(self.sp_model) + len(self.lang_code_to_id) + self.fairseq_offset + 1  # Plus 1 for the mask token



				@property
				def  _lowercase					(self    :				Optional[Any])   ->     str:
					return self._src_lang



				@src_lang.setter
				def  _lowercase					(self    :				Dict     ,  _A    :				str)   ->     None:
					__snake_case      :							int						=			new_src_lang
					self.set_src_lang_special_tokens(self._src_lang)



				def __getstate__(self    :				Optional[int])   ->     Dict:
					__snake_case      :							Optional[Any]						=			self.__dict__.copy()
					__snake_case      :							List[Any]						=			None
					return state



				def __setstate__(self    :				Any     ,  _A    :				Dict)   ->     None:
					__snake_case      :							str						=			d

					# for backward compatibility
					if not hasattr(self     ,  'sp_model_kwargs'):
						__snake_case      :							List[Any]						=			{}

					__snake_case      :							Tuple						=			spm.SentencePieceProcessor(**self.sp_model_kwargs)
					self.sp_model.Load(self.vocab_file)



				def  _lowercase					(self    :				Optional[int])   ->     Dict:
					__snake_case      :							Tuple						=			{self.convert_ids_to_tokens(_A): i for i in range(self.vocab_size)}
					vocab.update(self.added_tokens_encoder)
					return vocab



				def  _lowercase					(self    :				int     ,  _A    :				str)   ->     List[str]:
					return self.sp_model.encode(_A     ,  out_type=_A)



				def  _lowercase					(self    :				Any     ,  _A    :				str)   ->     int:
					if token in self.fairseq_tokens_to_ids:
						return self.fairseq_tokens_to_ids[token]
					__snake_case      :							List[Any]						=			self.sp_model.PieceToId(_A)

					# Need to return unknown token if the SP model returned 0
					return spm_id + self.fairseq_offset if spm_id else self.unk_token_id



				def  _lowercase					(self    :				Dict     ,  _A    :				int)   ->     str:
					if index in self.fairseq_ids_to_tokens:
						return self.fairseq_ids_to_tokens[index]
					return self.sp_model.IdToPiece(index - self.fairseq_offset)



				def  _lowercase					(self    :				Dict     ,  _A    :				List[str])   ->     List[Any]:
					__snake_case      :							Any						=			[]
					__snake_case      :							Dict						=			''
					__snake_case      :							Tuple						=			False
					for token in tokens:
						# make sure that special tokens are not decoded using sentencepiece model
						if token in self.all_special_tokens:
							if not prev_is_special:
								out_string += " "
							out_string += self.sp_model.decode(_A) + token
							__snake_case      :							int						=			True
							__snake_case      :							Optional[int]						=			[]
						else:
							current_sub_tokens.append(_A)
							__snake_case      :							Dict						=			False
					out_string += self.sp_model.decode(_A)
					return out_string.strip()



				def  _lowercase					(self    :				Tuple     ,  _A    :				str     ,  _A    :				Optional[str] = None)   ->     Tuple[str]:
					if not os.path.isdir(_A):
						logger.error(f"Vocabulary path ({save_directory}) should be a directory")
						return
					__snake_case      :							Any						=			os.path.join(
					    _A     ,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'])

					if os.path.abspath(self.vocab_file) != os.path.abspath(_A) and os.path.isfile(self.vocab_file):
						copyfile(self.vocab_file     ,  _A)
					elif not os.path.isfile(self.vocab_file):
						with open(_A     ,  'wb') as fi:
							__snake_case      :							List[str]						=			self.sp_model.serialized_model_proto()
							fi.write(_A)

					return (out_vocab_file,)



				def  _lowercase					(self    :				List[Any]     ,  _A    :				List[int]     ,  _A    :				Optional[List[int]] = None     ,  _A    :				bool = False)   ->     List[int]:

					if already_has_special_tokens:
						return super().get_special_tokens_mask(
						    token_ids_a=_A     ,  token_ids_a=_A     ,  already_has_special_tokens=_A)

					__snake_case      :							Any						=			[1] * len(self.prefix_tokens)
					__snake_case      :							int						=			[1] * len(self.suffix_tokens)
					if token_ids_a is None:
						return prefix_ones + ([0] * len(_A)) + suffix_ones
					return prefix_ones + ([0] * len(_A)) + ([0] * len(_A)) + suffix_ones



				def  _lowercase					(self    :				Optional[int]     ,  _A    :				List[int]     ,  _A    :				Optional[List[int]] = None)   ->     List[int]:
					if token_ids_a is None:
						return self.prefix_tokens + token_ids_a + self.suffix_tokens
					# We don't expect to process pairs, but leave the pair logic for API consistency
					return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens



				def  _lowercase					(self    :				Optional[Any]     ,  _A    :				Any     ,  _A    :				str     ,  _A    :				Optional[str]     ,  _A    :				Optional[str]     ,  **_A    :				List[Any])   ->     Tuple:
					if src_lang is None or tgt_lang is None:
						raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model')
					__snake_case      :							int						=			src_lang
					__snake_case      :							Any						=			self(_A     ,  add_special_tokens=_A     ,  return_tensors=_A     ,  **_A)
					__snake_case      :							int						=			self.convert_tokens_to_ids(_A)
					__snake_case      :							Tuple						=			tgt_lang_id
					return inputs



				def  _lowercase					(self    :				int     ,  _A    :				List[str]     ,  _A    :				str = "en_XX"     ,  _A    :				Optional[List[str]] = None     ,  _A    :				str = "ro_RO"     ,  **_A    :				Union[str, Any]     ,  )   ->     BatchEncoding:
					__snake_case      :							Optional[Any]						=			src_lang
					__snake_case      :							List[str]						=			tgt_lang
					return super().prepare_seqaseq_batch(_A     ,  _A     ,  **_A)



				def  _lowercase					(self    :				str)   ->     Any:
					return self.set_src_lang_special_tokens(self.src_lang)



				def  _lowercase					(self    :				Dict)   ->     Any:
					return self.set_tgt_lang_special_tokens(self.tgt_lang)



				def  _lowercase					(self    :				str     ,  _A    :				str)   ->     None:
					__snake_case      :							Optional[Any]						=			self.lang_code_to_id[src_lang]
					__snake_case      :							Tuple						=			[self.cur_lang_code_id]
					__snake_case      :							List[str]						=			[self.eos_token_id]






				def  _lowercase					(self    :				List[str]     ,  _A    :				str)   ->     None:
					__snake_case      :							Any						=			self.lang_code_to_id[tgt_lang]
					__snake_case      :							List[str]						=			[self.cur_lang_code_id]
					__snake_case      :							Optional[Any]						=			[self.eos_token_id]






"""simple docstring"""







def   __UpperCAmelCase    (   UpperCAmelCase_   :    list      ,							UpperCAmelCase_   :    int      ,							UpperCAmelCase_   :    int = 0      ,							UpperCAmelCase_   :    int = 0  )							->      int:

	'''simple docstring'''





	__snake_case      :							str						=			right or len(UpperCAmelCase_  ) - 1
	if left > right:
		return -1
	elif list_data[left] == key:
		return left
	elif list_data[right] == key:
		return right
	else:
		return search(UpperCAmelCase_      ,							UpperCAmelCase_      ,							left + 1      ,							right - 1  )


if __name__ == "__main__":
	import doctest

	doctest.testmod()





