"""simple docstring"""







import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


_a					:					Union[str, Any]=			logging.get_logger(__name__)

_a					:					Optional[Any]=			"▁"

_a					:					Optional[int]=			{
    "vocab_file": "vocab.json",
    "spm_file": "sentencepiece.bpe.model",
    "tokenizer_config_file": "tokenizer_config.json",
}

_a					:					Optional[Any]=			{
    "vocab_file": {
        "facebook/m2m100_418M": "https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json",
        "facebook/m2m100_1.2B": "https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json",
    },
    "spm_file": {
        "facebook/m2m100_418M": "https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model",
        "facebook/m2m100_1.2B": "https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model",
    },
    "tokenizer_config_file": {
        "facebook/m2m100_418M": "https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json",
        "facebook/m2m100_1.2B": "https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json",
    },
}

_a					:					Dict=			{
    "facebook/m2m100_418M": 1_024,
}

# fmt: off
_a					:					Union[str, Any]=			{
    "m2m100": ["af", "am", "ar", "ast", "az", "ba", "be", "bg", "bn", "br", "bs", "ca", "ceb", "cs", "cy", "da", "de", "el", "en", "es", "et", "fa", "ff", "fi", "fr", "fy", "ga", "gd", "gl", "gu", "ha", "he", "hi", "hr", "ht", "hu", "hy", "id", "ig", "ilo", "is", "it", "ja", "jv", "ka", "kk", "km", "kn", "ko", "lb", "lg", "ln", "lo", "lt", "lv", "mg", "mk", "ml", "mn", "mr", "ms", "my", "ne", "nl", "no", "ns", "oc", "or", "pa", "pl", "ps", "pt", "ro", "ru", "sd", "si", "sk", "sl", "so", "sq", "sr", "ss", "su", "sv", "sw", "ta", "th", "tl", "tn", "tr", "uk", "ur", "uz", "vi", "wo", "xh", "yi", "yo", "zh", "zu"],
    "wmt21": ["en", "ha", "is", "ja", "cs", "ru", "zh", "de"]
}

class 		UpperCamelCase      (   lowercase						):
				UpperCAmelCase    :				int   =		VOCAB_FILES_NAMES
				UpperCAmelCase    :				Optional[Any]   =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				UpperCAmelCase    :				Any   =		PRETRAINED_VOCAB_FILES_MAP
				UpperCAmelCase    :				List[Any]   =		["""input_ids""", """attention_mask"""]

				UpperCAmelCase    :				List[int]   =		[]
				UpperCAmelCase    :				List[int]   =		[]



				def __init__(self    :				Optional[Any]     ,  _A    :				Any     ,  _A    :				Optional[int]     ,  _A    :				Union[str, Any]=None     ,  _A    :				Union[str, Any]=None     ,  _A    :				List[str]="<s>"     ,  _A    :				Tuple="</s>"     ,  _A    :				Optional[int]="</s>"     ,  _A    :				int="<pad>"     ,  _A    :				str="<unk>"     ,  _A    :				Optional[Any]="m2m100"     ,  _A    :				Optional[Dict[str, Any]] = None     ,  _A    :				str=8     ,  **_A    :				List[Any]     ,  )   ->     None:
					__snake_case      :							Tuple						=			{} if sp_model_kwargs is None else sp_model_kwargs

					__snake_case      :							Optional[int]						=			language_codes
					__snake_case      :							Dict						=			FAIRSEQ_LANGUAGE_CODES[language_codes]
					__snake_case      :							Union[str, Any]						=			{lang_code: f"__{lang_code}__" for lang_code in fairseq_language_code}

					__snake_case      :							Dict						=			kwargs.get('additional_special_tokens'     ,  [])
					kwargs["additional_special_tokens"] += [
					    self.get_lang_token(_A)
					    for lang_code in fairseq_language_code
					    if self.get_lang_token(_A) not in kwargs["additional_special_tokens"]
					]

					super().__init__(
					    src_lang=_A     ,  tgt_lang=_A     ,  bos_token=_A     ,  eos_token=_A     ,  sep_token=_A     ,  unk_token=_A     ,  pad_token=_A     ,  language_codes=_A     ,  sp_model_kwargs=self.sp_model_kwargs     ,  num_madeup_words=_A     ,  **_A     ,  )

					__snake_case      :							Optional[int]						=			vocab_file
					__snake_case      :							Dict						=			load_json(_A)
					__snake_case      :							Tuple						=			{v: k for k, v in self.encoder.items()}
					__snake_case      :							str						=			spm_file
					__snake_case      :							List[Any]						=			load_spm(_A     ,  self.sp_model_kwargs)

					__snake_case      :							Dict						=			len(self.encoder)

					__snake_case      :							List[Any]						=			{
					    self.get_lang_token(_A): self.encoder_size + i for i, lang_code in enumerate(_A)
					}
					__snake_case      :							str						=			{lang_code: self.encoder_size + i for i, lang_code in enumerate(_A)}
					__snake_case      :							Tuple						=			{v: k for k, v in self.lang_token_to_id.items()}

					__snake_case      :							Optional[Any]						=			src_lang if src_lang is not None else 'en'
					__snake_case      :							List[Any]						=			tgt_lang
					__snake_case      :							Optional[Any]						=			self.get_lang_id(self._src_lang)
					self.set_src_lang_special_tokens(self._src_lang)

					__snake_case      :							Optional[Any]						=			num_madeup_words



				@property
				def  _lowercase					(self    :				str)   ->     int:
					return len(self.encoder) + len(self.lang_token_to_id)



				@property
				def  _lowercase					(self    :				Any)   ->     str:
					return self._src_lang



				@src_lang.setter
				def  _lowercase					(self    :				Optional[Any]     ,  _A    :				str)   ->     None:
					__snake_case      :							Union[str, Any]						=			new_src_lang
					self.set_src_lang_special_tokens(self._src_lang)



				def  _lowercase					(self    :				Optional[Any]     ,  _A    :				str)   ->     List[str]:
					return self.sp_model.encode(_A     ,  out_type=_A)



				def  _lowercase					(self    :				List[str]     ,  _A    :				List[str])   ->     List[str]:
					if token in self.lang_token_to_id:
						return self.lang_token_to_id[token]
					return self.encoder.get(_A     ,  self.encoder[self.unk_token])



				def  _lowercase					(self    :				int     ,  _A    :				int)   ->     str:
					if index in self.id_to_lang_token:
						return self.id_to_lang_token[index]
					return self.decoder.get(_A     ,  self.unk_token)



				def  _lowercase					(self    :				Any     ,  _A    :				Optional[Any])   ->     Optional[int]:
					__snake_case      :							Optional[int]						=			[]
					__snake_case      :							Union[str, Any]						=			''
					for token in tokens:
						# make sure that special tokens are not decoded using sentencepiece model
						if token in self.all_special_tokens:
							out_string += self.sp_model.decode(_A) + token
							__snake_case      :							str						=			[]
						else:
							current_sub_tokens.append(_A)
					out_string += self.sp_model.decode(_A)
					return out_string.strip()



				def  _lowercase					(self    :				Dict     ,  _A    :				List[int]     ,  _A    :				Optional[List[int]] = None     ,  _A    :				bool = False)   ->     List[int]:

					if already_has_special_tokens:
						return super().get_special_tokens_mask(
						    token_ids_a=_A     ,  token_ids_a=_A     ,  already_has_special_tokens=_A)

					__snake_case      :							int						=			[1] * len(self.prefix_tokens)
					__snake_case      :							Any						=			[1] * len(self.suffix_tokens)
					if token_ids_a is None:
						return prefix_ones + ([0] * len(_A)) + suffix_ones
					return prefix_ones + ([0] * len(_A)) + ([0] * len(_A)) + suffix_ones



				def  _lowercase					(self    :				Optional[Any]     ,  _A    :				List[int]     ,  _A    :				Optional[List[int]] = None)   ->     List[int]:
					if token_ids_a is None:
						return self.prefix_tokens + token_ids_a + self.suffix_tokens
					# We don't expect to process pairs, but leave the pair logic for API consistency
					return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens



				def  _lowercase					(self    :				Tuple)   ->     Dict:
					__snake_case      :							int						=			{self.convert_ids_to_tokens(_A): i for i in range(self.vocab_size)}
					vocab.update(self.added_tokens_encoder)
					return vocab



				def __getstate__(self    :				Any)   ->     Dict:
					__snake_case      :							List[Any]						=			self.__dict__.copy()
					__snake_case      :							Optional[int]						=			None
					return state



				def __setstate__(self    :				List[Any]     ,  _A    :				Dict)   ->     None:
					__snake_case      :							Optional[Any]						=			d

					# for backward compatibility
					if not hasattr(self     ,  'sp_model_kwargs'):
						__snake_case      :							int						=			{}

					__snake_case      :							Union[str, Any]						=			load_spm(self.spm_file     ,  self.sp_model_kwargs)



				def  _lowercase					(self    :				Any     ,  _A    :				str     ,  _A    :				Optional[str] = None)   ->     Tuple[str]:
					__snake_case      :							Union[str, Any]						=			Path(_A)
					if not save_dir.is_dir():
						raise OSError(f"{save_directory} should be a directory")
					__snake_case      :							str						=			save_dir / (
					    (filename_prefix + '-' if filename_prefix else '') + self.vocab_files_names['vocab_file']
					)
					__snake_case      :							str						=			save_dir / (
					    (filename_prefix + '-' if filename_prefix else '') + self.vocab_files_names['spm_file']
					)

					save_json(self.encoder     ,  _A)

					if os.path.abspath(self.spm_file) != os.path.abspath(_A) and os.path.isfile(self.spm_file):
						copyfile(self.spm_file     ,  _A)
					elif not os.path.isfile(self.spm_file):
						with open(_A     ,  'wb') as fi:
							__snake_case      :							Union[str, Any]						=			self.sp_model.serialized_model_proto()
							fi.write(_A)

					return (str(_A), str(_A))



				def  _lowercase					(self    :				Tuple     ,  _A    :				List[str]     ,  _A    :				str = "en"     ,  _A    :				Optional[List[str]] = None     ,  _A    :				str = "ro"     ,  **_A    :				List[Any]     ,  )   ->     BatchEncoding:
					__snake_case      :							int						=			src_lang
					__snake_case      :							Tuple						=			tgt_lang
					self.set_src_lang_special_tokens(self.src_lang)
					return super().prepare_seqaseq_batch(_A     ,  _A     ,  **_A)



				def  _lowercase					(self    :				Dict     ,  _A    :				Any     ,  _A    :				Optional[str]     ,  _A    :				Optional[str]     ,  **_A    :				Union[str, Any])   ->     List[Any]:
					if src_lang is None or tgt_lang is None:
						raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model')
					__snake_case      :							List[str]						=			src_lang
					__snake_case      :							Union[str, Any]						=			self(_A     ,  add_special_tokens=_A     ,  **_A)
					__snake_case      :							int						=			self.get_lang_id(_A)
					__snake_case      :							Union[str, Any]						=			tgt_lang_id
					return inputs



				def  _lowercase					(self    :				Union[str, Any])   ->     str:
					self.set_src_lang_special_tokens(self.src_lang)



				def  _lowercase					(self    :				Tuple)   ->     Dict:
					self.set_tgt_lang_special_tokens(self.tgt_lang)



				def  _lowercase					(self    :				Any     ,  _A    :				str)   ->     None:
					__snake_case      :							List[Any]						=			self.get_lang_token(_A)
					__snake_case      :							Union[str, Any]						=			self.lang_token_to_id[lang_token]
					__snake_case      :							Dict						=			[self.cur_lang_id]
					__snake_case      :							int						=			[self.eos_token_id]



				def  _lowercase					(self    :				List[str]     ,  _A    :				str)   ->     None:
					__snake_case      :							Optional[int]						=			self.get_lang_token(_A)
					__snake_case      :							Optional[Any]						=			self.lang_token_to_id[lang_token]
					__snake_case      :							Optional[int]						=			[self.cur_lang_id]
					__snake_case      :							int						=			[self.eos_token_id]



				def  _lowercase					(self    :				str     ,  _A    :				str)   ->     str:
					return self.lang_code_to_token[lang]






				def  _lowercase					(self    :				List[Any]     ,  _A    :				str)   ->     int:
					__snake_case      :							Optional[int]						=			self.get_lang_token(_A)
					return self.lang_token_to_id[lang_token]







def   __UpperCAmelCase    (   UpperCAmelCase_   :    str      ,							UpperCAmelCase_   :    Dict[str, Any]  )							->      sentencepiece.SentencePieceProcessor:

	'''simple docstring'''





	__snake_case      :							Any						=			sentencepiece.SentencePieceProcessor(**UpperCAmelCase_  )
	spm.Load(str(UpperCAmelCase_  )  )
	return spm
def   __UpperCAmelCase    (   UpperCAmelCase_   :    str  )							->      Union[Dict, List]:

	'''simple docstring'''





	with open(UpperCAmelCase_      ,							'r'  ) as f:
		return json.load(UpperCAmelCase_  )

def   __UpperCAmelCase    (   UpperCAmelCase_   :    str      ,							UpperCAmelCase_   :    str  )							->      None:

	'''simple docstring'''





	with open(UpperCAmelCase_      ,							'w'  ) as f:
		json.dump(UpperCAmelCase_      ,							UpperCAmelCase_      ,							indent=2  )






"""simple docstring"""







def   __UpperCAmelCase    (   UpperCAmelCase_   :    list      ,							UpperCAmelCase_   :    int      ,							UpperCAmelCase_   :    int = 0      ,							UpperCAmelCase_   :    int = 0  )							->      int:

	'''simple docstring'''





	__snake_case      :							str						=			right or len(UpperCAmelCase_  ) - 1
	if left > right:
		return -1
	elif list_data[left] == key:
		return left
	elif list_data[right] == key:
		return right
	else:
		return search(UpperCAmelCase_      ,							UpperCAmelCase_      ,							left + 1      ,							right - 1  )


if __name__ == "__main__":
	import doctest

	doctest.testmod()





