


import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


_snake_case					:		str      =    logging.get_logger(__name__)

_snake_case					:		Tuple      =    "▁"

_snake_case					:		Tuple      =    {"vocab_file": "sentencepiece.bpe.model"}

_snake_case					:		int      =    {
    "vocab_file": {
        "facebook/mbart-large-50-one-to-many-mmt": (
            "https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model"
        ),
    }
}

_snake_case					:		str      =    {
    "facebook/mbart-large-50-one-to-many-mmt": 1_024,
}

# fmt: off
_snake_case					:		str      =    ["ar_AR", "cs_CZ", "de_DE", "en_XX", "es_XX", "et_EE", "fi_FI", "fr_XX", "gu_IN", "hi_IN", "it_IT", "ja_XX", "kk_KZ", "ko_KR", "lt_LT", "lv_LV", "my_MM", "ne_NP", "nl_XX", "ro_RO", "ru_RU", "si_LK", "tr_TR", "vi_VN", "zh_CN", "af_ZA", "az_AZ", "bn_IN", "fa_IR", "he_IL", "hr_HR", "id_ID", "ka_GE", "km_KH", "mk_MK", "ml_IN", "mn_MN", "mr_IN", "pl_PL", "ps_AF", "pt_XX", "sv_SE", "sw_KE", "ta_IN", "te_IN", "th_TH", "tl_XX", "uk_UA", "ur_PK", "xh_ZA", "gl_ES", "sl_SI"]

class     a		(__lowerCamelCase				):




							"""simple docstring"""


							__UpperCAmelCase   :							Any					  =  VOCAB_FILES_NAMES
							__UpperCAmelCase   :							Union[str, Any]					  =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							__UpperCAmelCase   :							List[Any]					  =  PRETRAINED_VOCAB_FILES_MAP
							__UpperCAmelCase   :							Union[str, Any]					  =  ['input_ids', 'attention_mask']

							__UpperCAmelCase   :							List[int]					  =  []
							__UpperCAmelCase   :							List[int]					  =  []

							def __init__(							self	:  Tuple   ,     lowerCamelCase	:  str   ,     lowerCamelCase	:  Optional[int]=None   ,     lowerCamelCase	:  List[str]=None   ,     lowerCamelCase	:  Optional[Any]="</s>"   ,     lowerCamelCase	:  Dict="</s>"   ,     lowerCamelCase	:  List[str]="<s>"   ,     lowerCamelCase	:  Tuple="<unk>"   ,     lowerCamelCase	:  int="<pad>"   ,     lowerCamelCase	:  Optional[int]="<mask>"   ,     lowerCamelCase	:  Optional[Dict[str, Any]] = None   ,     **lowerCamelCase	:  Optional[Any]   ,     )     ->   Optional[Any]:
														# Mask token behave like a normal word, i.e. include the space before it
														__snake_case		:      List[str]										=       AddedToken(UpperCamelCase_   ,     lstrip=UpperCamelCase_   ,     rstrip=UpperCamelCase_    ) if isinstance(UpperCamelCase_   ,     UpperCamelCase_    ) else mask_token

														__snake_case		:      Optional[int]										=       {} if sp_model_kwargs is None else sp_model_kwargs

														__snake_case		:      Union[str, Any]										=       kwargs.get("additional_special_tokens"   ,     []    )
														kwargs["additional_special_tokens"] += [
														    code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
														]

														super().__init__(
														    src_lang=UpperCamelCase_   ,     tgt_lang=UpperCamelCase_   ,     eos_token=UpperCamelCase_   ,     unk_token=UpperCamelCase_   ,     sep_token=UpperCamelCase_   ,     cls_token=UpperCamelCase_   ,     pad_token=UpperCamelCase_   ,     mask_token=UpperCamelCase_   ,     sp_model_kwargs=self.sp_model_kwargs   ,     **UpperCamelCase_   ,     )

														__snake_case		:      List[Any]										=       spm.SentencePieceProcessor(**self.sp_model_kwargs    )
														self.sp_model.Load(str(UpperCamelCase_    )    )
														__snake_case		:      Tuple										=       vocab_file

														# Original fairseq vocab and spm vocab must be "aligned":
														# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
														# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
														# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
														# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

														# Mimic fairseq token-to-id alignment for the first 4 token
														__snake_case		:      Optional[Any]										=       {"<s>": 0, "<pad>": 1, "</s>": 2, "<unk>": 3}

														# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
														__snake_case		:      Optional[int]										=       1

														__snake_case		:      int										=       len(self.sp_model    )
														__snake_case		:      Optional[int]										=       {
														    code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(UpperCamelCase_    )
														}
														__snake_case		:      Any										=       {v: k for k, v in self.lang_code_to_id.items()}
														__snake_case		:      int										=       len(self.sp_model    ) + len(self.lang_code_to_id    ) + self.fairseq_offset

														self.fairseq_tokens_to_ids.update(self.lang_code_to_id    )
														__snake_case		:      Any										=       {v: k for k, v in self.fairseq_tokens_to_ids.items()}

														__snake_case		:      List[Any]										=       src_lang if src_lang is not None else "en_XX"
														__snake_case		:      Dict										=       self.lang_code_to_id[self._src_lang]
														__snake_case		:      int										=       tgt_lang
														self.set_src_lang_special_tokens(self._src_lang    )

							@property
							def    __snake_case (							self	:  Dict    )     ->   Optional[Any]:
														return len(self.sp_model    ) + len(self.lang_code_to_id    ) + self.fairseq_offset + 1  # Plus 1 for the mask token

							@property
							def    __snake_case (							self	:  Dict    )     ->   str:
														return self._src_lang

							@src_lang.setter
							def    __snake_case (							self	:  Union[str, Any]   ,     lowerCamelCase	:  str    )     ->   List[str]:
														__snake_case		:      List[Any]										=       new_src_lang
														self.set_src_lang_special_tokens(self._src_lang    )

							def __getstate__(							self	:  Optional[int]    )     ->   int:
														__snake_case		:      Any										=       self.__dict__.copy()
														__snake_case		:      Dict										=       None
														return state

							def __setstate__(							self	:  Tuple   ,     lowerCamelCase	:  Dict    )     ->   Optional[Any]:
														__snake_case		:      Dict										=       d

														# for backward compatibility
														if not hasattr(self   ,     "sp_model_kwargs"    ):
																					__snake_case		:      str										=       {}

														__snake_case		:      Optional[Any]										=       spm.SentencePieceProcessor(**self.sp_model_kwargs    )
														self.sp_model.Load(self.vocab_file    )

							def    __snake_case (							self	:  Optional[int]    )     ->   Optional[Any]:
														__snake_case		:      Optional[Any]										=       {self.convert_ids_to_tokens(UpperCamelCase_    ): i for i in range(self.vocab_size    )}
														vocab.update(self.added_tokens_encoder    )
														return vocab

							def    __snake_case (							self	:  int   ,     lowerCamelCase	:  str    )     ->   int:
														return self.sp_model.encode(UpperCamelCase_   ,     out_type=UpperCamelCase_    )

							def    __snake_case (							self	:  List[Any]   ,     lowerCamelCase	:  str    )     ->   List[Any]:
														if token in self.fairseq_tokens_to_ids:
																					return self.fairseq_tokens_to_ids[token]
														__snake_case		:      str										=       self.sp_model.PieceToId(UpperCamelCase_    )

														# Need to return unknown token if the SP model returned 0
														return spm_id + self.fairseq_offset if spm_id else self.unk_token_id

							def    __snake_case (							self	:  Any   ,     lowerCamelCase	:  int    )     ->   Any:
														if index in self.fairseq_ids_to_tokens:
																					return self.fairseq_ids_to_tokens[index]
														return self.sp_model.IdToPiece(index - self.fairseq_offset    )

							def    __snake_case (							self	:  Union[str, Any]   ,     lowerCamelCase	:  Dict    )     ->   str:
														__snake_case		:      Optional[Any]										=       []
														__snake_case		:      List[str]										=       ""
														__snake_case		:      Dict										=       False
														for token in tokens:
																					# make sure that special tokens are not decoded using sentencepiece model
																					if token in self.all_special_tokens:
																												if not prev_is_special:
																																			out_string += " "
																												out_string += self.sp_model.decode(UpperCamelCase_    ) + token
																												__snake_case		:      Union[str, Any]										=       True
																												__snake_case		:      Union[str, Any]										=       []
																					else:
																												current_sub_tokens.append(UpperCamelCase_    )
																												__snake_case		:      List[str]										=       False
														out_string += self.sp_model.decode(UpperCamelCase_    )
														return out_string.strip()

							def    __snake_case (							self	:  Optional[int]   ,     lowerCamelCase	:  str   ,     lowerCamelCase	:  Optional[str] = None    )     ->   Optional[Any]:
														if not os.path.isdir(UpperCamelCase_    ):
																					logger.error(F'Vocabulary path ({save_directory}) should be a directory'    )
																					return
														__snake_case		:      List[str]										=       os.path.join(
														    UpperCamelCase_   ,     (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]    )

														if os.path.abspath(self.vocab_file    ) != os.path.abspath(UpperCamelCase_    ) and os.path.isfile(self.vocab_file    ):
																					copyfile(self.vocab_file   ,     UpperCamelCase_    )
														elif not os.path.isfile(self.vocab_file    ):
																					with open(UpperCamelCase_   ,     "wb"    ) as fi:
																												__snake_case		:      str										=       self.sp_model.serialized_model_proto()
																												fi.write(UpperCamelCase_    )

														return (out_vocab_file,)

							def    __snake_case (							self	:  List[Any]   ,     lowerCamelCase	:  List[int]   ,     lowerCamelCase	:  Optional[List[int]] = None   ,     lowerCamelCase	:  bool = False    )     ->   Union[str, Any]:

														if already_has_special_tokens:
																					return super().get_special_tokens_mask(
																					    token_ids_a=UpperCamelCase_   ,     token_ids_a=UpperCamelCase_   ,     already_has_special_tokens=UpperCamelCase_    )

														__snake_case		:      int										=       [1] * len(self.prefix_tokens    )
														__snake_case		:      Tuple										=       [1] * len(self.suffix_tokens    )
														if token_ids_a is None:
																					return prefix_ones + ([0] * len(UpperCamelCase_    )) + suffix_ones
														return prefix_ones + ([0] * len(UpperCamelCase_    )) + ([0] * len(UpperCamelCase_    )) + suffix_ones

							def    __snake_case (							self	:  List[str]   ,     lowerCamelCase	:  List[int]   ,     lowerCamelCase	:  Optional[List[int]] = None    )     ->   int:
														if token_ids_a is None:
																					return self.prefix_tokens + token_ids_a + self.suffix_tokens
														# We don't expect to process pairs, but leave the pair logic for API consistency
														return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens

							def    __snake_case (							self	:  Union[str, Any]   ,     lowerCamelCase	:  Any   ,     lowerCamelCase	:  str   ,     lowerCamelCase	:  Optional[str]   ,     lowerCamelCase	:  Optional[str]   ,     **lowerCamelCase	:  Optional[int]    )     ->   int:
														if src_lang is None or tgt_lang is None:
																					raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model"    )
														__snake_case		:      str										=       src_lang
														__snake_case		:      List[str]										=       self(UpperCamelCase_   ,     add_special_tokens=UpperCamelCase_   ,     return_tensors=UpperCamelCase_   ,     **UpperCamelCase_    )
														__snake_case		:      Optional[Any]										=       self.convert_tokens_to_ids(UpperCamelCase_    )
														__snake_case		:      Tuple										=       tgt_lang_id
														return inputs

							def    __snake_case (							self	:  str   ,     lowerCamelCase	:  List[str]   ,     lowerCamelCase	:  str = "en_XX"   ,     lowerCamelCase	:  Optional[List[str]] = None   ,     lowerCamelCase	:  str = "ro_RO"   ,     **lowerCamelCase	:  Union[str, Any]   ,     )     ->   Optional[int]:
														__snake_case		:      Tuple										=       src_lang
														__snake_case		:      Union[str, Any]										=       tgt_lang
														return super().prepare_seqaseq_batch(UpperCamelCase_   ,     UpperCamelCase_   ,     **UpperCamelCase_    )

							def    __snake_case (							self	:  Any    )     ->   int:
														return self.set_src_lang_special_tokens(self.src_lang    )

							def    __snake_case (							self	:  List[Any]    )     ->   Union[str, Any]:
														return self.set_tgt_lang_special_tokens(self.tgt_lang    )

							def    __snake_case (							self	:  Optional[Any]   ,     lowerCamelCase	:  str    )     ->   Optional[Any]:
														__snake_case		:      Tuple										=       self.lang_code_to_id[src_lang]
														__snake_case		:      Union[str, Any]										=       [self.cur_lang_code_id]
														__snake_case		:      Optional[int]										=       [self.eos_token_id]






							def    __snake_case (							self	:  Dict   ,     lowerCamelCase	:  str    )     ->   Dict:
														__snake_case		:      Optional[Any]										=       self.lang_code_to_id[tgt_lang]
														__snake_case		:      List[Any]										=       [self.cur_lang_code_id]
														__snake_case		:      int										=       [self.eos_token_id]





from typing import Dict

import numpy as np
import torch

from . import residue_constants as rc
from .tensor_utils import tensor_tree_map, tree_map







def     lowerCAmelCase_					( __lowerCamelCase     ):
							__snake_case		:      Any										=       []
							__snake_case		:      Optional[Any]										=       []
							__snake_case		:      List[Any]										=       []

							for rt in rc.restypes:
														__snake_case		:      Any										=       rc.restype_name_to_atomaa_names[rc.restype_atoa[rt]]
														restype_atomaa_to_atomaa_list.append([(rc.atom_order[name] if name else 0) for name in atom_names]     )
														__snake_case		:      Tuple										=       {name: i for i, name in enumerate(__lowerCamelCase     )}
														restype_atomaa_to_atomaa_list.append(
														    [(atom_name_to_idxaa[name] if name in atom_name_to_idxaa else 0) for name in rc.atom_types]     )

														restype_atomaa_mask_list.append([(1.0 if name else 0.0) for name in atom_names]     )

							# Add dummy mapping for restype 'UNK'
							restype_atomaa_to_atomaa_list.append([0] * 1_4     )
							restype_atomaa_to_atomaa_list.append([0] * 3_7     )
							restype_atomaa_mask_list.append([0.0] * 1_4     )

							__snake_case		:      int										=       torch.tensor(
							    __lowerCamelCase							,     dtype=torch.intaa							,     device=protein["aatype"].device							,     )
							__snake_case		:      List[str]										=       torch.tensor(
							    __lowerCamelCase							,     dtype=torch.intaa							,     device=protein["aatype"].device							,     )
							__snake_case		:      Optional[Any]										=       torch.tensor(
							    __lowerCamelCase							,     dtype=torch.floataa							,     device=protein["aatype"].device							,     )
							__snake_case		:      Optional[int]										=       protein["aatype"].to(torch.long     )

							# create the mapping for (residx, atom14) --> atom37, i.e. an array
							# with shape (num_res, 14) containing the atom37 indices for this protein
							__snake_case		:      Optional[Any]										=       restype_atomaa_to_atomaa[protein_aatype]
							__snake_case		:      Tuple										=       restype_atomaa_mask[protein_aatype]

							__snake_case		:      Optional[Any]										=       residx_atomaa_mask
							__snake_case		:      Union[str, Any]										=       residx_atomaa_to_atomaa.long()

							# create the gather indices for mapping back
							__snake_case		:      Dict										=       restype_atomaa_to_atomaa[protein_aatype]
							__snake_case		:      Dict										=       residx_atomaa_to_atomaa.long()

							# create the corresponding mask
							__snake_case		:      List[str]										=       torch.zeros([2_1, 3_7]							,     dtype=torch.floataa							,     device=protein["aatype"].device     )
							for restype, restype_letter in enumerate(rc.restypes     ):
														__snake_case		:      List[str]										=       rc.restype_atoa[restype_letter]
														__snake_case		:      List[Any]										=       rc.residue_atoms[restype_name]
														for atom_name in atom_names:
																					__snake_case		:      Union[str, Any]										=       rc.atom_order[atom_name]
																					__snake_case		:      str										=       1

							__snake_case		:      List[str]										=       restype_atomaa_mask[protein_aatype]
							__snake_case		:      List[str]										=       residx_atomaa_mask

							return protein







def     lowerCAmelCase_					( __lowerCamelCase     ):
							__snake_case		:      str										=       tree_map(lambda __lowerCamelCase     : torch.tensor(__lowerCamelCase							,     device=batch["aatype"].device     )							,     __lowerCamelCase							,     np.ndarray     )
							__snake_case		:      str										=       tensor_tree_map(lambda __lowerCamelCase     : np.array(__lowerCamelCase     )							,     make_atomaa_masks(__lowerCamelCase     )     )
							return out

