



'''simple docstring'''






import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase: Tuple							=						logging.get_logger(__name__)

lowerCAmelCase: List[str]							=						"▁"

lowerCAmelCase: Tuple							=						{"vocab_file": "sentencepiece.bpe.model"}

lowerCAmelCase: List[Any]							=						{
    "vocab_file": {
        "facebook/mbart-large-50-one-to-many-mmt": (
            "https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model"
        ),
    }
}

lowerCAmelCase: List[str]							=						{
    "facebook/mbart-large-50-one-to-many-mmt": 1_0_2_4,
}

# fmt: off
lowerCAmelCase: Union[str, Any]							=						["ar_AR", "cs_CZ", "de_DE", "en_XX", "es_XX", "et_EE", "fi_FI", "fr_XX", "gu_IN", "hi_IN", "it_IT", "ja_XX", "kk_KZ", "ko_KR", "lt_LT", "lv_LV", "my_MM", "ne_NP", "nl_XX", "ro_RO", "ru_RU", "si_LK", "tr_TR", "vi_VN", "zh_CN", "af_ZA", "az_AZ", "bn_IN", "fa_IR", "he_IL", "hr_HR", "id_ID", "ka_GE", "km_KH", "mk_MK", "ml_IN", "mn_MN", "mr_IN", "pl_PL", "ps_AF", "pt_XX", "sv_SE", "sw_KE", "ta_IN", "te_IN", "th_TH", "tl_XX", "uk_UA", "ur_PK", "xh_ZA", "gl_ES", "sl_SI"]

class 							a__( lowercase_     ):
    lowercase__			=   VOCAB_FILES_NAMES
    lowercase__			=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowercase__			=   PRETRAINED_VOCAB_FILES_MAP
    lowercase__			=   ["input_ids", "attention_mask"]

    lowercase__			=   []
    lowercase__			=   []







    def __init__(				self			:  List[Any]						,      __snake_case			:  Optional[int]						,      __snake_case			:  List[Any]=None						,      __snake_case			:  List[str]=None						,      __snake_case			:  Optional[int]="</s>"						,      __snake_case			:  List[str]="</s>"						,      __snake_case			:  str="<s>"						,      __snake_case			:  str="<unk>"						,      __snake_case			:  List[Any]="<pad>"						,      __snake_case			:  List[Any]="<mask>"						,      __snake_case			:  Optional[Any] = None						,      **__snake_case			:  Any						,      ):
      a						: List[Any]					       =					AddedToken(a__						,      lstrip=a__						,      rstrip=a__     ) if isinstance(a__						,      a__     ) else mask_token

      a						: str					       =					{} if sp_model_kwargs is None else sp_model_kwargs

      a						: Union[str, Any]					       =					kwargs.get('additional_special_tokens'						,      []     )
      kwargs["additional_special_tokens"] += [
          code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
      ]

      super().__init__(
          src_lang=a__						,      tgt_lang=a__						,      eos_token=a__						,      unk_token=a__						,      sep_token=a__						,      cls_token=a__						,      pad_token=a__						,      mask_token=a__						,      sp_model_kwargs=self.sp_model_kwargs						,      **a__						,      )

      a						: Dict					       =					spm.SentencePieceProcessor(**self.sp_model_kwargs     )
      self.sp_model.Load(str(a__     )     )
      a						: Tuple					       =					vocab_file

      # Original fairseq vocab and spm vocab must be "aligned":
      # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
      # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
      # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
      # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

      # Mimic fairseq token-to-id alignment for the first 4 token
      a						: Any					       =					{'<s>': 0, '<pad>': 1, '</s>': 2, '<unk>': 3}

      # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
      a						: Optional[Any]					       =					1

      a						: Dict					       =					len(self.sp_model     )
      a						: str					       =					{
          code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(a__     )
      }
      a						: Optional[Any]					       =					{v: k for k, v in self.lang_code_to_id.items()}
      a						: Tuple					       =					len(self.sp_model     ) + len(self.lang_code_to_id     ) + self.fairseq_offset

      self.fairseq_tokens_to_ids.update(self.lang_code_to_id     )
      a						: Tuple					       =					{v: k for k, v in self.fairseq_tokens_to_ids.items()}

      a						: List[str]					       =					src_lang if src_lang is not None else 'en_XX'
      a						: Dict					       =					self.lang_code_to_id[self._src_lang]
      a						: Tuple					       =					tgt_lang
      self.set_src_lang_special_tokens(self._src_lang     )







    @property
    def       lowercase_	(				self			:  int     ):
      return len(self.sp_model     ) + len(self.lang_code_to_id     ) + self.fairseq_offset + 1  # Plus 1 for the mask token







    @property
    def       lowercase_	(				self			:  List[str]     ):
      return self._src_lang







    @src_lang.setter
    def       lowercase_	(				self			:  Optional[int]						,      __snake_case			:  Any     ):
      a						: Optional[Any]					       =					new_src_lang
      self.set_src_lang_special_tokens(self._src_lang     )







    def __getstate__(				self			:  Union[str, Any]     ):
      a						: Optional[int]					       =					self.__dict__.copy()
      a						: Optional[int]					       =					None
      return state







    def __setstate__(				self			:  Any						,      __snake_case			:  Optional[int]     ):
      a						: Dict					       =					d

      # for backward compatibility
      if not hasattr(self						,      'sp_model_kwargs'     ):
        a						: Any					       =					{}

      a						: str					       =					spm.SentencePieceProcessor(**self.sp_model_kwargs     )
      self.sp_model.Load(self.vocab_file     )







    def       lowercase_	(				self			:  Optional[int]     ):
      a						: List[Any]					       =					{self.convert_ids_to_tokens(a__     ): i for i in range(self.vocab_size     )}
      vocab.update(self.added_tokens_encoder     )
      return vocab







    def       lowercase_	(				self			:  List[Any]						,      __snake_case			:  Union[str, Any]     ):
      return self.sp_model.encode(a__						,      out_type=a__     )







    def       lowercase_	(				self			:  List[Any]						,      __snake_case			:  Dict     ):




      if token in self.fairseq_tokens_to_ids:
        return self.fairseq_tokens_to_ids[token]
      a						: List[str]					       =					self.sp_model.PieceToId(a__     )

      # Need to return unknown token if the SP model returned 0
      return spm_id + self.fairseq_offset if spm_id else self.unk_token_id







    def       lowercase_	(				self			:  List[Any]						,      __snake_case			:  Optional[int]     ):




      if index in self.fairseq_ids_to_tokens:
        return self.fairseq_ids_to_tokens[index]
      return self.sp_model.IdToPiece(index - self.fairseq_offset     )







    def       lowercase_	(				self			:  List[str]						,      __snake_case			:  int     ):
      a						: List[str]					       =					[]
      a						: List[Any]					       =					''
      a						: Optional[Any]					       =					False
      for token in tokens:
        # make sure that special tokens are not decoded using sentencepiece model
        if token in self.all_special_tokens:
          if not prev_is_special:
            out_string += " "
          out_string += self.sp_model.decode(a__     ) + token
          a						: str					       =					True
          a						: Optional[Any]					       =					[]
        else:
          current_sub_tokens.append(a__     )
          a						: Union[str, Any]					       =					False
      out_string += self.sp_model.decode(a__     )
      return out_string.strip()







    def       lowercase_	(				self			:  Dict						,      __snake_case			:  Dict						,      __snake_case			:  Any = None     ):




      if not os.path.isdir(a__     ):
        logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""     )
        return
      a						: Union[str, Any]					       =					os.path.join(
          a__						,      (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']     )

      if os.path.abspath(self.vocab_file     ) != os.path.abspath(a__     ) and os.path.isfile(self.vocab_file     ):
        copyfile(self.vocab_file						,      a__     )
      elif not os.path.isfile(self.vocab_file     ):
        with open(a__						,      'wb'     ) as fi:
          a						: List[Any]					       =					self.sp_model.serialized_model_proto()
          fi.write(a__     )

      return (out_vocab_file,)







    def       lowercase_	(				self			:  Dict						,      __snake_case			:  str						,      __snake_case			:  int = None						,      __snake_case			:  Optional[int] = False     ):




      if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_a=a__						,      token_ids_a=a__						,      already_has_special_tokens=a__     )

      a						: Union[str, Any]					       =					[1] * len(self.prefix_tokens     )
      a						: Tuple					       =					[1] * len(self.suffix_tokens     )
      if token_ids_a is None:
        return prefix_ones + ([0] * len(a__     )) + suffix_ones
      return prefix_ones + ([0] * len(a__     )) + ([0] * len(a__     )) + suffix_ones







    def       lowercase_	(				self			:  List[str]						,      __snake_case			:  Any						,      __snake_case			:  str = None     ):




      if token_ids_a is None:
        return self.prefix_tokens + token_ids_a + self.suffix_tokens
      # We don't expect to process pairs, but leave the pair logic for API consistency
      return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens







    def       lowercase_	(				self			:  int						,      __snake_case			:  List[str]						,      __snake_case			:  str						,      __snake_case			:  Union[str, Any]						,      __snake_case			:  List[str]						,      **__snake_case			:  Union[str, Any]     ):




      if src_lang is None or tgt_lang is None:
        raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'     )
      a						: List[str]					       =					src_lang
      a						: Dict					       =					self(a__						,      add_special_tokens=a__						,      return_tensors=a__						,      **a__     )
      a						: Optional[Any]					       =					self.convert_tokens_to_ids(a__     )
      a						: Optional[int]					       =					tgt_lang_id
      return inputs







    def       lowercase_	(				self			:  int						,      __snake_case			:  Dict						,      __snake_case			:  int = "en_XX"						,      __snake_case			:  str = None						,      __snake_case			:  List[Any] = "ro_RO"						,      **__snake_case			:  Optional[int]						,      ):
      a						: int					       =					src_lang
      a						: int					       =					tgt_lang
      return super().prepare_seqaseq_batch(a__						,      a__						,      **a__     )







    def       lowercase_	(				self			:  Tuple     ):
      return self.set_src_lang_special_tokens(self.src_lang     )







    def       lowercase_	(				self			:  Tuple     ):
      return self.set_tgt_lang_special_tokens(self.tgt_lang     )







    def       lowercase_	(				self			:  List[Any]						,      __snake_case			:  Dict     ):
      a						: List[str]					       =					self.lang_code_to_id[src_lang]
      a						: Union[str, Any]					       =					[self.cur_lang_code_id]
      a						: Union[str, Any]					       =					[self.eos_token_id]







    def       lowercase_	(				self			:  List[str]						,      __snake_case			:  Dict     ):
      a						: Optional[int]					       =					self.lang_code_to_id[tgt_lang]
      a						: Optional[int]					       =					[self.cur_lang_code_id]
      a						: List[Any]					       =					[self.eos_token_id]




'''simple docstring'''






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tokenizers_available,
    is_torch_available,
)


lowerCAmelCase: List[Any]							=						{}

try:
     if not is_sentencepiece_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     lowerCAmelCase: Optional[int]							=						['NllbTokenizer']

try:
     if not is_tokenizers_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     lowerCAmelCase: Dict							=						['NllbTokenizerFast']


if TYPE_CHECKING:
     try:
          if not is_sentencepiece_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .tokenization_nllb import NllbTokenizer

     try:
          if not is_tokenizers_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .tokenization_nllb_fast import NllbTokenizerFast

else:
     import sys

     lowerCAmelCase: Tuple							=						_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)