'''simple docstring'''





import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE_:					List[Any]  =logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_:					Optional[int]  ='▁'

SCREAMING_SNAKE_CASE_:					List[Any]  ={'vocab_file': 'sentencepiece.bpe.model'}

SCREAMING_SNAKE_CASE_:					Any  ={
    'vocab_file': {
        'facebook/mbart-large-en-ro': (
            'https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model'
        ),
        'facebook/mbart-large-cc25': (
            'https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model'
        ),
    }
}

SCREAMING_SNAKE_CASE_:					List[str]  ={
    'facebook/mbart-large-en-ro': 10_24,
    'facebook/mbart-large-cc25': 10_24,
}

# fmt: off
SCREAMING_SNAKE_CASE_:					Tuple  =['ar_AR', 'cs_CZ', 'de_DE', 'en_XX', 'es_XX', 'et_EE', 'fi_FI', 'fr_XX', 'gu_IN', 'hi_IN', 'it_IT', 'ja_XX', 'kk_KZ', 'ko_KR', 'lt_LT', 'lv_LV', 'my_MM', 'ne_NP', 'nl_XX', 'ro_RO', 'ru_RU', 'si_LK', 'tr_TR', 'vi_VN', 'zh_CN']




class 	__A				(			SCREAMING_SNAKE_CASE__     ):
    a__	:						Any	       =  VOCAB_FILES_NAMES
    a__	:						Optional[int]	       =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    a__	:						List[str]	       =  PRETRAINED_VOCAB_FILES_MAP
    a__	:						List[Any]	       =  ["""input_ids""", """attention_mask"""]

    a__	:						Union[str, Any]	       =  []
    a__	:						Optional[int]	       =  []



    def __init__(self   :   List[str]       ,      __a   :   List[Any]       ,      __a   :   Union[str, Any]="<s>"       ,      __a   :   Union[str, Any]="</s>"       ,      __a   :   List[Any]="</s>"       ,      __a   :   str="<s>"       ,      __a   :   Tuple="<unk>"       ,      __a   :   str="<pad>"       ,      __a   :   List[str]="<mask>"       ,      __a   :   Any=None       ,      __a   :   int=None       ,      __a   :   Tuple=None       ,      __a   :   Optional[Dict[str, Any]] = None       ,      __a   :   List[str]=None       ,      **__a   :   Tuple       ,      ):
         # Mask token behave like a normal word, i.e. include the space before it
         UpperCAmelCase_	   =       AddedToken(__a       ,      lstrip=__a       ,      rstrip=__a			) if isinstance(__a       ,      __a			) else mask_token

         UpperCAmelCase_	   =       {} if sp_model_kwargs is None else sp_model_kwargs

         super().__init__(
             bos_token=__a       ,      eos_token=__a       ,      unk_token=__a       ,      sep_token=__a       ,      cls_token=__a       ,      pad_token=__a       ,      mask_token=__a       ,      tokenizer_file=__a       ,      src_lang=__a       ,      tgt_lang=__a       ,      additional_special_tokens=__a       ,      sp_model_kwargs=self.sp_model_kwargs       ,      **__a       ,      )

         UpperCAmelCase_	   =       spm.SentencePieceProcessor(**self.sp_model_kwargs			)
         self.sp_model.Load(str(__a			)			)
         UpperCAmelCase_	   =       vocab_file

         # Original fairseq vocab and spm vocab must be "aligned":
         # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
         # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
         # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
         # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

         # Mimic fairseq token-to-id alignment for the first 4 token
         UpperCAmelCase_	   =       {'<s>': 0, '<pad>': 1, '</s>': 2, '<unk>': 3}

         # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
         UpperCAmelCase_	   =       1

         UpperCAmelCase_	   =       len(self.sp_model			)
         UpperCAmelCase_	   =       {
             code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(__a			)
         }
         UpperCAmelCase_	   =       {v: k for k, v in self.lang_code_to_id.items()}
         UpperCAmelCase_	   =       len(self.sp_model			) + len(self.lang_code_to_id			) + self.fairseq_offset

         self.fairseq_tokens_to_ids.update(self.lang_code_to_id			)
         UpperCAmelCase_	   =       {v: k for k, v in self.fairseq_tokens_to_ids.items()}
         UpperCAmelCase_	   =       list(self.lang_code_to_id.keys()			)

         if additional_special_tokens is not None:
              # Only add those special tokens if they are not already there.
              self._additional_special_tokens.extend(
                  [t for t in additional_special_tokens if t not in self._additional_special_tokens]			)

         UpperCAmelCase_	   =       src_lang if src_lang is not None else 'en_XX'
         UpperCAmelCase_	   =       self.lang_code_to_id[self._src_lang]
         UpperCAmelCase_	   =       tgt_lang
         self.set_src_lang_special_tokens(self._src_lang			)



    def __getstate__(self   :   Optional[int]			):
         UpperCAmelCase_	   =       self.__dict__.copy()
         UpperCAmelCase_	   =       None
         UpperCAmelCase_	   =       self.sp_model.serialized_model_proto()
         return state



    def __setstate__(self   :   Any       ,      __a   :   Dict			):
         UpperCAmelCase_	   =       d

         # for backward compatibility
         if not hasattr(self       ,      "sp_model_kwargs"			):
              UpperCAmelCase_	   =       {}

         UpperCAmelCase_	   =       spm.SentencePieceProcessor(**self.sp_model_kwargs			)
         self.sp_model.LoadFromSerializedProto(self.sp_model_proto			)



    @property
    def  _lowercase						(self   :   List[Any]			):
         return len(self.sp_model			) + len(self.lang_code_to_id			) + self.fairseq_offset + 1  # Plus 1 for the mask token



    @property
    def  _lowercase						(self   :   Union[str, Any]			):
         return self._src_lang



    @src_lang.setter
    def  _lowercase						(self   :   Union[str, Any]       ,      __a   :   str			):
         UpperCAmelCase_	   =       new_src_lang
         self.set_src_lang_special_tokens(self._src_lang			)



    def  _lowercase						(self   :   str       ,      __a   :   List[int]       ,      __a   :   Optional[List[int]] = None       ,      __a   :   bool = False			):

         if already_has_special_tokens:
              return super().get_special_tokens_mask(
                  token_ids_a=__a       ,      token_ids_a=__a       ,      already_has_special_tokens=__a			)

         UpperCAmelCase_	   =       [1] * len(self.prefix_tokens			)
         UpperCAmelCase_	   =       [1] * len(self.suffix_tokens			)
         if token_ids_a is None:
              return prefix_ones + ([0] * len(__a			)) + suffix_ones
         return prefix_ones + ([0] * len(__a			)) + ([0] * len(__a			)) + suffix_ones



    def  _lowercase						(self   :   Any       ,      __a   :   List[int]       ,      __a   :   Optional[List[int]] = None			):
         if token_ids_a is None:
              return self.prefix_tokens + token_ids_a + self.suffix_tokens
         # We don't expect to process pairs, but leave the pair logic for API consistency
         return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens



    def  _lowercase						(self   :   Tuple       ,      __a   :   List[int]       ,      __a   :   Optional[List[int]] = None			):
         UpperCAmelCase_	   =       [self.sep_token_id]
         UpperCAmelCase_	   =       [self.cls_token_id]

         if token_ids_a is None:
              return len(cls + token_ids_a + sep			) * [0]
         return len(cls + token_ids_a + sep + sep + token_ids_a + sep			) * [0]



    def  _lowercase						(self   :   str       ,      __a   :   Optional[int]       ,      __a   :   str       ,      __a   :   Optional[str]       ,      __a   :   Optional[str]       ,      **__a   :   str			):
         if src_lang is None or tgt_lang is None:
              raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model"			)
         UpperCAmelCase_	   =       src_lang
         UpperCAmelCase_	   =       self(__a       ,      add_special_tokens=__a       ,      return_tensors=__a       ,      **__a			)
         UpperCAmelCase_	   =       self.convert_tokens_to_ids(__a			)
         UpperCAmelCase_	   =       tgt_lang_id
         return inputs



    def  _lowercase						(self   :   List[Any]			):
         UpperCAmelCase_	   =       {self.convert_ids_to_tokens(__a			): i for i in range(self.vocab_size			)}
         vocab.update(self.added_tokens_encoder			)
         return vocab



    def  _lowercase						(self   :   Union[str, Any]       ,      __a   :   str			):
         return self.sp_model.encode(__a       ,      out_type=__a			)



    def  _lowercase						(self   :   Union[str, Any]       ,      __a   :   List[Any]			):
         if token in self.fairseq_tokens_to_ids:
              return self.fairseq_tokens_to_ids[token]
         UpperCAmelCase_	   =       self.sp_model.PieceToId(__a			)

         # Need to return unknown token if the SP model returned 0
         return spm_id + self.fairseq_offset if spm_id else self.unk_token_id



    def  _lowercase						(self   :   str       ,      __a   :   Tuple			):
         if index in self.fairseq_ids_to_tokens:
              return self.fairseq_ids_to_tokens[index]
         return self.sp_model.IdToPiece(index - self.fairseq_offset			)



    def  _lowercase						(self   :   int       ,      __a   :   Tuple			):
         UpperCAmelCase_	   =       ''.join(__a			).replace(__a       ,      " "			).strip()
         return out_string



    def  _lowercase						(self   :   List[Any]       ,      __a   :   str       ,      __a   :   Optional[str] = None			):
         if not os.path.isdir(__a			):
              logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""			)
              return
         UpperCAmelCase_	   =       os.path.join(
             __a       ,      (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]			)

         if os.path.abspath(self.vocab_file			) != os.path.abspath(__a			) and os.path.isfile(self.vocab_file			):
              copyfile(self.vocab_file       ,      __a			)
         elif not os.path.isfile(self.vocab_file			):
              with open(__a       ,      "wb"			) as fi:
                   UpperCAmelCase_	   =       self.sp_model.serialized_model_proto()
                   fi.write(__a			)

         return (out_vocab_file,)



    def  _lowercase						(self   :   List[Any]       ,      __a   :   List[str]       ,      __a   :   str = "en_XX"       ,      __a   :   Optional[List[str]] = None       ,      __a   :   str = "ro_RO"       ,      **__a   :   Dict       ,      ):
         UpperCAmelCase_	   =       src_lang
         UpperCAmelCase_	   =       tgt_lang
         return super().prepare_seqaseq_batch(__a       ,      __a       ,      **__a			)



    def  _lowercase						(self   :   List[Any]			):
         return self.set_src_lang_special_tokens(self.src_lang			)



    def  _lowercase						(self   :   Any			):
         return self.set_tgt_lang_special_tokens(self.tgt_lang			)



    def  _lowercase						(self   :   Optional[Any]       ,      __a   :   Union[str, Any]			):
         UpperCAmelCase_	   =       self.lang_code_to_id[src_lang]
         UpperCAmelCase_	   =       []
         UpperCAmelCase_	   =       [self.eos_token_id, self.cur_lang_code]






    def  _lowercase						(self   :   Optional[Any]       ,      __a   :   str			):
         UpperCAmelCase_	   =       self.lang_code_to_id[lang]
         UpperCAmelCase_	   =       []
         UpperCAmelCase_	   =       [self.eos_token_id, self.cur_lang_code]





'''simple docstring'''





# Lint as: python3
import sys
from collections.abc import Mapping
from typing import TYPE_CHECKING, Dict, Optional

import numpy as np
import pyarrow as pa

from .. import config
from ..utils.logging import get_logger
from ..utils.py_utils import map_nested
from .formatting import TensorFormatter


if TYPE_CHECKING:
    import jax
    import jaxlib

SCREAMING_SNAKE_CASE_:					List[str]  =get_logger()

SCREAMING_SNAKE_CASE_:					Optional[dict]  =None




class 	__A				(			TensorFormatter[Mapping, """jax.Array""", Mapping]     ):



    def __init__(self   :   List[Any]       ,      __a   :   Optional[int]=None       ,      __a   :   Any=None       ,      **__a   :   Dict			):
         super().__init__(features=__a			)
         import jax
         from jaxlib.xla_client import Device

         if isinstance(__a       ,      __a			):
              raise ValueError(
                  f"""Expected {device} to be a `str` not {type(__a			)}, as `jaxlib.xla_extension.Device` """
                  "is not serializable neither with `pickle` nor with `dill`. Instead you can surround "
                  "the device with `str()` to get its string identifier that will be internally mapped "
                  "to the actual `jaxlib.xla_extension.Device`."			)
         UpperCAmelCase_	   =       device if isinstance(__a       ,      __a			) else str(jax.devices()[0]			)
         # using global variable since `jaxlib.xla_extension.Device` is not serializable neither
         # with `pickle` nor with `dill`, so we need to use a global variable instead
         global DEVICE_MAPPING
         if DEVICE_MAPPING is None:
              UpperCAmelCase_	   =       self._map_devices_to_str()
         if self.device not in list(DEVICE_MAPPING.keys()			):
              logger.warning(
                  f"""Device with string identifier {self.device} not listed among the available """
                  f"""devices: {list(DEVICE_MAPPING.keys()			)}, so falling back to the default """
                  f"""device: {str(jax.devices()[0]			)}."""			)
              UpperCAmelCase_	   =       str(jax.devices()[0]			)
         UpperCAmelCase_	   =       jnp_array_kwargs



    @staticmethod
    def  _lowercase						():
         import jax

         return {str(__a			): device for device in jax.devices()}



    def  _lowercase						(self   :   str       ,      __a   :   Tuple			):
         import jax
         import jax.numpy as jnp

         if isinstance(__a       ,      __a			) and column:
              if all(
                  isinstance(__a       ,      jax.Array			) and x.shape == column[0].shape and x.dtype == column[0].dtype for x in column			):
                   return jnp.stack(__a       ,      axis=0			)
         return column



    def  _lowercase						(self   :   Any       ,      __a   :   Optional[int]			):
         import jax
         import jax.numpy as jnp

         if isinstance(__a       ,      (str, bytes, type(__a			))			):
              return value
         elif isinstance(__a       ,      (np.character, np.ndarray)			) and np.issubdtype(value.dtype       ,      np.character			):
              return value.tolist()

         UpperCAmelCase_	   =       {}

         if isinstance(__a       ,      (np.number, np.ndarray)			) and np.issubdtype(value.dtype       ,      np.integer			):
              # the default int precision depends on the jax config
              # see https://jax.readthedocs.io/en/latest/notebooks/Common_Gotchas_in_JAX.html#double-64bit-precision
              if jax.config.jax_enable_xaa:
                   UpperCAmelCase_	   =       {"dtype": jnp.intaa}
              else:
                   UpperCAmelCase_	   =       {"dtype": jnp.intaa}
         elif isinstance(__a       ,      (np.number, np.ndarray)			) and np.issubdtype(value.dtype       ,      np.floating			):
              UpperCAmelCase_	   =       {"dtype": jnp.floataa}
         elif config.PIL_AVAILABLE and "PIL" in sys.modules:
              import PIL.Image

              if isinstance(__a       ,      PIL.Image.Image			):
                   UpperCAmelCase_	   =       np.asarray(__a			)

        # using global variable since `jaxlib.xla_extension.Device` is not serializable neither
        # with `pickle` nor with `dill`, so we need to use a global variable instead
         global DEVICE_MAPPING
         if DEVICE_MAPPING is None:
              UpperCAmelCase_	   =       self._map_devices_to_str()

         with jax.default_device(DEVICE_MAPPING[self.device]			):
              # calling jnp.array on a np.ndarray does copy the data
              # see https://github.com/google/jax/issues/4486
              return jnp.array(__a       ,      **{**default_dtype, **self.jnp_array_kwargs}			)



    def  _lowercase						(self   :   int       ,      __a   :   Any			):
         import jax

         # support for torch, tf, jax etc.
         if config.TORCH_AVAILABLE and "torch" in sys.modules:
              import torch

              if isinstance(__a       ,      torch.Tensor			):
                   return self._tensorize(data_struct.detach().cpu().numpy()[()]			)
         if hasattr(__a       ,      "__array__"			) and not isinstance(__a       ,      jax.Array			):
              UpperCAmelCase_	   =       data_struct.__array__()
         # support for nested types like struct of list of struct
         if isinstance(__a       ,      np.ndarray			):
              if data_struct.dtype == object:  # jax arrays cannot be instantied from an array of objects
                   return self._consolidate([self.recursive_tensorize(__a			) for substruct in data_struct]			)
         elif isinstance(__a       ,      (list, tuple)			):
              return self._consolidate([self.recursive_tensorize(__a			) for substruct in data_struct]			)
         return self._tensorize(__a			)



    def  _lowercase						(self   :   Union[str, Any]       ,      __a   :   dict			):
         return map_nested(self._recursive_tensorize       ,      __a       ,      map_list=__a			)



    def  _lowercase						(self   :   str       ,      __a   :   pa.Table			):
         UpperCAmelCase_	   =       self.numpy_arrow_extractor().extract_row(__a			)
         UpperCAmelCase_	   =       self.python_features_decoder.decode_row(__a			)
         return self.recursive_tensorize(__a			)



    def  _lowercase						(self   :   Tuple       ,      __a   :   pa.Table			):
         UpperCAmelCase_	   =       self.numpy_arrow_extractor().extract_column(__a			)
         UpperCAmelCase_	   =       self.python_features_decoder.decode_column(__a       ,      pa_table.column_names[0]			)
         UpperCAmelCase_	   =       self.recursive_tensorize(__a			)
         UpperCAmelCase_	   =       self._consolidate(__a			)
         return column






    def  _lowercase						(self   :   str       ,      __a   :   pa.Table			):
         UpperCAmelCase_	   =       self.numpy_arrow_extractor().extract_batch(__a			)
         UpperCAmelCase_	   =       self.python_features_decoder.decode_batch(__a			)
         UpperCAmelCase_	   =       self.recursive_tensorize(__a			)
         for column_name in batch:
              UpperCAmelCase_	   =       self._consolidate(batch[column_name]			)
         return batch




