


'''simple docstring'''






import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__A :  int    							=				logging.get_logger(__name__)

__A :  Optional[int]    							=				{"vocab_file": "spiece.model"}

__A :  Optional[int]    							=				{
    "vocab_file": {
        "bert_for_seq_generation": (
            "https://huggingface.co/google/bert_for_seq_generation_L-24_bbc_encoder/resolve/main/spiece.model"
        ),
    }
}

__A :  int    							=				{"bert_for_seq_generation": 512}






class        __snake_case	(       _SCREAMING_SNAKE_CASE):







    """simple docstring"""



    lowercase       			= VOCAB_FILES_NAMES
    lowercase       			= PRETRAINED_VOCAB_FILES_MAP
    lowercase       			= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowercase       			= []
    lowercase       			= ['input_ids', 'attention_mask']

    def __init__(       self   :  Optional[Any]   ,						lowerCamelCase   :  List[str]   ,						lowerCamelCase   :  Tuple="<s>"   ,						lowerCamelCase   :  List[Any]="</s>"   ,						lowerCamelCase   :  Union[str, Any]="<unk>"   ,						lowerCamelCase   :  Optional[int]="<pad>"   ,						lowerCamelCase   :  List[str]="<::::>"   ,						lowerCamelCase   :  Optional[Dict[str, Any]] = None   ,						**lowerCamelCase   :  List[Any]   ,						)       ->  None:
      lowerCAmelCase_      :				Union[str, Any]      =       {} if sp_model_kwargs is None else sp_model_kwargs

      # Add extra_ids to the special token list
      super().__init__(
          bos_token=lowerCamelCase   ,						eos_token=lowerCamelCase   ,						unk_token=lowerCamelCase   ,						pad_token=lowerCamelCase   ,						sep_token=lowerCamelCase   ,						sp_model_kwargs=self.sp_model_kwargs   ,						**lowerCamelCase   ,						)

      lowerCAmelCase_      :				int      =       vocab_file

      lowerCAmelCase_      :				Any      =       spm.SentencePieceProcessor(**self.sp_model_kwargs		)
      self.sp_model.Load(lowerCamelCase		)

    @property
    def __lowercase       (       self   :  Optional[int]		)       ->  Dict:
      return self.sp_model.get_piece_size()

    def __lowercase       (       self   :  Union[str, Any]		)       ->  int:
      lowerCAmelCase_      :				List[Any]      =       {self.convert_ids_to_tokens(lowerCamelCase		): i for i in range(self.vocab_size		)}
      vocab.update(self.added_tokens_encoder		)
      return vocab

    def __getstate__(       self   :  List[str]		)       ->  int:
      lowerCAmelCase_      :				Tuple      =       self.__dict__.copy()
      lowerCAmelCase_      :				Dict      =       None
      return state

    def __setstate__(       self   :  Optional[int]   ,						lowerCamelCase   :  List[Any]		)       ->  List[Any]:
      lowerCAmelCase_      :				str      =       d

      # for backward compatibility
      if not hasattr(self   ,						"""sp_model_kwargs"""		):
        lowerCAmelCase_      :				Optional[Any]      =       {}

      lowerCAmelCase_      :				Optional[Any]      =       spm.SentencePieceProcessor(**self.sp_model_kwargs		)
      self.sp_model.Load(self.vocab_file		)

    def __lowercase       (       self   :  Union[str, Any]   ,						lowerCamelCase   :  str		)       ->  List[str]:
      return self.sp_model.encode(lowerCamelCase   ,						out_type=lowerCamelCase		)

    def __lowercase       (       self   :  List[str]   ,						lowerCamelCase   :  str		)       ->  int:
      return self.sp_model.piece_to_id(lowerCamelCase		)

    def __lowercase       (       self   :  Tuple   ,						lowerCamelCase   :  int		)       ->  Any:
      lowerCAmelCase_      :				Optional[int]      =       self.sp_model.IdToPiece(lowerCamelCase		)
      return token

    def __lowercase       (       self   :  Tuple   ,						lowerCamelCase   :  List[Any]		)       ->  List[Any]:
      lowerCAmelCase_      :				str      =       []
      lowerCAmelCase_      :				List[str]      =       """"""
      for token in tokens:
        # make sure that special tokens are not decoded using sentencepiece model
        if token in self.all_special_tokens:
          out_string += self.sp_model.decode(lowerCamelCase		) + token
          lowerCAmelCase_      :				Optional[int]      =       []
        else:
          current_sub_tokens.append(lowerCamelCase		)
      out_string += self.sp_model.decode(lowerCamelCase		)
      return out_string.strip()





    def __lowercase       (       self   :  List[str]   ,						lowerCamelCase   :  str   ,						lowerCamelCase   :  Optional[str] = None		)       ->  Tuple[str]:
      if not os.path.isdir(lowerCamelCase		):
        logger.error(F'Vocabulary path ({save_directory}) should be a directory'		)
        return
      lowerCAmelCase_      :				Dict      =       os.path.join(
          lowerCamelCase   ,						(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]		)

      if os.path.abspath(self.vocab_file		) != os.path.abspath(lowerCamelCase		) and os.path.isfile(self.vocab_file		):
        copyfile(self.vocab_file   ,						lowerCamelCase		)
      elif not os.path.isfile(self.vocab_file		):
        with open(lowerCamelCase   ,						"""wb"""		) as fi:
          lowerCAmelCase_      :				Dict      =       self.sp_model.serialized_model_proto()
          fi.write(lowerCamelCase		)

      return (out_vocab_file,)







'''simple docstring'''






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_lxmert import LxmertTokenizer


__A :  Any    							=				{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

__A :  int    							=				{
    "vocab_file": {
        "unc-nlp/lxmert-base-uncased": "https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/vocab.txt",
    },
    "tokenizer_file": {
        "unc-nlp/lxmert-base-uncased": (
            "https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/tokenizer.json"
        ),
    },
}

__A :  List[Any]    							=				{
    "unc-nlp/lxmert-base-uncased": 512,
}

__A :  Optional[Any]    							=				{
    "unc-nlp/lxmert-base-uncased": {"do_lower_case": True},
}






class        __snake_case	(       _SCREAMING_SNAKE_CASE):







    """simple docstring"""



    lowercase       			= VOCAB_FILES_NAMES
    lowercase       			= PRETRAINED_VOCAB_FILES_MAP
    lowercase       			= PRETRAINED_INIT_CONFIGURATION
    lowercase       			= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowercase       			= LxmertTokenizer

    def __init__(       self   :  Dict   ,						lowerCamelCase   :  Tuple=None   ,						lowerCamelCase   :  List[str]=None   ,						lowerCamelCase   :  Dict=True   ,						lowerCamelCase   :  List[Any]="[UNK]"   ,						lowerCamelCase   :  Tuple="[SEP]"   ,						lowerCamelCase   :  str="[PAD]"   ,						lowerCamelCase   :  Any="[CLS]"   ,						lowerCamelCase   :  str="[MASK]"   ,						lowerCamelCase   :  Dict=True   ,						lowerCamelCase   :  Union[str, Any]=None   ,						**lowerCamelCase   :  Optional[int]   ,						)       ->  Optional[Any]:
      super().__init__(
          lowerCamelCase   ,						tokenizer_file=lowerCamelCase   ,						do_lower_case=lowerCamelCase   ,						unk_token=lowerCamelCase   ,						sep_token=lowerCamelCase   ,						pad_token=lowerCamelCase   ,						cls_token=lowerCamelCase   ,						mask_token=lowerCamelCase   ,						tokenize_chinese_chars=lowerCamelCase   ,						strip_accents=lowerCamelCase   ,						**lowerCamelCase   ,						)

      lowerCAmelCase_      :				Optional[Any]      =       json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
      if (
          normalizer_state.get("""lowercase"""   ,						lowerCamelCase		) != do_lower_case
          or normalizer_state.get("""strip_accents"""   ,						lowerCamelCase		) != strip_accents
          or normalizer_state.get("""handle_chinese_chars"""   ,						lowerCamelCase		) != tokenize_chinese_chars
      ):
        lowerCAmelCase_      :				Tuple      =       getattr(lowerCamelCase   ,						normalizer_state.pop("""type"""		)		)
        lowerCAmelCase_      :				Optional[Any]      =       do_lower_case
        lowerCAmelCase_      :				Dict      =       strip_accents
        lowerCAmelCase_      :				List[Any]      =       tokenize_chinese_chars
        lowerCAmelCase_      :				List[Any]      =       normalizer_class(**lowerCamelCase		)

      lowerCAmelCase_      :				Tuple      =       do_lower_case

    def __lowercase       (       self   :  Tuple   ,						lowerCamelCase   :  str   ,						lowerCamelCase   :  Optional[Any]=None		)       ->  int:
      lowerCAmelCase_      :				Any      =       [self.cls_token_id] + token_ids_a + [self.sep_token_id]

      if token_ids_a:
        output += token_ids_a + [self.sep_token_id]

      return output

    def __lowercase       (       self   :  str   ,						lowerCamelCase   :  List[int]   ,						lowerCamelCase   :  Optional[List[int]] = None		)       ->  List[int]:
      lowerCAmelCase_      :				str      =       [self.sep_token_id]
      lowerCAmelCase_      :				Optional[int]      =       [self.cls_token_id]
      if token_ids_a is None:
        return len(cls + token_ids_a + sep		) * [0]
      return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]





    def __lowercase       (       self   :  Tuple   ,						lowerCamelCase   :  str   ,						lowerCamelCase   :  Optional[str] = None		)       ->  Tuple[str]:
      lowerCAmelCase_      :				int      =       self._tokenizer.model.save(lowerCamelCase   ,						name=lowerCamelCase		)
      return tuple(lowerCamelCase		)



