


'''simple docstring'''






import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__A :  Optional[Any]    							=				logging.get_logger(__name__)

__A :  int    							=				"▁"

__A :  Tuple    							=				{"vocab_file": "sentencepiece.bpe.model", "monolingual_vocab_file": "dict.txt"}

__A :  Union[str, Any]    							=				{
    "vocab_file": {
        "vinai/bartpho-syllable": "https://huggingface.co/vinai/bartpho-syllable/resolve/main/sentencepiece.bpe.model",
    },
    "monolingual_vocab_file": {
        "vinai/bartpho-syllable": "https://huggingface.co/vinai/bartpho-syllable/resolve/main/dict.txt",
    },
}

__A :  Dict    							=				{"vinai/bartpho-syllable": 1024}






class        __snake_case	(       _SCREAMING_SNAKE_CASE):







    """simple docstring"""



    lowercase       			= VOCAB_FILES_NAMES
    lowercase       			= PRETRAINED_VOCAB_FILES_MAP
    lowercase       			= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowercase       			= ['input_ids', 'attention_mask']

    def __init__(       self   :  Dict   ,						lowerCamelCase   :  Optional[Any]   ,						lowerCamelCase   :  Any   ,						lowerCamelCase   :  Dict="<s>"   ,						lowerCamelCase   :  Dict="</s>"   ,						lowerCamelCase   :  Optional[int]="</s>"   ,						lowerCamelCase   :  Optional[int]="<s>"   ,						lowerCamelCase   :  Optional[int]="<unk>"   ,						lowerCamelCase   :  str="<pad>"   ,						lowerCamelCase   :  Any="<mask>"   ,						lowerCamelCase   :  Optional[Dict[str, Any]] = None   ,						**lowerCamelCase   :  Tuple   ,						)       ->  None:
      # Mask token behave like a normal word, i.e. include the space before it
      lowerCAmelCase_      :				List[Any]      =       AddedToken(lowerCamelCase   ,						lstrip=lowerCamelCase   ,						rstrip=lowerCamelCase		) if isinstance(lowerCamelCase   ,						lowerCamelCase		) else mask_token

      lowerCAmelCase_      :				Optional[int]      =       {} if sp_model_kwargs is None else sp_model_kwargs

      super().__init__(
          bos_token=lowerCamelCase   ,						eos_token=lowerCamelCase   ,						unk_token=lowerCamelCase   ,						sep_token=lowerCamelCase   ,						cls_token=lowerCamelCase   ,						pad_token=lowerCamelCase   ,						mask_token=lowerCamelCase   ,						sp_model_kwargs=self.sp_model_kwargs   ,						**lowerCamelCase   ,						)

      lowerCAmelCase_      :				Any      =       vocab_file
      lowerCAmelCase_      :				Union[str, Any]      =       monolingual_vocab_file
      lowerCAmelCase_      :				str      =       spm.SentencePieceProcessor(**self.sp_model_kwargs		)
      self.sp_model.Load(str(lowerCamelCase		)		)

      # Load the reduced vocab

      # Keep order of special tokens for backward compatibility
      lowerCAmelCase_      :				int      =       {}
      lowerCAmelCase_      :				str      =       0
      for token in [bos_token, pad_token, eos_token, unk_token, sep_token, cls_token]:
        if str(lowerCamelCase		) not in self.fairseq_tokens_to_ids:
          lowerCAmelCase_      :				str      =       cnt
          cnt += 1
      with open(lowerCamelCase   ,						"""r"""   ,						encoding="""utf-8"""		) as f:
        for line in f.readlines():
          lowerCAmelCase_      :				int      =       line.strip().split()[0]
          lowerCAmelCase_      :				str      =       len(self.fairseq_tokens_to_ids		)
      if str(lowerCamelCase		) not in self.fairseq_tokens_to_ids:
        lowerCAmelCase_      :				List[str]      =       len(self.fairseq_tokens_to_ids		)

      lowerCAmelCase_      :				Any      =       {v: k for k, v in self.fairseq_tokens_to_ids.items()}

    def __getstate__(       self   :  int		)       ->  List[str]:
      lowerCAmelCase_      :				List[Any]      =       self.__dict__.copy()
      lowerCAmelCase_      :				Union[str, Any]      =       None
      lowerCAmelCase_      :				Any      =       self.sp_model.serialized_model_proto()
      return state

    def __setstate__(       self   :  List[str]   ,						lowerCamelCase   :  Union[str, Any]		)       ->  Any:
      lowerCAmelCase_      :				List[str]      =       d

      # for backward compatibility
      if not hasattr(self   ,						"""sp_model_kwargs"""		):
        lowerCAmelCase_      :				Dict      =       {}

      lowerCAmelCase_      :				Optional[Any]      =       spm.SentencePieceProcessor(**self.sp_model_kwargs		)
      self.sp_model.LoadFromSerializedProto(self.sp_model_proto		)

    def __lowercase       (       self   :  int   ,						lowerCamelCase   :  List[int]   ,						lowerCamelCase   :  Optional[List[int]] = None		)       ->  List[int]:

      if token_ids_a is None:
        return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
      lowerCAmelCase_      :				Optional[Any]      =       [self.cls_token_id]
      lowerCAmelCase_      :				Any      =       [self.sep_token_id]
      return cls + token_ids_a + sep + sep + token_ids_a + sep

    def __lowercase       (       self   :  int   ,						lowerCamelCase   :  List[int]   ,						lowerCamelCase   :  Optional[List[int]] = None   ,						lowerCamelCase   :  bool = False		)       ->  List[int]:

      if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_a=lowerCamelCase   ,						token_ids_a=lowerCamelCase   ,						already_has_special_tokens=lowerCamelCase		)

      if token_ids_a is None:
        return [1] + ([0] * len(lowerCamelCase		)) + [1]
      return [1] + ([0] * len(lowerCamelCase		)) + [1, 1] + ([0] * len(lowerCamelCase		)) + [1]

    def __lowercase       (       self   :  int   ,						lowerCamelCase   :  List[int]   ,						lowerCamelCase   :  Optional[List[int]] = None		)       ->  List[int]:
      lowerCAmelCase_      :				Dict      =       [self.sep_token_id]
      lowerCAmelCase_      :				Dict      =       [self.cls_token_id]

      if token_ids_a is None:
        return len(cls + token_ids_a + sep		) * [0]
      return len(cls + token_ids_a + sep + sep + token_ids_a + sep		) * [0]

    @property
    def __lowercase       (       self   :  Optional[int]		)       ->  Optional[int]:
      return len(self.fairseq_ids_to_tokens		)

    def __lowercase       (       self   :  str		)       ->  str:
      lowerCAmelCase_      :				Optional[int]      =       {self.convert_ids_to_tokens(lowerCamelCase		): i for i in range(self.vocab_size		)}
      vocab.update(self.added_tokens_encoder		)
      return vocab

    def __lowercase       (       self   :  Any   ,						lowerCamelCase   :  str		)       ->  List[str]:
      return self.sp_model.encode(lowerCamelCase   ,						out_type=lowerCamelCase		)

    def __lowercase       (       self   :  Optional[Any]   ,						lowerCamelCase   :  List[Any]		)       ->  Optional[int]:
      if token in self.fairseq_tokens_to_ids:
        return self.fairseq_tokens_to_ids[token]
      else:
        return self.unk_token_id

    def __lowercase       (       self   :  Dict   ,						lowerCamelCase   :  Union[str, Any]		)       ->  Dict:
      return self.fairseq_ids_to_tokens[index]

    def __lowercase       (       self   :  Optional[Any]   ,						lowerCamelCase   :  int		)       ->  Union[str, Any]:
      lowerCAmelCase_      :				Dict      =       """""".join(lowerCamelCase		).replace(lowerCamelCase   ,						""" """		).strip()
      return out_string





    def __lowercase       (       self   :  Any   ,						lowerCamelCase   :  str   ,						lowerCamelCase   :  Optional[str] = None		)       ->  Tuple[str]:
      if not os.path.isdir(lowerCamelCase		):
        logger.error(F'Vocabulary path ({save_directory}) should be a directory'		)
        return
      lowerCAmelCase_      :				Union[str, Any]      =       os.path.join(
          lowerCamelCase   ,						(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]		)
      lowerCAmelCase_      :				str      =       os.path.join(
          lowerCamelCase   ,						(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""monolingual_vocab_file"""]   ,						)

      if os.path.abspath(self.vocab_file		) != os.path.abspath(lowerCamelCase		) and os.path.isfile(self.vocab_file		):
        copyfile(self.vocab_file   ,						lowerCamelCase		)
      elif not os.path.isfile(self.vocab_file		):
        with open(lowerCamelCase   ,						"""wb"""		) as fi:
          lowerCAmelCase_      :				List[Any]      =       self.sp_model.serialized_model_proto()
          fi.write(lowerCamelCase		)

      if os.path.abspath(self.monolingual_vocab_file		) != os.path.abspath(
          lowerCamelCase		) and os.path.isfile(self.monolingual_vocab_file		):
        copyfile(self.monolingual_vocab_file   ,						lowerCamelCase		)
      elif not os.path.isfile(self.monolingual_vocab_file		):
        with open(lowerCamelCase   ,						"""w"""   ,						encoding="""utf-8"""		) as fp:
          for token in self.fairseq_tokens_to_ids:
            if token not in self.all_special_tokens:
              fp.write(F'{str(lowerCamelCase		)} \n'		)

      return out_vocab_file, out_monolingual_vocab_file







'''simple docstring'''






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_lxmert import LxmertTokenizer


__A :  Any    							=				{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

__A :  int    							=				{
    "vocab_file": {
        "unc-nlp/lxmert-base-uncased": "https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/vocab.txt",
    },
    "tokenizer_file": {
        "unc-nlp/lxmert-base-uncased": (
            "https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/tokenizer.json"
        ),
    },
}

__A :  List[Any]    							=				{
    "unc-nlp/lxmert-base-uncased": 512,
}

__A :  Optional[Any]    							=				{
    "unc-nlp/lxmert-base-uncased": {"do_lower_case": True},
}






class        __snake_case	(       _SCREAMING_SNAKE_CASE):







    """simple docstring"""



    lowercase       			= VOCAB_FILES_NAMES
    lowercase       			= PRETRAINED_VOCAB_FILES_MAP
    lowercase       			= PRETRAINED_INIT_CONFIGURATION
    lowercase       			= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowercase       			= LxmertTokenizer

    def __init__(       self   :  Dict   ,						lowerCamelCase   :  Tuple=None   ,						lowerCamelCase   :  List[str]=None   ,						lowerCamelCase   :  Dict=True   ,						lowerCamelCase   :  List[Any]="[UNK]"   ,						lowerCamelCase   :  Tuple="[SEP]"   ,						lowerCamelCase   :  str="[PAD]"   ,						lowerCamelCase   :  Any="[CLS]"   ,						lowerCamelCase   :  str="[MASK]"   ,						lowerCamelCase   :  Dict=True   ,						lowerCamelCase   :  Union[str, Any]=None   ,						**lowerCamelCase   :  Optional[int]   ,						)       ->  Optional[Any]:
      super().__init__(
          lowerCamelCase   ,						tokenizer_file=lowerCamelCase   ,						do_lower_case=lowerCamelCase   ,						unk_token=lowerCamelCase   ,						sep_token=lowerCamelCase   ,						pad_token=lowerCamelCase   ,						cls_token=lowerCamelCase   ,						mask_token=lowerCamelCase   ,						tokenize_chinese_chars=lowerCamelCase   ,						strip_accents=lowerCamelCase   ,						**lowerCamelCase   ,						)

      lowerCAmelCase_      :				Optional[Any]      =       json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
      if (
          normalizer_state.get("""lowercase"""   ,						lowerCamelCase		) != do_lower_case
          or normalizer_state.get("""strip_accents"""   ,						lowerCamelCase		) != strip_accents
          or normalizer_state.get("""handle_chinese_chars"""   ,						lowerCamelCase		) != tokenize_chinese_chars
      ):
        lowerCAmelCase_      :				Tuple      =       getattr(lowerCamelCase   ,						normalizer_state.pop("""type"""		)		)
        lowerCAmelCase_      :				Optional[Any]      =       do_lower_case
        lowerCAmelCase_      :				Dict      =       strip_accents
        lowerCAmelCase_      :				List[Any]      =       tokenize_chinese_chars
        lowerCAmelCase_      :				List[Any]      =       normalizer_class(**lowerCamelCase		)

      lowerCAmelCase_      :				Tuple      =       do_lower_case

    def __lowercase       (       self   :  Tuple   ,						lowerCamelCase   :  str   ,						lowerCamelCase   :  Optional[Any]=None		)       ->  int:
      lowerCAmelCase_      :				Any      =       [self.cls_token_id] + token_ids_a + [self.sep_token_id]

      if token_ids_a:
        output += token_ids_a + [self.sep_token_id]

      return output

    def __lowercase       (       self   :  str   ,						lowerCamelCase   :  List[int]   ,						lowerCamelCase   :  Optional[List[int]] = None		)       ->  List[int]:
      lowerCAmelCase_      :				str      =       [self.sep_token_id]
      lowerCAmelCase_      :				Optional[int]      =       [self.cls_token_id]
      if token_ids_a is None:
        return len(cls + token_ids_a + sep		) * [0]
      return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]





    def __lowercase       (       self   :  Tuple   ,						lowerCamelCase   :  str   ,						lowerCamelCase   :  Optional[str] = None		)       ->  Tuple[str]:
      lowerCAmelCase_      :				int      =       self._tokenizer.model.save(lowerCamelCase   ,						name=lowerCamelCase		)
      return tuple(lowerCamelCase		)



