




'''simple docstring'''






import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
  if is_torch_available():
    import torch
  if is_tf_available():
    import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


__A :  str    							=				logging.get_logger(__name__)

__A :  Union[str, Any]    							=				{"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

__A :  int    							=				{
    "vocab_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json",
    },
    "merges_file": {
        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt",
    },
    "tokenizer_file": {
        "Salesforce/codegen-350M-mono": (
            "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json"
        ),
    },
}

__A :  Union[str, Any]    							=				{
    "Salesforce/codegen-350M-mono": 2048,
}






class        __snake_case	(       _SCREAMING_SNAKE_CASE):







    """simple docstring"""



    lowercase       			= VOCAB_FILES_NAMES
    lowercase       			= PRETRAINED_VOCAB_FILES_MAP
    lowercase       			= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowercase       			= ["""input_ids""", """attention_mask"""]
    lowercase       			= CodeGenTokenizer

    def __init__(       self   :  str   ,						lowerCamelCase   :  str=None   ,						lowerCamelCase   :  Optional[Any]=None   ,						lowerCamelCase   :  Optional[Any]=None   ,						lowerCamelCase   :  Optional[int]="<|endoftext|>"   ,						lowerCamelCase   :  Optional[Any]="<|endoftext|>"   ,						lowerCamelCase   :  Optional[Any]="<|endoftext|>"   ,						lowerCamelCase   :  str=False   ,						**lowerCamelCase   :  Tuple   ,						)       ->  Union[str, Any]:
      super().__init__(
          _A   ,						_A   ,						tokenizer_file=_A   ,						unk_token=_A   ,						bos_token=_A   ,						eos_token=_A   ,						add_prefix_space=_A   ,						**_A   ,						)

      if kwargs.pop("""add_bos_token"""   ,						_A		):
        lowerCAmelCase_      :				int      =       kwargs.pop("""name_or_path"""   ,						""""""		)
        raise ValueError(
            """Currenty GPT2\'s fast tokenizer does NOT support adding a BOS token."""
            """Instead you should use GPT2\'s slow tokenizer class `CodeGenTokenizer` as follows: \n"""
            F'`CodeGenTokenizer.from_pretrained(\'{model_id}\')`\nor\n'
            F'`AutoTokenizer.from_pretrained(\'{model_id}\', use_fast=False)`\n'
            """This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005."""
            """ so that the fast tokenizer works correctly."""		)

      lowerCAmelCase_      :				int      =       json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()		)
      if pre_tok_state.get("""add_prefix_space"""   ,						_A		) != add_prefix_space:
        lowerCAmelCase_      :				List[str]      =       getattr(_A   ,						pre_tok_state.pop("""type"""		)		)
        lowerCAmelCase_      :				Tuple      =       add_prefix_space
        lowerCAmelCase_      :				List[Any]      =       pre_tok_class(**_A		)

      lowerCAmelCase_      :				Any      =       add_prefix_space

    def __lowercase       (       self   :  Dict   ,						*lowerCamelCase   :  Any   ,						**lowerCamelCase   :  Optional[Any]		)       ->  BatchEncoding:
      lowerCAmelCase_      :				Any      =       kwargs.get("""is_split_into_words"""   ,						_A		)
      assert self.add_prefix_space or not is_split_into_words, (
          F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
          "to use it with pretokenized inputs."
      )

      return super()._batch_encode_plus(*_A   ,						**_A		)

    def __lowercase       (       self   :  List[str]   ,						*lowerCamelCase   :  Any   ,						**lowerCamelCase   :  Optional[int]		)       ->  BatchEncoding:
      lowerCAmelCase_      :				Optional[Any]      =       kwargs.get("""is_split_into_words"""   ,						_A		)

      assert self.add_prefix_space or not is_split_into_words, (
          F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
          "to use it with pretokenized inputs."
      )

      return super()._encode_plus(*_A   ,						**_A		)

    def __lowercase       (       self   :  int   ,						lowerCamelCase   :  str   ,						lowerCamelCase   :  Optional[str] = None		)       ->  Tuple[str]:
      lowerCAmelCase_      :				str      =       self._tokenizer.model.save(_A   ,						name=_A		)
      return tuple(_A		)

    def __lowercase       (       self   :  List[str]   ,						lowerCamelCase   :  Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"]   ,						lowerCamelCase   :  bool = False   ,						lowerCamelCase   :  bool = None   ,						lowerCamelCase   :  Optional[List[str]] = None   ,						**lowerCamelCase   :  List[Any]   ,						)       ->  str:
      lowerCAmelCase_      :				Tuple      =       super().decode(
          token_ids=_A   ,						skip_special_tokens=_A   ,						clean_up_tokenization_spaces=_A   ,						**_A   ,						)

      if truncate_before_pattern is not None and len(_A		) > 0:
        lowerCAmelCase_      :				Union[str, Any]      =       self.truncate(_A   ,						_A		)

      return decoded_text





    def __lowercase       (       self   :  int   ,						lowerCamelCase   :  Optional[Any]   ,						lowerCamelCase   :  int		)       ->  Optional[Any]:
      def find_re(lowerCamelCase   :  str   ,						lowerCamelCase   :  List[str]   ,						lowerCamelCase   :  str		):
        lowerCAmelCase_      :				Any      =       pattern.search(_A   ,						_A		)
        return m.start() if m else -1

      lowerCAmelCase_      :				Tuple      =       [re.compile(_A   ,						re.MULTILINE		) for pattern in truncate_before_pattern]

      lowerCAmelCase_      :				Any      =       list(re.finditer("""^print"""   ,						_A   ,						re.MULTILINE		)		)

      if len(_A		) > 1:
        lowerCAmelCase_      :				List[str]      =       completion[: prints[1].start()]

      lowerCAmelCase_      :				Dict      =       list(re.finditer("""^def"""   ,						_A   ,						re.MULTILINE		)		)

      if len(_A		) > 1:
        lowerCAmelCase_      :				List[Any]      =       completion[: defs[1].start()]

      lowerCAmelCase_      :				List[str]      =       0

      lowerCAmelCase_      :				str      =       [
          pos for pos in [find_re(_A   ,						_A   ,						_A		) for terminal in terminals] if pos != -1
      ]

      if len(_A		) > 0:
        return completion[: min(_A		)]
      else:
        return completion







'''simple docstring'''






from ..utils import DummyObject, requires_backends






class        __snake_case	(       metaclass=_SCREAMING_SNAKE_CASE):







    """simple docstring"""



    lowercase       			= ['note_seq']

    def __init__(       self   :  Tuple   ,						*lowerCamelCase   :  Optional[int]   ,						**lowerCamelCase   :  Tuple		)       ->  Any:
      requires_backends(self   ,						["""note_seq"""]		)

    @classmethod
    def __lowercase       (       cls   :  Optional[Any]   ,						*lowerCamelCase   :  List[Any]   ,						**lowerCamelCase   :  Dict		)       ->  List[Any]:
      requires_backends(cls   ,						["""note_seq"""]		)





    @classmethod
    def __lowercase       (       cls   :  Union[str, Any]   ,						*lowerCamelCase   :  Union[str, Any]   ,						**lowerCamelCase   :  List[Any]		)       ->  Optional[int]:
      requires_backends(cls   ,						["""note_seq"""]		)



