

import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
   if is_torch_available():
      import torch
   if is_tf_available():
      import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


__UpperCAmelCase								=			logging.get_logger(__name__)

__UpperCAmelCase								=			{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

__UpperCAmelCase								=			{
    """vocab_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """Salesforce/codegen-350M-mono""": (
            """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json"""
        ),
    },
}

__UpperCAmelCase								=			{
    """Salesforce/codegen-350M-mono""": 2_048,
}



class SCREAMING_SNAKE_CASE     (  a_			):
  """simple docstring"""






  lowerCamelCase   :      str										=VOCAB_FILES_NAMES
  lowerCamelCase   :      Union[str, Any]										=PRETRAINED_VOCAB_FILES_MAP
  lowerCamelCase   :      Any										=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  lowerCamelCase   :      Tuple										=['''input_ids''', '''attention_mask''']
  lowerCamelCase   :      int										=CodeGenTokenizer







  def __init__(      self			:   Dict	,							lowerCAmelCase			:   Optional[int]=None	,							lowerCAmelCase			:   List[Any]=None	,							lowerCAmelCase			:   Tuple=None	,							lowerCAmelCase			:   Optional[int]="<|endoftext|>"	,							lowerCAmelCase			:   Any="<|endoftext|>"	,							lowerCAmelCase			:   Optional[Any]="<|endoftext|>"	,							lowerCAmelCase			:   Tuple=False	,							**lowerCAmelCase			:   List[str]	,							)							->     int:






     """simple docstring"""







     super().__init__(
         lowercase_	,							lowercase_	,							tokenizer_file=lowercase_	,							unk_token=lowercase_	,							bos_token=lowercase_	,							eos_token=lowercase_	,							add_prefix_space=lowercase_	,							**lowercase_	,							)

     if kwargs.pop("""add_bos_token"""	,							lowercase_   ):
        __lowerCAmelCase  :      Optional[int]         =						kwargs.pop("""name_or_path"""	,							""""""   )
        raise ValueError(
            """Currenty GPT2\'s fast tokenizer does NOT support adding a BOS token."""
            """Instead you should use GPT2\'s slow tokenizer class `CodeGenTokenizer` as follows: \n"""
            f'''`CodeGenTokenizer.from_pretrained(\'{model_id}\')`\nor\n'''
            f'''`AutoTokenizer.from_pretrained(\'{model_id}\', use_fast=False)`\n'''
            """This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005."""
            """ so that the fast tokenizer works correctly."""   )

     __lowerCAmelCase  :      List[Any]         =						json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()   )
     if pre_tok_state.get("""add_prefix_space"""	,							lowercase_   ) != add_prefix_space:
        __lowerCAmelCase  :      Union[str, Any]         =						getattr(lowercase_	,							pre_tok_state.pop("""type"""   )   )
        __lowerCAmelCase  :      List[Any]         =						add_prefix_space
        __lowerCAmelCase  :      List[Any]         =						pre_tok_class(**lowercase_   )

     __lowerCAmelCase  :      Optional[Any]         =						add_prefix_space







  def 	SCREAMING_SNAKE_CASE   (      self			:   Union[str, Any]	,							*lowerCAmelCase			:   int	,							**lowerCAmelCase			:   Dict   )							->     BatchEncoding:






     """simple docstring"""







     __lowerCAmelCase  :      Dict         =						kwargs.get("""is_split_into_words"""	,							lowercase_   )
     assert self.add_prefix_space or not is_split_into_words, (
         f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
         "to use it with pretokenized inputs."
     )

     return super()._batch_encode_plus(*lowercase_	,							**lowercase_   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   List[Any]	,							*lowerCAmelCase			:   List[str]	,							**lowerCAmelCase			:   Any   )							->     BatchEncoding:






     """simple docstring"""







     __lowerCAmelCase  :      str         =						kwargs.get("""is_split_into_words"""	,							lowercase_   )

     assert self.add_prefix_space or not is_split_into_words, (
         f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
         "to use it with pretokenized inputs."
     )

     return super()._encode_plus(*lowercase_	,							**lowercase_   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Tuple	,							lowerCAmelCase			:   Optional[Any]	,							lowerCAmelCase			:   int = None   )							->     Tuple[str]:






     """simple docstring"""







     __lowerCAmelCase  :      Tuple         =						self._tokenizer.model.save(lowercase_	,							name=lowercase_   )
     return tuple(lowercase_   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   List[Any]	,							lowerCAmelCase			:   Optional[Any]	,							lowerCAmelCase			:   Any = False	,							lowerCAmelCase			:   List[str] = None	,							lowerCAmelCase			:   List[str] = None	,							**lowerCAmelCase			:   List[Any]	,							)							->     str:






     """simple docstring"""







     __lowerCAmelCase  :      Dict         =						super().decode(
         token_ids=lowercase_	,							skip_special_tokens=lowercase_	,							clean_up_tokenization_spaces=lowercase_	,							**lowercase_	,							)

     if truncate_before_pattern is not None and len(lowercase_   ) > 0:
        __lowerCAmelCase  :      Any         =						self.truncate(lowercase_	,							lowercase_   )

     return decoded_text







  def 	SCREAMING_SNAKE_CASE   (      self			:   Any	,							lowerCAmelCase			:   Tuple	,							lowerCAmelCase			:   Any   )							->     Tuple:






     """simple docstring"""







     def find_re(lowerCAmelCase			:   Dict	,							lowerCAmelCase			:   Union[str, Any]	,							lowerCAmelCase			:   Union[str, Any]   ):
        __lowerCAmelCase  :      Optional[int]         =						pattern.search(lowercase_	,							lowercase_   )
        return m.start() if m else -1

     __lowerCAmelCase  :      Any         =						[re.compile(lowercase_	,							re.MULTILINE   ) for pattern in truncate_before_pattern]

     __lowerCAmelCase  :      Optional[Any]         =						list(re.finditer("""^print"""	,							lowercase_	,							re.MULTILINE   )   )

     if len(lowercase_   ) > 1:
        __lowerCAmelCase  :      Dict         =						completion[: prints[1].start()]

     __lowerCAmelCase  :      List[Any]         =						list(re.finditer("""^def"""	,							lowercase_	,							re.MULTILINE   )   )

     if len(lowercase_   ) > 1:
        __lowerCAmelCase  :      int         =						completion[: defs[1].start()]

     __lowerCAmelCase  :      Optional[int]         =						0

     __lowerCAmelCase  :      Optional[Any]         =						[
         pos for pos in [find_re(lowercase_	,							lowercase_	,							lowercase_   ) for terminal in terminals] if pos != -1
     ]

     if len(lowercase_   ) > 0:
        return completion[: min(lowercase_   )]
     else:
        return completion





import importlib.util
import json
import os
import warnings
from dataclasses import dataclass, field

import torch

from ..training_args import TrainingArguments
from ..utils import cached_property, is_sagemaker_dp_enabled, logging


__UpperCAmelCase								=			logging.get_logger(__name__)





def      snake_case_				()      ->     str:
   # Get the sagemaker specific mp parameters from smp_options variable.
   __lowerCAmelCase  :      List[Any]    =						os.getenv("""SM_HP_MP_PARAMETERS""" ,       """{}"""					)
   try:
      # Parse it and check the field "partitions" is included, it is required for model parallel.
      __lowerCAmelCase  :      Any    =						json.loads(__A					)
      if "partitions" not in smp_options:
         return False
   except json.JSONDecodeError:
      return False

   # Get the sagemaker specific framework parameters from mpi_options variable.
   __lowerCAmelCase  :      Any    =						os.getenv("""SM_FRAMEWORK_PARAMS""" ,       """{}"""					)
   try:
      # Parse it and check the field "sagemaker_distributed_dataparallel_enabled".
      __lowerCAmelCase  :      Union[str, Any]    =						json.loads(__A					)
      if not mpi_options.get("""sagemaker_mpi_enabled""" ,       __A					):
         return False
   except json.JSONDecodeError:
      return False
   # Lastly, check if the `smdistributed` module is present.
   return importlib.util.find_spec("""smdistributed"""					) is not None


if is_sagemaker_model_parallel_available():
   import smdistributed.modelparallel.torch as smp

   smp.init()



@dataclass
class SCREAMING_SNAKE_CASE     (  a_			):
  """simple docstring"""






  lowerCamelCase   :      str										=field(
      default=""       ,							metadata={"help": "Used by the SageMaker launcher to send mp-specific args. Ignored in SageMakerTrainer"}       ,							)







  def 	SCREAMING_SNAKE_CASE   (      self			:   List[Any]   )							->     Optional[int]:






     """simple docstring"""







     super().__post_init__()
     warnings.warn(
         """`SageMakerTrainingArguments` is deprecated and will be removed in v5 of Transformers. You can use """
         """`TrainingArguments` instead."""	,							lowerCAmelCase	,							)







  @cached_property
  def 	SCREAMING_SNAKE_CASE   (      self			:   Any   )							->     "torch.device":






     """simple docstring"""







     logger.info("""PyTorch: setting up devices"""   )
     if torch.distributed.is_available() and torch.distributed.is_initialized() and self.local_rank == -1:
        logger.warning(
            """torch.distributed process group is initialized, but local_rank == -1. """
            """In order to use Torch DDP, launch your script with `python -m torch.distributed.launch"""   )
     if self.no_cuda:
        __lowerCAmelCase  :      Optional[Any]    =						torch.device("""cpu"""   )
        __lowerCAmelCase  :      Union[str, Any]    =						0
     elif is_sagemaker_model_parallel_available():
        __lowerCAmelCase  :      int    =						smp.local_rank()
        __lowerCAmelCase  :      str    =						torch.device("""cuda"""	,							lowerCAmelCase   )
        __lowerCAmelCase  :      Tuple    =						1
     elif is_sagemaker_dp_enabled():
        import smdistributed.dataparallel.torch.torch_smddp  # noqa: F401

        torch.distributed.init_process_group(backend="""smddp"""	,							timeout=self.ddp_timeout_delta   )
        __lowerCAmelCase  :      List[str]    =						int(os.getenv("""SMDATAPARALLEL_LOCAL_RANK"""   )   )
        __lowerCAmelCase  :      Tuple    =						torch.device("""cuda"""	,							self.local_rank   )
        __lowerCAmelCase  :      Optional[int]    =						1
     elif self.local_rank == -1:
        # if n_gpu is > 1 we'll use nn.DataParallel.
        # If you only want to use a specific subset of GPUs use `CUDA_VISIBLE_DEVICES=0`
        # Explicitly set CUDA to the first (index 0) CUDA device, otherwise `set_device` will
        # trigger an error that a device index is missing. Index 0 takes into account the
        # GPUs available in the environment, so `CUDA_VISIBLE_DEVICES=1,2` with `cuda:0`
        # will use the first GPU in that env, i.e. GPU#1
        __lowerCAmelCase  :      List[Any]    =						torch.device("""cuda:0""" if torch.cuda.is_available() else """cpu"""   )
        # Sometimes the line in the postinit has not been run before we end up here, so just checking we're not at
        # the default value.
        __lowerCAmelCase  :      Optional[int]    =						torch.cuda.device_count()
     else:
        # Here, we'll use torch.distributed.
        # Initializes the distributed backend which will take care of synchronizing nodes/GPUs
        if not torch.distributed.is_initialized():
           torch.distributed.init_process_group(backend="""nccl"""	,							timeout=self.ddp_timeout_delta   )
        __lowerCAmelCase  :      List[str]    =						torch.device("""cuda"""	,							self.local_rank   )
        __lowerCAmelCase  :      int    =						1

     if device.type == "cuda":
        torch.cuda.set_device(lowerCAmelCase   )

     return device







  @property
  def 	SCREAMING_SNAKE_CASE   (      self			:   int   )							->     Any:






     """simple docstring"""







     if is_sagemaker_model_parallel_available():
        return smp.dp_size()

     return super().world_size







  @property
  def 	SCREAMING_SNAKE_CASE   (      self			:   Optional[int]   )							->     Tuple:






     """simple docstring"""







     return not is_sagemaker_model_parallel_available()







  @property
  def 	SCREAMING_SNAKE_CASE   (      self			:   int   )							->     Tuple:






     """simple docstring"""







     return False


