

import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__UpperCAmelCase								=			logging.get_logger(__name__)

__UpperCAmelCase								=			"""▁"""

__UpperCAmelCase								=			{"""vocab_file""": """sentencepiece.bpe.model""", """monolingual_vocab_file""": """dict.txt"""}

__UpperCAmelCase								=			{
    """vocab_file""": {
        """vinai/bartpho-syllable""": """https://huggingface.co/vinai/bartpho-syllable/resolve/main/sentencepiece.bpe.model""",
    },
    """monolingual_vocab_file""": {
        """vinai/bartpho-syllable""": """https://huggingface.co/vinai/bartpho-syllable/resolve/main/dict.txt""",
    },
}

__UpperCAmelCase								=			{"""vinai/bartpho-syllable""": 1_024}



class SCREAMING_SNAKE_CASE     (  a_			):
  """simple docstring"""






  lowerCamelCase   :      int										=VOCAB_FILES_NAMES
  lowerCamelCase   :      Any										=PRETRAINED_VOCAB_FILES_MAP
  lowerCamelCase   :      Tuple										=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  lowerCamelCase   :      Any										=["input_ids", "attention_mask"]







  def __init__(      self			:   Tuple	,							lowerCAmelCase			:   Union[str, Any]	,							lowerCAmelCase			:   int	,							lowerCAmelCase			:   List[Any]="<s>"	,							lowerCAmelCase			:   List[Any]="</s>"	,							lowerCAmelCase			:   Any="</s>"	,							lowerCAmelCase			:   Optional[Any]="<s>"	,							lowerCAmelCase			:   List[str]="<unk>"	,							lowerCAmelCase			:   Optional[int]="<pad>"	,							lowerCAmelCase			:   Dict="<mask>"	,							lowerCAmelCase			:   Optional[Dict[str, Any]] = None	,							**lowerCAmelCase			:   str	,							)							->     None:






     """simple docstring"""







     __lowerCAmelCase  :      Any         =						AddedToken(lowerCAmelCase	,							lstrip=lowerCAmelCase	,							rstrip=lowerCAmelCase   ) if isinstance(lowerCAmelCase	,							lowerCAmelCase   ) else mask_token

     __lowerCAmelCase  :      List[Any]         =						{} if sp_model_kwargs is None else sp_model_kwargs

     super().__init__(
         bos_token=lowerCAmelCase	,							eos_token=lowerCAmelCase	,							unk_token=lowerCAmelCase	,							sep_token=lowerCAmelCase	,							cls_token=lowerCAmelCase	,							pad_token=lowerCAmelCase	,							mask_token=lowerCAmelCase	,							sp_model_kwargs=self.sp_model_kwargs	,							**lowerCAmelCase	,							)

     __lowerCAmelCase  :      str         =						vocab_file
     __lowerCAmelCase  :      int         =						monolingual_vocab_file
     __lowerCAmelCase  :      Any         =						spm.SentencePieceProcessor(**self.sp_model_kwargs   )
     self.sp_model.Load(str(lowerCAmelCase   )   )

     # Load the reduced vocab

     # Keep order of special tokens for backward compatibility
     __lowerCAmelCase  :      Dict         =						{}
     __lowerCAmelCase  :      Tuple         =						0
     for token in [bos_token, pad_token, eos_token, unk_token, sep_token, cls_token]:
        if str(lowerCAmelCase   ) not in self.fairseq_tokens_to_ids:
           __lowerCAmelCase  :      Dict         =						cnt
           cnt += 1
     with open(lowerCAmelCase	,							"""r"""	,							encoding="""utf-8"""   ) as f:
        for line in f.readlines():
           __lowerCAmelCase  :      Union[str, Any]         =						line.strip().split()[0]
           __lowerCAmelCase  :      Union[str, Any]         =						len(self.fairseq_tokens_to_ids   )
     if str(lowerCAmelCase   ) not in self.fairseq_tokens_to_ids:
        __lowerCAmelCase  :      str         =						len(self.fairseq_tokens_to_ids   )

     __lowerCAmelCase  :      int         =						{v: k for k, v in self.fairseq_tokens_to_ids.items()}







  def __getstate__(      self			:   List[Any]   )							->     List[str]:






     """simple docstring"""







     __lowerCAmelCase  :      Any         =						self.__dict__.copy()
     __lowerCAmelCase  :      Optional[Any]         =						None
     __lowerCAmelCase  :      Optional[int]         =						self.sp_model.serialized_model_proto()
     return state







  def __setstate__(      self			:   Optional[int]	,							lowerCAmelCase			:   Optional[Any]   )							->     Optional[Any]:






     """simple docstring"""







     __lowerCAmelCase  :      List[str]         =						d

     # for backward compatibility
     if not hasattr(self	,							"""sp_model_kwargs"""   ):
        __lowerCAmelCase  :      int         =						{}

     __lowerCAmelCase  :      str         =						spm.SentencePieceProcessor(**self.sp_model_kwargs   )
     self.sp_model.LoadFromSerializedProto(self.sp_model_proto   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Tuple	,							lowerCAmelCase			:   List[int]	,							lowerCAmelCase			:   Optional[List[int]] = None   )							->     List[int]:






     """simple docstring"""







     if token_ids_a is None:
        return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
     __lowerCAmelCase  :      Optional[Any]         =						[self.cls_token_id]
     __lowerCAmelCase  :      Union[str, Any]         =						[self.sep_token_id]
     return cls + token_ids_a + sep + sep + token_ids_a + sep







  def 	SCREAMING_SNAKE_CASE   (      self			:   str	,							lowerCAmelCase			:   List[int]	,							lowerCAmelCase			:   Optional[List[int]] = None	,							lowerCAmelCase			:   bool = False   )							->     List[int]:






     """simple docstring"""







     if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_a=lowerCAmelCase	,							token_ids_a=lowerCAmelCase	,							already_has_special_tokens=lowerCAmelCase   )

     if token_ids_a is None:
        return [1] + ([0] * len(lowerCAmelCase   )) + [1]
     return [1] + ([0] * len(lowerCAmelCase   )) + [1, 1] + ([0] * len(lowerCAmelCase   )) + [1]







  def 	SCREAMING_SNAKE_CASE   (      self			:   int	,							lowerCAmelCase			:   List[int]	,							lowerCAmelCase			:   Optional[List[int]] = None   )							->     List[int]:






     """simple docstring"""







     __lowerCAmelCase  :      Dict         =						[self.sep_token_id]
     __lowerCAmelCase  :      Dict         =						[self.cls_token_id]

     if token_ids_a is None:
        return len(cls + token_ids_a + sep   ) * [0]
     return len(cls + token_ids_a + sep + sep + token_ids_a + sep   ) * [0]







  @property
  def 	SCREAMING_SNAKE_CASE   (      self			:   Optional[int]   )							->     Optional[int]:






     """simple docstring"""







     return len(self.fairseq_ids_to_tokens   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   List[Any]   )							->     Optional[int]:






     """simple docstring"""







     __lowerCAmelCase  :      List[str]         =						{self.convert_ids_to_tokens(lowerCAmelCase   ): i for i in range(self.vocab_size   )}
     vocab.update(self.added_tokens_encoder   )
     return vocab







  def 	SCREAMING_SNAKE_CASE   (      self			:   List[str]	,							lowerCAmelCase			:   str   )							->     List[str]:






     """simple docstring"""







     return self.sp_model.encode(lowerCAmelCase	,							out_type=lowerCAmelCase   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Any	,							lowerCAmelCase			:   Any   )							->     Dict:






     """simple docstring"""







     if token in self.fairseq_tokens_to_ids:
        return self.fairseq_tokens_to_ids[token]
     else:
        return self.unk_token_id







  def 	SCREAMING_SNAKE_CASE   (      self			:   Optional[Any]	,							lowerCAmelCase			:   int   )							->     Optional[int]:






     """simple docstring"""







     return self.fairseq_ids_to_tokens[index]







  def 	SCREAMING_SNAKE_CASE   (      self			:   Union[str, Any]	,							lowerCAmelCase			:   Union[str, Any]   )							->     List[Any]:






     """simple docstring"""







     __lowerCAmelCase  :      List[Any]         =						"""""".join(lowerCAmelCase   ).replace(lowerCAmelCase	,							""" """   ).strip()
     return out_string







  def 	SCREAMING_SNAKE_CASE   (      self			:   Dict	,							lowerCAmelCase			:   str	,							lowerCAmelCase			:   Optional[str] = None   )							->     Tuple[str]:






     """simple docstring"""







     if not os.path.isdir(lowerCAmelCase   ):
        logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''   )
        return
     __lowerCAmelCase  :      Union[str, Any]         =						os.path.join(
         lowerCAmelCase	,							(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]   )
     __lowerCAmelCase  :      List[str]         =						os.path.join(
         lowerCAmelCase	,							(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""monolingual_vocab_file"""]	,							)

     if os.path.abspath(self.vocab_file   ) != os.path.abspath(lowerCAmelCase   ) and os.path.isfile(self.vocab_file   ):
        copyfile(self.vocab_file	,							lowerCAmelCase   )
     elif not os.path.isfile(self.vocab_file   ):
        with open(lowerCAmelCase	,							"""wb"""   ) as fi:
           __lowerCAmelCase  :      str         =						self.sp_model.serialized_model_proto()
           fi.write(lowerCAmelCase   )

     if os.path.abspath(self.monolingual_vocab_file   ) != os.path.abspath(
         lowerCAmelCase   ) and os.path.isfile(self.monolingual_vocab_file   ):
        copyfile(self.monolingual_vocab_file	,							lowerCAmelCase   )
     elif not os.path.isfile(self.monolingual_vocab_file   ):
        with open(lowerCAmelCase	,							"""w"""	,							encoding="""utf-8"""   ) as fp:
           for token in self.fairseq_tokens_to_ids:
              if token not in self.all_special_tokens:
                 fp.write(f'''{str(lowerCAmelCase   )} \n'''   )

     return out_vocab_file, out_monolingual_vocab_file





import importlib.util
import json
import os
import warnings
from dataclasses import dataclass, field

import torch

from ..training_args import TrainingArguments
from ..utils import cached_property, is_sagemaker_dp_enabled, logging


__UpperCAmelCase								=			logging.get_logger(__name__)





def      snake_case_				()      ->     str:
   # Get the sagemaker specific mp parameters from smp_options variable.
   __lowerCAmelCase  :      List[Any]    =						os.getenv("""SM_HP_MP_PARAMETERS""" ,       """{}"""					)
   try:
      # Parse it and check the field "partitions" is included, it is required for model parallel.
      __lowerCAmelCase  :      Any    =						json.loads(__A					)
      if "partitions" not in smp_options:
         return False
   except json.JSONDecodeError:
      return False

   # Get the sagemaker specific framework parameters from mpi_options variable.
   __lowerCAmelCase  :      Any    =						os.getenv("""SM_FRAMEWORK_PARAMS""" ,       """{}"""					)
   try:
      # Parse it and check the field "sagemaker_distributed_dataparallel_enabled".
      __lowerCAmelCase  :      Union[str, Any]    =						json.loads(__A					)
      if not mpi_options.get("""sagemaker_mpi_enabled""" ,       __A					):
         return False
   except json.JSONDecodeError:
      return False
   # Lastly, check if the `smdistributed` module is present.
   return importlib.util.find_spec("""smdistributed"""					) is not None


if is_sagemaker_model_parallel_available():
   import smdistributed.modelparallel.torch as smp

   smp.init()



@dataclass
class SCREAMING_SNAKE_CASE     (  a_			):
  """simple docstring"""






  lowerCamelCase   :      str										=field(
      default=""       ,							metadata={"help": "Used by the SageMaker launcher to send mp-specific args. Ignored in SageMakerTrainer"}       ,							)







  def 	SCREAMING_SNAKE_CASE   (      self			:   List[Any]   )							->     Optional[int]:






     """simple docstring"""







     super().__post_init__()
     warnings.warn(
         """`SageMakerTrainingArguments` is deprecated and will be removed in v5 of Transformers. You can use """
         """`TrainingArguments` instead."""	,							lowerCAmelCase	,							)







  @cached_property
  def 	SCREAMING_SNAKE_CASE   (      self			:   Any   )							->     "torch.device":






     """simple docstring"""







     logger.info("""PyTorch: setting up devices"""   )
     if torch.distributed.is_available() and torch.distributed.is_initialized() and self.local_rank == -1:
        logger.warning(
            """torch.distributed process group is initialized, but local_rank == -1. """
            """In order to use Torch DDP, launch your script with `python -m torch.distributed.launch"""   )
     if self.no_cuda:
        __lowerCAmelCase  :      Optional[Any]    =						torch.device("""cpu"""   )
        __lowerCAmelCase  :      Union[str, Any]    =						0
     elif is_sagemaker_model_parallel_available():
        __lowerCAmelCase  :      int    =						smp.local_rank()
        __lowerCAmelCase  :      str    =						torch.device("""cuda"""	,							lowerCAmelCase   )
        __lowerCAmelCase  :      Tuple    =						1
     elif is_sagemaker_dp_enabled():
        import smdistributed.dataparallel.torch.torch_smddp  # noqa: F401

        torch.distributed.init_process_group(backend="""smddp"""	,							timeout=self.ddp_timeout_delta   )
        __lowerCAmelCase  :      List[str]    =						int(os.getenv("""SMDATAPARALLEL_LOCAL_RANK"""   )   )
        __lowerCAmelCase  :      Tuple    =						torch.device("""cuda"""	,							self.local_rank   )
        __lowerCAmelCase  :      Optional[int]    =						1
     elif self.local_rank == -1:
        # if n_gpu is > 1 we'll use nn.DataParallel.
        # If you only want to use a specific subset of GPUs use `CUDA_VISIBLE_DEVICES=0`
        # Explicitly set CUDA to the first (index 0) CUDA device, otherwise `set_device` will
        # trigger an error that a device index is missing. Index 0 takes into account the
        # GPUs available in the environment, so `CUDA_VISIBLE_DEVICES=1,2` with `cuda:0`
        # will use the first GPU in that env, i.e. GPU#1
        __lowerCAmelCase  :      List[Any]    =						torch.device("""cuda:0""" if torch.cuda.is_available() else """cpu"""   )
        # Sometimes the line in the postinit has not been run before we end up here, so just checking we're not at
        # the default value.
        __lowerCAmelCase  :      Optional[int]    =						torch.cuda.device_count()
     else:
        # Here, we'll use torch.distributed.
        # Initializes the distributed backend which will take care of synchronizing nodes/GPUs
        if not torch.distributed.is_initialized():
           torch.distributed.init_process_group(backend="""nccl"""	,							timeout=self.ddp_timeout_delta   )
        __lowerCAmelCase  :      List[str]    =						torch.device("""cuda"""	,							self.local_rank   )
        __lowerCAmelCase  :      int    =						1

     if device.type == "cuda":
        torch.cuda.set_device(lowerCAmelCase   )

     return device







  @property
  def 	SCREAMING_SNAKE_CASE   (      self			:   int   )							->     Any:






     """simple docstring"""







     if is_sagemaker_model_parallel_available():
        return smp.dp_size()

     return super().world_size







  @property
  def 	SCREAMING_SNAKE_CASE   (      self			:   Optional[int]   )							->     Tuple:






     """simple docstring"""







     return not is_sagemaker_model_parallel_available()







  @property
  def 	SCREAMING_SNAKE_CASE   (      self			:   int   )							->     Tuple:






     """simple docstring"""







     return False


