




'''simple docstring'''





import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
   import torch


if TYPE_CHECKING:
   from transformers.pipelines.conversational import Conversation


UpperCamelCase_						:      Dict       =						logging.get_logger(__name__)
UpperCamelCase_						:      str       =						{'''vocab_file''': '''spiece.model'''}

UpperCamelCase_						:      Tuple       =						{
    '''vocab_file''': {
        '''AI-Sweden/gpt-sw3-126m''': '''https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-350m''': '''https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-1.6b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-6.7b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-20b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model''',
    }
}

UpperCamelCase_						:      str       =						{
    '''AI-Sweden/gpt-sw3-126m''': 2048,
    '''AI-Sweden/gpt-sw3-350m''': 2048,
    '''AI-Sweden/gpt-sw3-1.6b''': 2048,
    '''AI-Sweden/gpt-sw3-6.7b''': 2048,
    '''AI-Sweden/gpt-sw3-20b''': 2048,
}

class        _a    (						a__							):
     SCREAMING_SNAKE_CASE_							:   Union[str, Any]       =						VOCAB_FILES_NAMES
     SCREAMING_SNAKE_CASE_							:   Any       =						PRETRAINED_VOCAB_FILES_MAP
     SCREAMING_SNAKE_CASE_							:   int       =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     SCREAMING_SNAKE_CASE_							:   Optional[int]       =						["input_ids", "attention_mask"]






     def __init__(			self ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE=False ,_SCREAMING_SNAKE_CASE=False ,_SCREAMING_SNAKE_CASE=False ,_SCREAMING_SNAKE_CASE=None ,_SCREAMING_SNAKE_CASE=None ,_SCREAMING_SNAKE_CASE=None ,_SCREAMING_SNAKE_CASE=None ,_SCREAMING_SNAKE_CASE = None ,**_SCREAMING_SNAKE_CASE ,)							->      None:
          _snake_case							=				{} if sp_model_kwargs is None else sp_model_kwargs

          _snake_case							=				kwargs.get("name_or_path"   )
          if name_or_path is None:
               logger.warning(
                   "name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,"
                   " you are testing the model, this can safely be ignored"   )
               _snake_case							=				'None'

          # Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
          _snake_case							=				'<|endoftext|>' if eos_token is None else eos_token
          _snake_case							=				'<unk>' if unk_token is None else unk_token
          if "gpt-sw3-7b" in name_or_path:
               _snake_case							=				unk_token if pad_token is None else pad_token
               _snake_case							=				eos_token if bos_token is None else bos_token
          else:
               _snake_case							=				'<pad>' if pad_token is None else pad_token
               _snake_case							=				'<s>' if bos_token is None else bos_token

          super().__init__(
              do_lower_case=SCREAMING_SNAKE_CASE_ ,remove_space=SCREAMING_SNAKE_CASE_ ,keep_accents=SCREAMING_SNAKE_CASE_ ,bos_token=SCREAMING_SNAKE_CASE_ ,eos_token=SCREAMING_SNAKE_CASE_ ,unk_token=SCREAMING_SNAKE_CASE_ ,pad_token=SCREAMING_SNAKE_CASE_ ,sp_model_kwargs=self.sp_model_kwargs ,**SCREAMING_SNAKE_CASE_ ,)

          _snake_case							=				do_lower_case
          _snake_case							=				remove_space
          _snake_case							=				keep_accents
          _snake_case							=				vocab_file

          _snake_case							=				spm.SentencePieceProcessor(**self.sp_model_kwargs   )
          self.sp_model.Load(SCREAMING_SNAKE_CASE_   )

          # Used for whitespace normalization in input texts
          # fmt : off
          _snake_case							=				{' ', ' ', ' ', ' ', ' ', '　', ' ', ' ', ' ', ' ', '￼', ''}
          # fmt : on

          # Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
          _snake_case							=				re.compile(
              f"""[{''.join(map(SCREAMING_SNAKE_CASE_ ,list(range(0 ,9   )   ) + list(range(11 ,32   )   ) + list(range(127 ,160   )   ) + [160, 173, 8_203]   )   )}]"""   )






     def __getstate__(			self   )							->      Tuple:
          _snake_case							=				self.__dict__.copy()
          _snake_case							=				None
          return state






     def __setstate__(			self ,_SCREAMING_SNAKE_CASE   )							->      Any:
          _snake_case							=				d

          # for backward compatibility
          if not hasattr(self ,"sp_model_kwargs"   ):
               _snake_case							=				{}

          _snake_case							=				spm.SentencePieceProcessor(**self.sp_model_kwargs   )
          self.sp_model.Load(self.vocab_file   )






     @property
     # Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
     def     _lowercase     (			self   )							->      int:
          return len(self.sp_model   )






     def     _lowercase     (			self ,_SCREAMING_SNAKE_CASE   )							->      str:
          _snake_case							=				self.non_printing_characters_re.sub("" ,SCREAMING_SNAKE_CASE_   )

          # Normalize whitespaces
          _snake_case							=				''.join([char if char not in self.whitespaces else " " for char in text]   )

          # NFC Unicode normalization
          _snake_case							=				unicodedata.normalize("NFC" ,SCREAMING_SNAKE_CASE_   )
          return text






     def     _lowercase     (			self ,_SCREAMING_SNAKE_CASE ,**_SCREAMING_SNAKE_CASE   )							->      List[str]:
          _snake_case							=				self.preprocess_text(SCREAMING_SNAKE_CASE_   )
          return self.sp_model.encode(SCREAMING_SNAKE_CASE_ ,out_type=SCREAMING_SNAKE_CASE_   )






     def     _lowercase     (			self ,_SCREAMING_SNAKE_CASE   )							->      int:
          return self.sp_model.PieceToId(SCREAMING_SNAKE_CASE_   )






     def     _lowercase     (			self ,_SCREAMING_SNAKE_CASE   )							->      str:
          return self.sp_model.IdToPiece(SCREAMING_SNAKE_CASE_   )






     @staticmethod
     def     _lowercase     (			_SCREAMING_SNAKE_CASE   )							->      str:
          return out_string






     def     _lowercase     (			self ,_SCREAMING_SNAKE_CASE   )							->      str:
          _snake_case							=				[]
          _snake_case							=				''
          _snake_case							=				False
          for token in tokens:
               # make sure that special tokens are not decoded using sentencepiece model
               if token in self.all_special_tokens:
                    # TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
                    if not prev_is_special:
                         out_string += " "

                    out_string += self.sp_model.decode(SCREAMING_SNAKE_CASE_   ) + token
                    _snake_case							=				True
                    _snake_case							=				[]
               else:
                    current_sub_tokens.append(SCREAMING_SNAKE_CASE_   )
                    _snake_case							=				False
          out_string += self.sp_model.decode(SCREAMING_SNAKE_CASE_   )

          return out_string






     def     _lowercase     (			self   )							->      Dict[str, int]:
          _snake_case							=				{self.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_   ): i for i in range(self.vocab_size   )}
          vocab.update(self.added_tokens_encoder   )
          return vocab






     def     _lowercase     (			self ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE = None   )							->      Tuple[str]:
          if not os.path.isdir(SCREAMING_SNAKE_CASE_   ):
               logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""   )
               return
          _snake_case							=				os.path.join(
              SCREAMING_SNAKE_CASE_ ,(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]   )

          if os.path.abspath(self.vocab_file   ) != os.path.abspath(SCREAMING_SNAKE_CASE_   ) and os.path.isfile(self.vocab_file   ):
               copyfile(self.vocab_file ,SCREAMING_SNAKE_CASE_   )
          elif not os.path.isfile(self.vocab_file   ):
               with open(SCREAMING_SNAKE_CASE_ ,"wb"   ) as fi:
                    _snake_case							=				self.sp_model.serialized_model_proto()
                    fi.write(SCREAMING_SNAKE_CASE_   )

          return (out_vocab_file,)






     def     _lowercase     (			self ,_SCREAMING_SNAKE_CASE ,_SCREAMING_SNAKE_CASE = False   )							->      Union[List[int], List[List[int]], "torch.Tensor"]:

          if isinstance(SCREAMING_SNAKE_CASE_ ,SCREAMING_SNAKE_CASE_   ):
               _snake_case							=				self.preprocess_text(SCREAMING_SNAKE_CASE_   )
               _snake_case							=				self.sp_model.encode(SCREAMING_SNAKE_CASE_   )
          else:
               _snake_case							=				[self.preprocess_text(SCREAMING_SNAKE_CASE_   ) for t in text]
               _snake_case							=				self.sp_model.encode(SCREAMING_SNAKE_CASE_   )

          if return_tensors is True or return_tensors == "pt":
               _snake_case							=				torch.tensor(SCREAMING_SNAKE_CASE_   )

          return token_ids






     def     _lowercase     (			self ,_SCREAMING_SNAKE_CASE   )							->      str:
          return self.sp_model.decode(SCREAMING_SNAKE_CASE_   )






     def     _lowercase     (			self ,_SCREAMING_SNAKE_CASE   )							->      List[int]:
          _snake_case							=				[f"""User: {text}""" if is_user else f"""Bot: {text}""" for is_user, text in conversation.iter_texts()]
          _snake_case							=				(
              f"""{self.eos_token}{self.bos_token}""" + f"""{self.bos_token}""".join(SCREAMING_SNAKE_CASE_   ) + f"""{self.bos_token}Bot:"""
          )
          return self.encode(text=SCREAMING_SNAKE_CASE_   )






'''simple docstring'''





from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tokenizers_available,
    is_torch_available,
)


UpperCamelCase_						:      Union[str, Any]       =						{'''configuration_plbart''': ['''PLBART_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''PLBartConfig''']}

try:
   if not is_sentencepiece_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCamelCase_						:      Optional[int]       =						['''PLBartTokenizer''']

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCamelCase_						:      int       =						[
       '''PLBART_PRETRAINED_MODEL_ARCHIVE_LIST''',
       '''PLBartForCausalLM''',
       '''PLBartForConditionalGeneration''',
       '''PLBartForSequenceClassification''',
       '''PLBartModel''',
       '''PLBartPreTrainedModel''',
   ]


if TYPE_CHECKING:
   from .configuration_plbart import PLBART_PRETRAINED_CONFIG_ARCHIVE_MAP, PLBartConfig

   try:
      if not is_sentencepiece_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .tokenization_plbart import PLBartTokenizer

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_plbart import (
          PLBART_PRETRAINED_MODEL_ARCHIVE_LIST,
          PLBartForCausalLM,
          PLBartForConditionalGeneration,
          PLBartForSequenceClassification,
          PLBartModel,
          PLBartPreTrainedModel,
      )


else:
   import sys

   UpperCamelCase_						:      str       =						_LazyModule(__name__, globals()['''__file__'''], _import_structure)


