

'''simple docstring'''







import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


if TYPE_CHECKING:
      from transformers.pipelines.conversational import Conversation


__lowerCamelCase						    =      logging.get_logger(__name__)

__lowerCamelCase						    =      {'''tokenizer_file''': '''tokenizer.json'''}

__lowerCamelCase						    =      {
    '''tokenizer_file''': {
        '''bigscience/tokenizer''': '''https://huggingface.co/bigscience/tokenizer/blob/main/tokenizer.json''',
        '''bigscience/bloom-560m''': '''https://huggingface.co/bigscience/bloom-560m/blob/main/tokenizer.json''',
        '''bigscience/bloom-1b1''': '''https://huggingface.co/bigscience/bloom-1b1/blob/main/tokenizer.json''',
        '''bigscience/bloom-1b7''': '''https://huggingface.co/bigscience/bloom-1b7/blob/main/tokenizer.json''',
        '''bigscience/bloom-3b''': '''https://huggingface.co/bigscience/bloom-3b/blob/main/tokenizer.json''',
        '''bigscience/bloom-7b1''': '''https://huggingface.co/bigscience/bloom-7b1/blob/main/tokenizer.json''',
        '''bigscience/bloom''': '''https://huggingface.co/bigscience/bloom/blob/main/tokenizer.json''',
    },
}


class    A__ ( _snake_case     ):
    lowercase		   =						VOCAB_FILES_NAMES
    lowercase		   =						PRETRAINED_VOCAB_FILES_MAP
    lowercase		   =						["input_ids", "attention_mask"]
    lowercase		   =						None



    def __init__(   self						,			UpperCamelCase__=None						,			UpperCamelCase__=None						,			UpperCamelCase__=None						,			UpperCamelCase__="<unk>"						,			UpperCamelCase__="<s>"						,			UpperCamelCase__="</s>"						,			UpperCamelCase__="<pad>"						,			UpperCamelCase__=False						,			UpperCamelCase__=False						,			**UpperCamelCase__						,			)  -> Optional[Any]:





           '''simple docstring'''
           super().__init__(
               UpperCamelCase__						,			UpperCamelCase__						,			tokenizer_file=UpperCamelCase__						,			unk_token=UpperCamelCase__						,			bos_token=UpperCamelCase__						,			eos_token=UpperCamelCase__						,			pad_token=UpperCamelCase__						,			add_prefix_space=UpperCamelCase__						,			clean_up_tokenization_spaces=UpperCamelCase__						,			**UpperCamelCase__						,			)
           A_		   = json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()    )
           if pre_tok_state.get("""add_prefix_space"""						,			UpperCamelCase__    ) != add_prefix_space:
                  A_		   = getattr(UpperCamelCase__						,			pre_tok_state.pop("""type"""    )    )
                  A_		   = add_prefix_space
                  A_		   = pre_tok_class(**UpperCamelCase__    )

           A_		   = add_prefix_space



    def    snake_case_     (   self						,			*UpperCamelCase__						,			**UpperCamelCase__    )  -> BatchEncoding:





           '''simple docstring'''
           A_		   = kwargs.get("""is_split_into_words"""						,			UpperCamelCase__    )
           if not (self.add_prefix_space or not is_split_into_words):
                  raise Exception(
                      f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with'''
                      """ pretokenized inputs."""    )

           return super()._batch_encode_plus(*UpperCamelCase__						,			**UpperCamelCase__    )



    def    snake_case_     (   self						,			*UpperCamelCase__						,			**UpperCamelCase__    )  -> BatchEncoding:





           '''simple docstring'''
           A_		   = kwargs.get("""is_split_into_words"""						,			UpperCamelCase__    )

           if not (self.add_prefix_space or not is_split_into_words):
                  raise Exception(
                      f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with'''
                      """ pretokenized inputs."""    )

           return super()._encode_plus(*UpperCamelCase__						,			**UpperCamelCase__    )



    def    snake_case_     (   self						,			UpperCamelCase__						,			UpperCamelCase__ = None    )  -> Tuple[str]:





           '''simple docstring'''
           A_		   = self._tokenizer.model.save(UpperCamelCase__						,			name=UpperCamelCase__    )
           return tuple(UpperCamelCase__    )







    def    snake_case_     (   self						,			UpperCamelCase__    )  -> List[int]:





           '''simple docstring'''
           A_		   = []
           for is_user, text in conversation.iter_texts():
                  input_ids.extend(self.encode(UpperCamelCase__						,			add_special_tokens=UpperCamelCase__    ) + [self.eos_token_id]    )

           if len(UpperCamelCase__    ) > self.model_max_length:
                  A_		   = input_ids[-self.model_max_length :]
           return input_ids



'''simple docstring'''







import multiprocessing
from typing import TYPE_CHECKING, Optional, Union

from .. import Dataset, Features, config
from ..formatting import query_table
from ..packaged_modules.sql.sql import Sql
from ..utils import logging
from .abc import AbstractDatasetInputStream


if TYPE_CHECKING:
      import sqlitea

      import sqlalchemy


class    A__ ( _snake_case     ):



    def __init__(   self						,			UpperCamelCase__						,			UpperCamelCase__						,			UpperCamelCase__ = None						,			UpperCamelCase__ = None						,			UpperCamelCase__ = False						,			**UpperCamelCase__						,			)  -> Optional[Any]:





           '''simple docstring'''
           super().__init__(features=UpperCamelCase__						,			cache_dir=UpperCamelCase__						,			keep_in_memory=UpperCamelCase__						,			**UpperCamelCase__    )
           A_		   = Sql(
               cache_dir=UpperCamelCase__						,			features=UpperCamelCase__						,			sql=UpperCamelCase__						,			con=UpperCamelCase__						,			**UpperCamelCase__						,			)







    def    snake_case_     (   self    )  -> Optional[Any]:





           '''simple docstring'''
           A_		   = None
           A_		   = None
           A_		   = None
           A_		   = None

           self.builder.download_and_prepare(
               download_config=UpperCamelCase__						,			download_mode=UpperCamelCase__						,			verification_mode=UpperCamelCase__						,			base_path=UpperCamelCase__						,			)

           # Build dataset for splits
           A_		   = self.builder.as_dataset(
               split="""train"""						,			verification_mode=UpperCamelCase__						,			in_memory=self.keep_in_memory    )
           return dataset


class    A__ :



    def __init__(   self						,			UpperCamelCase__						,			UpperCamelCase__						,			UpperCamelCase__						,			UpperCamelCase__ = None						,			UpperCamelCase__ = None						,			**UpperCamelCase__						,			)  -> List[Any]:





           '''simple docstring'''
           if num_proc is not None and num_proc <= 0:
                  raise ValueError(f'''num_proc {num_proc} must be an integer > 0.'''    )

           A_		   = dataset
           A_		   = name
           A_		   = con
           A_		   = batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
           A_		   = num_proc
           A_		   = to_sql_kwargs



    def    snake_case_     (   self    )  -> int:





           '''simple docstring'''
           A_		   = self.to_sql_kwargs.pop("""sql"""						,			UpperCamelCase__    )
           A_		   = self.to_sql_kwargs.pop("""con"""						,			UpperCamelCase__    )
           A_		   = self.to_sql_kwargs.pop("""index"""						,			UpperCamelCase__    )

           A_		   = self._write(index=UpperCamelCase__						,			**self.to_sql_kwargs    )
           return written



    def    snake_case_     (   self						,			UpperCamelCase__    )  -> Union[str, Any]:





           '''simple docstring'''
           A_   ,     A_   ,     A_		   = args
           A_		   = {**to_sql_kwargs, """if_exists""": """append"""} if offset > 0 else to_sql_kwargs
           A_		   = query_table(
               table=self.dataset.data						,			key=slice(UpperCamelCase__						,			offset + self.batch_size    )						,			indices=self.dataset._indices						,			)
           A_		   = batch.to_pandas()
           A_		   = df.to_sql(self.name						,			self.con						,			index=UpperCamelCase__						,			**UpperCamelCase__    )
           return num_rows or len(UpperCamelCase__    )







    def    snake_case_     (   self						,			UpperCamelCase__						,			**UpperCamelCase__    )  -> int:





           '''simple docstring'''
           A_		   = 0

           if self.num_proc is None or self.num_proc == 1:
                  for offset in logging.tqdm(
                      range(0						,			len(self.dataset    )						,			self.batch_size    )						,			unit="""ba"""						,			disable=not logging.is_progress_bar_enabled()						,			desc="""Creating SQL from Arrow format"""						,			):
                         written += self._batch_sql((offset, index, to_sql_kwargs)    )
           else:
                  A_   ,     A_		   = len(self.dataset    ), self.batch_size
                  with multiprocessing.Pool(self.num_proc    ) as pool:
                         for num_rows in logging.tqdm(
                             pool.imap(
                                 self._batch_sql						,			[(offset, index, to_sql_kwargs) for offset in range(0						,			UpperCamelCase__						,			UpperCamelCase__    )]						,			)						,			total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size						,			unit="""ba"""						,			disable=not logging.is_progress_bar_enabled()						,			desc="""Creating SQL from Arrow format"""						,			):
                                written += num_rows

           return written
