

'''simple docstring'''







from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


if is_sentencepiece_available():
      from ..ta.tokenization_ta import TaTokenizer
else:
      from ...utils.dummy_sentencepiece_objects import TaTokenizer

__lowerCamelCase						    =      TaTokenizer

if is_tokenizers_available():
      from ..ta.tokenization_ta_fast import TaTokenizerFast
else:
      from ...utils.dummy_tokenizers_objects import TaTokenizerFast

__lowerCamelCase						    =      TaTokenizerFast

__lowerCamelCase						    =      {'''configuration_mt5''': ['''MT5Config''', '''MT5OnnxConfig''']}

try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      __lowerCamelCase						    =      [
          '''MT5EncoderModel''',
          '''MT5ForConditionalGeneration''',
          '''MT5ForQuestionAnswering''',
          '''MT5Model''',
          '''MT5PreTrainedModel''',
          '''MT5Stack''',
      ]

try:
      if not is_tf_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      __lowerCamelCase						    =      ['''TFMT5EncoderModel''', '''TFMT5ForConditionalGeneration''', '''TFMT5Model''']

try:
      if not is_flax_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      __lowerCamelCase						    =      ['''FlaxMT5EncoderModel''', '''FlaxMT5ForConditionalGeneration''', '''FlaxMT5Model''']


if TYPE_CHECKING:
      from .configuration_mta import MTaConfig, MTaOnnxConfig

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_mta import (
                MTaEncoderModel,
                MTaForConditionalGeneration,
                MTaForQuestionAnswering,
                MTaModel,
                MTaPreTrainedModel,
                MTaStack,
            )

      try:
            if not is_tf_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_tf_mta import TFMTaEncoderModel, TFMTaForConditionalGeneration, TFMTaModel

      try:
            if not is_flax_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_flax_mta import FlaxMTaEncoderModel, FlaxMTaForConditionalGeneration, FlaxMTaModel

else:
      import sys

      __lowerCamelCase						    =      _LazyModule(
          __name__,
          globals()['''__file__'''],
          _import_structure,
          extra_objects={'''MT5Tokenizer''': MTaTokenizer, '''MT5TokenizerFast''': MTaTokenizerFast},
          module_spec=__spec__,
      )



'''simple docstring'''







import multiprocessing
from typing import TYPE_CHECKING, Optional, Union

from .. import Dataset, Features, config
from ..formatting import query_table
from ..packaged_modules.sql.sql import Sql
from ..utils import logging
from .abc import AbstractDatasetInputStream


if TYPE_CHECKING:
      import sqlitea

      import sqlalchemy


class    A__ ( _snake_case     ):



    def __init__(   self						,			UpperCamelCase__						,			UpperCamelCase__						,			UpperCamelCase__ = None						,			UpperCamelCase__ = None						,			UpperCamelCase__ = False						,			**UpperCamelCase__						,			)  -> Optional[Any]:





           '''simple docstring'''
           super().__init__(features=UpperCamelCase__						,			cache_dir=UpperCamelCase__						,			keep_in_memory=UpperCamelCase__						,			**UpperCamelCase__    )
           A_		   = Sql(
               cache_dir=UpperCamelCase__						,			features=UpperCamelCase__						,			sql=UpperCamelCase__						,			con=UpperCamelCase__						,			**UpperCamelCase__						,			)







    def    snake_case_     (   self    )  -> Optional[Any]:





           '''simple docstring'''
           A_		   = None
           A_		   = None
           A_		   = None
           A_		   = None

           self.builder.download_and_prepare(
               download_config=UpperCamelCase__						,			download_mode=UpperCamelCase__						,			verification_mode=UpperCamelCase__						,			base_path=UpperCamelCase__						,			)

           # Build dataset for splits
           A_		   = self.builder.as_dataset(
               split="""train"""						,			verification_mode=UpperCamelCase__						,			in_memory=self.keep_in_memory    )
           return dataset


class    A__ :



    def __init__(   self						,			UpperCamelCase__						,			UpperCamelCase__						,			UpperCamelCase__						,			UpperCamelCase__ = None						,			UpperCamelCase__ = None						,			**UpperCamelCase__						,			)  -> List[Any]:





           '''simple docstring'''
           if num_proc is not None and num_proc <= 0:
                  raise ValueError(f'''num_proc {num_proc} must be an integer > 0.'''    )

           A_		   = dataset
           A_		   = name
           A_		   = con
           A_		   = batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
           A_		   = num_proc
           A_		   = to_sql_kwargs



    def    snake_case_     (   self    )  -> int:





           '''simple docstring'''
           A_		   = self.to_sql_kwargs.pop("""sql"""						,			UpperCamelCase__    )
           A_		   = self.to_sql_kwargs.pop("""con"""						,			UpperCamelCase__    )
           A_		   = self.to_sql_kwargs.pop("""index"""						,			UpperCamelCase__    )

           A_		   = self._write(index=UpperCamelCase__						,			**self.to_sql_kwargs    )
           return written



    def    snake_case_     (   self						,			UpperCamelCase__    )  -> Union[str, Any]:





           '''simple docstring'''
           A_   ,     A_   ,     A_		   = args
           A_		   = {**to_sql_kwargs, """if_exists""": """append"""} if offset > 0 else to_sql_kwargs
           A_		   = query_table(
               table=self.dataset.data						,			key=slice(UpperCamelCase__						,			offset + self.batch_size    )						,			indices=self.dataset._indices						,			)
           A_		   = batch.to_pandas()
           A_		   = df.to_sql(self.name						,			self.con						,			index=UpperCamelCase__						,			**UpperCamelCase__    )
           return num_rows or len(UpperCamelCase__    )







    def    snake_case_     (   self						,			UpperCamelCase__						,			**UpperCamelCase__    )  -> int:





           '''simple docstring'''
           A_		   = 0

           if self.num_proc is None or self.num_proc == 1:
                  for offset in logging.tqdm(
                      range(0						,			len(self.dataset    )						,			self.batch_size    )						,			unit="""ba"""						,			disable=not logging.is_progress_bar_enabled()						,			desc="""Creating SQL from Arrow format"""						,			):
                         written += self._batch_sql((offset, index, to_sql_kwargs)    )
           else:
                  A_   ,     A_		   = len(self.dataset    ), self.batch_size
                  with multiprocessing.Pool(self.num_proc    ) as pool:
                         for num_rows in logging.tqdm(
                             pool.imap(
                                 self._batch_sql						,			[(offset, index, to_sql_kwargs) for offset in range(0						,			UpperCamelCase__						,			UpperCamelCase__    )]						,			)						,			total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size						,			unit="""ba"""						,			disable=not logging.is_progress_bar_enabled()						,			desc="""Creating SQL from Arrow format"""						,			):
                                written += num_rows

           return written
