


"""simple docstring"""







from __future__ import annotations

import unittest

from transformers import RoFormerConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
     import tensorflow as tf

     from transformers import (
         TFRoFormerForCausalLM,
         TFRoFormerForMaskedLM,
         TFRoFormerForMultipleChoice,
         TFRoFormerForQuestionAnswering,
         TFRoFormerForSequenceClassification,
         TFRoFormerForTokenClassification,
         TFRoFormerModel,
     )
     from transformers.models.roformer.modeling_tf_roformer import (
         TFRoFormerSelfAttention,
         TFRoFormerSinusoidalPositionalEmbedding,
     )


class 				SCREAMING_SNAKE_CASE__    :
       """simple docstring"""


       def __init__(						self		,					snake_case__		,					snake_case__=13		,					snake_case__=7		,					snake_case__=True		,					snake_case__=True		,					snake_case__=True		,					snake_case__=True		,					snake_case__=99		,					snake_case__=32		,					snake_case__=2		,					snake_case__=4		,					snake_case__=37		,					snake_case__="gelu"		,					snake_case__=0.1		,					snake_case__=0.1		,					snake_case__=512		,					snake_case__=16		,					snake_case__=2		,					snake_case__=0.02		,					snake_case__=3		,					snake_case__=4		,					snake_case__=None		,					):






         """simple docstring"""





         lowerCAmelCase					:					int								=							parent
         lowerCAmelCase					:					str								=							13
         lowerCAmelCase					:					int								=							7
         lowerCAmelCase					:					Optional[int]								=							True
         lowerCAmelCase					:					str								=							True
         lowerCAmelCase					:					List[Any]								=							True
         lowerCAmelCase					:					int								=							True
         lowerCAmelCase					:					Union[str, Any]								=							99
         lowerCAmelCase					:					str								=							32
         lowerCAmelCase					:					Union[str, Any]								=							2
         lowerCAmelCase					:					Tuple								=							4
         lowerCAmelCase					:					Tuple								=							37
         lowerCAmelCase					:					str								=							"gelu"
         lowerCAmelCase					:					Dict								=							0.1
         lowerCAmelCase					:					str								=							0.1
         lowerCAmelCase					:					Tuple								=							512
         lowerCAmelCase					:					Any								=							16
         lowerCAmelCase					:					Optional[int]								=							2
         lowerCAmelCase					:					str								=							0.02
         lowerCAmelCase					:					Optional[int]								=							3
         lowerCAmelCase					:					Optional[int]								=							4
         lowerCAmelCase					:					Optional[int]								=							None

       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					str								=							ids_tensor([self.batch_size, self.seq_length]		,					self.vocab_size						)

         lowerCAmelCase					:					List[Any]								=							None
         if self.use_input_mask:
           lowerCAmelCase					:					str								=							random_attention_mask([self.batch_size, self.seq_length]						)

         lowerCAmelCase					:					str								=							None
         if self.use_token_type_ids:
           lowerCAmelCase					:					Tuple								=							ids_tensor([self.batch_size, self.seq_length]		,					self.type_vocab_size						)

         lowerCAmelCase					:					Any								=							None
         lowerCAmelCase					:					Optional[int]								=							None
         lowerCAmelCase					:					Optional[int]								=							None
         if self.use_labels:
           lowerCAmelCase					:					Dict								=							ids_tensor([self.batch_size]		,					self.type_sequence_label_size						)
           lowerCAmelCase					:					Union[str, Any]								=							ids_tensor([self.batch_size, self.seq_length]		,					self.num_labels						)
           lowerCAmelCase					:					Tuple								=							ids_tensor([self.batch_size]		,					self.num_choices						)

         lowerCAmelCase					:					Optional[Any]								=							RoFormerConfig(
             vocab_size=self.vocab_size		,					hidden_size=self.hidden_size		,					num_hidden_layers=self.num_hidden_layers		,					num_attention_heads=self.num_attention_heads		,					intermediate_size=self.intermediate_size		,					hidden_act=self.hidden_act		,					hidden_dropout_prob=self.hidden_dropout_prob		,					attention_probs_dropout_prob=self.attention_probs_dropout_prob		,					max_position_embeddings=self.max_position_embeddings		,					type_vocab_size=self.type_vocab_size		,					initializer_range=self.initializer_range		,					return_dict=snake_case__		,					)

         return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels

       def 	lowercase__							(						self		,					snake_case__		,					snake_case__		,					snake_case__		,					snake_case__		,					snake_case__		,					snake_case__		,					snake_case__						):






         """simple docstring"""





         lowerCAmelCase					:					Optional[int]								=							TFRoFormerModel(config=snake_case__						)
         lowerCAmelCase					:					Optional[Any]								=							{"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}

         lowerCAmelCase					:					List[Any]								=							[input_ids, input_mask]
         lowerCAmelCase					:					Optional[int]								=							model(snake_case__						)

         lowerCAmelCase					:					Any								=							model(snake_case__						)

         self.parent.assertEqual(result.last_hidden_state.shape		,					(self.batch_size, self.seq_length, self.hidden_size)						)

       def 	lowercase__							(						self		,					snake_case__		,					snake_case__		,					snake_case__		,					snake_case__		,					snake_case__		,					snake_case__		,					snake_case__						):






         """simple docstring"""





         lowerCAmelCase					:					Optional[int]								=							True
         lowerCAmelCase					:					str								=							TFRoFormerForCausalLM(config=snake_case__						)
         lowerCAmelCase					:					Optional[Any]								=							{
             "input_ids": input_ids,
             "attention_mask": input_mask,
             "token_type_ids": token_type_ids,
         }
         lowerCAmelCase					:					Optional[int]								=							model(snake_case__						)["logits"]
         self.parent.assertListEqual(
             list(prediction_scores.numpy().shape						)		,					[self.batch_size, self.seq_length, self.vocab_size]						)

       def 	lowercase__							(						self		,					snake_case__		,					snake_case__		,					snake_case__		,					snake_case__		,					snake_case__		,					snake_case__		,					snake_case__						):






         """simple docstring"""





         lowerCAmelCase					:					int								=							TFRoFormerForMaskedLM(config=snake_case__						)
         lowerCAmelCase					:					Tuple								=							{
             "input_ids": input_ids,
             "attention_mask": input_mask,
             "token_type_ids": token_type_ids,
         }
         lowerCAmelCase					:					Tuple								=							model(snake_case__						)
         self.parent.assertEqual(result.logits.shape		,					(self.batch_size, self.seq_length, self.vocab_size)						)

       def 	lowercase__							(						self		,					snake_case__		,					snake_case__		,					snake_case__		,					snake_case__		,					snake_case__		,					snake_case__		,					snake_case__						):






         """simple docstring"""





         lowerCAmelCase					:					List[str]								=							self.num_labels
         lowerCAmelCase					:					Optional[Any]								=							TFRoFormerForSequenceClassification(config=snake_case__						)
         lowerCAmelCase					:					str								=							{
             "input_ids": input_ids,
             "attention_mask": input_mask,
             "token_type_ids": token_type_ids,
         }

         lowerCAmelCase					:					Tuple								=							model(snake_case__						)
         self.parent.assertEqual(result.logits.shape		,					(self.batch_size, self.num_labels)						)

       def 	lowercase__							(						self		,					snake_case__		,					snake_case__		,					snake_case__		,					snake_case__		,					snake_case__		,					snake_case__		,					snake_case__						):






         """simple docstring"""





         lowerCAmelCase					:					List[Any]								=							self.num_choices
         lowerCAmelCase					:					Optional[Any]								=							TFRoFormerForMultipleChoice(config=snake_case__						)
         lowerCAmelCase					:					str								=							tf.tile(tf.expand_dims(snake_case__		,					1						)		,					(1, self.num_choices, 1)						)
         lowerCAmelCase					:					Optional[int]								=							tf.tile(tf.expand_dims(snake_case__		,					1						)		,					(1, self.num_choices, 1)						)
         lowerCAmelCase					:					Union[str, Any]								=							tf.tile(tf.expand_dims(snake_case__		,					1						)		,					(1, self.num_choices, 1)						)
         lowerCAmelCase					:					Optional[int]								=							{
             "input_ids": multiple_choice_inputs_ids,
             "attention_mask": multiple_choice_input_mask,
             "token_type_ids": multiple_choice_token_type_ids,
         }
         lowerCAmelCase					:					List[Any]								=							model(snake_case__						)
         self.parent.assertEqual(result.logits.shape		,					(self.batch_size, self.num_choices)						)

       def 	lowercase__							(						self		,					snake_case__		,					snake_case__		,					snake_case__		,					snake_case__		,					snake_case__		,					snake_case__		,					snake_case__						):






         """simple docstring"""





         lowerCAmelCase					:					List[Any]								=							self.num_labels
         lowerCAmelCase					:					Optional[Any]								=							TFRoFormerForTokenClassification(config=snake_case__						)
         lowerCAmelCase					:					Tuple								=							{
             "input_ids": input_ids,
             "attention_mask": input_mask,
             "token_type_ids": token_type_ids,
         }
         lowerCAmelCase					:					int								=							model(snake_case__						)
         self.parent.assertEqual(result.logits.shape		,					(self.batch_size, self.seq_length, self.num_labels)						)

       def 	lowercase__							(						self		,					snake_case__		,					snake_case__		,					snake_case__		,					snake_case__		,					snake_case__		,					snake_case__		,					snake_case__						):






         """simple docstring"""





         lowerCAmelCase					:					List[Any]								=							TFRoFormerForQuestionAnswering(config=snake_case__						)
         lowerCAmelCase					:					List[Any]								=							{
             "input_ids": input_ids,
             "attention_mask": input_mask,
             "token_type_ids": token_type_ids,
         }

         lowerCAmelCase					:					Tuple								=							model(snake_case__						)
         self.parent.assertEqual(result.start_logits.shape		,					(self.batch_size, self.seq_length)						)
         self.parent.assertEqual(result.end_logits.shape		,					(self.batch_size, self.seq_length)						)







       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					Dict								=							self.prepare_config_and_inputs()
         (
             (
             lowerCAmelCase
         )						,  (
             lowerCAmelCase
         )						,  (
             lowerCAmelCase
         )						,  (
             lowerCAmelCase
         )						,  (
             lowerCAmelCase
         )						,  (
             lowerCAmelCase
         )						,  (
             lowerCAmelCase
         )						,  
         )					:					int								=							config_and_inputs
         lowerCAmelCase					:					List[str]								=							{"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
         return config, inputs_dict


@require_tf
class 				SCREAMING_SNAKE_CASE__    (		lowercase       ,   lowercase       ,   unittest.TestCase    ):
       """simple docstring"""


       a  :    Any       =(
           (
               TFRoFormerModel,
               TFRoFormerForCausalLM,
               TFRoFormerForMaskedLM,
               TFRoFormerForQuestionAnswering,
               TFRoFormerForSequenceClassification,
               TFRoFormerForTokenClassification,
               TFRoFormerForMultipleChoice,
           )
           if is_tf_available()
           else ()
       )
       a  :    Optional[int]       =(
           {
               "feature-extraction": TFRoFormerModel,
               "fill-mask": TFRoFormerForMaskedLM,
               "question-answering": TFRoFormerForQuestionAnswering,
               "text-classification": TFRoFormerForSequenceClassification,
               "text-generation": TFRoFormerForCausalLM,
               "token-classification": TFRoFormerForTokenClassification,
               "zero-shot": TFRoFormerForSequenceClassification,
           }
           if is_tf_available()
           else {}
       )

       a  :    Dict       =False
       a  :    int       =False

       def 	lowercase__							(						self		,					snake_case__		,					snake_case__		,					snake_case__		,					snake_case__		,					snake_case__						):






         """simple docstring"""





         if pipeline_test_casse_name == "TextGenerationPipelineTests":
           return True

         return False

       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					Optional[int]								=							TFRoFormerModelTester(self						)
         lowerCAmelCase					:					Optional[int]								=							ConfigTester(self		,					config_class=snake_case__		,					hidden_size=37						)

       def 	lowercase__							(						self						):






         """simple docstring"""





         self.config_tester.run_common_tests()

       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					List[str]								=							self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(*snake_case__						)

       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					Any								=							self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_masked_lm(*snake_case__						)

       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					Union[str, Any]								=							self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_lm_head(*snake_case__						)

       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					int								=							self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_multiple_choice(*snake_case__						)

       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					List[Any]								=							self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_question_answering(*snake_case__						)

       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					Optional[int]								=							self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_sequence_classification(*snake_case__						)

       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					List[Any]								=							self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_token_classification(*snake_case__						)







       @slow
       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					Any								=							TFRoFormerModel.from_pretrained("junnyu/roformer_chinese_base"						)
         self.assertIsNotNone(snake_case__						)


@require_tf
class 				SCREAMING_SNAKE_CASE__    (		unittest.TestCase    ):
       """simple docstring"""


       @slow
       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					List[str]								=							TFRoFormerForMaskedLM.from_pretrained("junnyu/roformer_chinese_base"						)
         lowerCAmelCase					:					Any								=							tf.constant([[0, 1, 2, 3, 4, 5]]						)
         lowerCAmelCase					:					int								=							model(snake_case__						)[0]

         # TODO Replace vocab size
         lowerCAmelCase					:					Union[str, Any]								=							50_000

         lowerCAmelCase					:					Union[str, Any]								=							[1, 6, vocab_size]
         self.assertEqual(output.shape		,					snake_case__						)

         print(output[:, :3, :3]						)

         # TODO Replace values below with what was printed above.
         lowerCAmelCase					:					List[str]								=							tf.constant(
             [
                 [
                     [-0.12053341, -1.0264901, 0.29221946],
                     [-1.5133783, 0.197433, 0.15190607],
                     [-5.0135403, -3.900256, -0.84038764],
                 ]
             ]						)
         tf.debugging.assert_near(output[:, :3, :3]		,					snake_case__		,					atol=1e-4						)


@require_tf
class 				SCREAMING_SNAKE_CASE__    (		unittest.TestCase    ):
       """simple docstring"""


       a  :    Any       =1E-4

       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					Dict								=							tf.constant([[4, 10]]						)
         lowerCAmelCase					:					Optional[Any]								=							TFRoFormerSinusoidalPositionalEmbedding(num_positions=6		,					embedding_dim=6						)

         lowerCAmelCase					:					str								=							emba(input_ids.shape						)
         lowerCAmelCase					:					Union[str, Any]								=							tf.constant(
             [[0.0000, 0.0000, 0.0000, 1.0000, 1.0000, 1.0000], [0.8415, 0.0464, 0.0022, 0.5403, 0.9989, 1.0000]]						)

         tf.debugging.assert_near(snake_case__		,					snake_case__		,					atol=self.tolerance						)







       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					str								=							tf.constant(
             [
                 [0.0000, 0.0000, 0.0000, 0.0000, 0.0000],
                 [0.8415, 0.8219, 0.8020, 0.7819, 0.7617],
                 [0.9093, 0.9364, 0.9581, 0.9749, 0.9870],
             ]						)
         lowerCAmelCase					:					List[str]								=							TFRoFormerSinusoidalPositionalEmbedding(num_positions=512		,					embedding_dim=512						)
         emba([2, 16, 512]						)
         lowerCAmelCase					:					Optional[int]								=							emba.weight[:3, :5]

         tf.debugging.assert_near(snake_case__		,					snake_case__		,					atol=self.tolerance						)


@require_tf
class 				SCREAMING_SNAKE_CASE__    (		unittest.TestCase    ):
       """simple docstring"""


       a  :    List[str]       =1E-4

       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					Tuple								=							tf.reshape(tf.range(2 * 12 * 16 * 64		,					dtype=tf.floataa						)		,					shape=(2, 12, 16, 64)						) / 100

         lowerCAmelCase					:					Optional[Any]								=							-tf.reshape(tf.range(2 * 12 * 16 * 64		,					dtype=tf.floataa						)		,					shape=(2, 12, 16, 64)						) / 100

         lowerCAmelCase					:					Union[str, Any]								=							TFRoFormerSinusoidalPositionalEmbedding(num_positions=32		,					embedding_dim=64						)
         lowerCAmelCase					:					Optional[int]								=							embed_positions([2, 16, 768]						)[None, None, :, :]

         lowerCAmelCase						,  lowerCAmelCase					:					List[str]								=							TFRoFormerSelfAttention.apply_rotary_position_embeddings(
             snake_case__		,					snake_case__		,					snake_case__						)

         lowerCAmelCase					:					Optional[int]								=							tf.constant(
             [
                 [0.0000, 0.0100, 0.0200, 0.0300, 0.0400, 0.0500, 0.0600, 0.0700],
                 [-0.2012, 0.8897, 0.0263, 0.9401, 0.2074, 0.9463, 0.3481, 0.9343],
                 [-1.7057, 0.6271, -1.2145, 1.3897, -0.6303, 1.7647, -0.1173, 1.8985],
                 [-2.1731, -1.6397, -2.7358, 0.2854, -2.1840, 1.7183, -1.3018, 2.4871],
                 [0.2717, -3.6173, -2.9206, -2.1988, -3.6638, 0.3858, -2.9155, 2.2980],
                 [3.9859, -2.1580, -0.7984, -4.4904, -4.1181, -2.0252, -4.4782, 1.1253],
             ]						)
         lowerCAmelCase					:					Any								=							tf.constant(
             [
                 [0.0000, -0.0100, -0.0200, -0.0300, -0.0400, -0.0500, -0.0600, -0.0700],
                 [0.2012, -0.8897, -0.0263, -0.9401, -0.2074, -0.9463, -0.3481, -0.9343],
                 [1.7057, -0.6271, 1.2145, -1.3897, 0.6303, -1.7647, 0.1173, -1.8985],
                 [2.1731, 1.6397, 2.7358, -0.2854, 2.1840, -1.7183, 1.3018, -2.4871],
                 [-0.2717, 3.6173, 2.9206, 2.1988, 3.6638, -0.3858, 2.9155, -2.2980],
                 [-3.9859, 2.1580, 0.7984, 4.4904, 4.1181, 2.0252, 4.4782, -1.1253],
             ]						)

         tf.debugging.assert_near(query_layer[0, 0, :6, :8]		,					snake_case__		,					atol=self.tolerance						)
         tf.debugging.assert_near(key_layer[0, 0, :6, :8]		,					snake_case__		,					atol=self.tolerance						)









"""simple docstring"""







import multiprocessing
from typing import TYPE_CHECKING, Optional, Union

from .. import Dataset, Features, config
from ..formatting import query_table
from ..packaged_modules.sql.sql import Sql
from ..utils import logging
from .abc import AbstractDatasetInputStream


if TYPE_CHECKING:
     import sqlitea

     import sqlalchemy


class 				SCREAMING_SNAKE_CASE__    (		lowercase    ):
       """simple docstring"""


       def __init__(						self		,					snake_case__		,					snake_case__		,					snake_case__ = None		,					snake_case__ = None		,					snake_case__ = False		,					**snake_case__		,					):






         """simple docstring"""





         super().__init__(features=snake_case__		,					cache_dir=snake_case__		,					keep_in_memory=snake_case__		,					**snake_case__						)
         lowerCAmelCase					:					Dict								=							Sql(
             cache_dir=snake_case__		,					features=snake_case__		,					sql=snake_case__		,					con=snake_case__		,					**snake_case__		,					)







       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					Tuple								=							None
         lowerCAmelCase					:					Optional[int]								=							None
         lowerCAmelCase					:					str								=							None
         lowerCAmelCase					:					int								=							None

         self.builder.download_and_prepare(
             download_config=snake_case__		,					download_mode=snake_case__		,					verification_mode=snake_case__		,					base_path=snake_case__		,					)

         # Build dataset for splits
         lowerCAmelCase					:					str								=							self.builder.as_dataset(
             split="train"		,					verification_mode=snake_case__		,					in_memory=self.keep_in_memory						)
         return dataset


class 				SCREAMING_SNAKE_CASE__    :
       """simple docstring"""


       def __init__(						self		,					snake_case__		,					snake_case__		,					snake_case__		,					snake_case__ = None		,					snake_case__ = None		,					**snake_case__		,					):






         """simple docstring"""





         if num_proc is not None and num_proc <= 0:
           raise ValueError(f"""num_proc {num_proc} must be an integer > 0."""						)

         lowerCAmelCase					:					Tuple								=							dataset
         lowerCAmelCase					:					Tuple								=							name
         lowerCAmelCase					:					List[str]								=							con
         lowerCAmelCase					:					List[Any]								=							batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
         lowerCAmelCase					:					Optional[int]								=							num_proc
         lowerCAmelCase					:					Optional[int]								=							to_sql_kwargs

       def 	lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase					:					List[str]								=							self.to_sql_kwargs.pop("sql"		,					snake_case__						)
         lowerCAmelCase					:					List[Any]								=							self.to_sql_kwargs.pop("con"		,					snake_case__						)
         lowerCAmelCase					:					Dict								=							self.to_sql_kwargs.pop("index"		,					snake_case__						)

         lowerCAmelCase					:					Dict								=							self._write(index=snake_case__		,					**self.to_sql_kwargs						)
         return written

       def 	lowercase__							(						self		,					snake_case__						):






         """simple docstring"""





         lowerCAmelCase						,  lowerCAmelCase						,  lowerCAmelCase					:					Optional[int]								=							args
         lowerCAmelCase					:					Optional[int]								=							{**to_sql_kwargs, "if_exists": "append"} if offset > 0 else to_sql_kwargs
         lowerCAmelCase					:					List[Any]								=							query_table(
             table=self.dataset.data		,					key=slice(snake_case__		,					offset + self.batch_size						)		,					indices=self.dataset._indices		,					)
         lowerCAmelCase					:					Tuple								=							batch.to_pandas()
         lowerCAmelCase					:					str								=							df.to_sql(self.name		,					self.con		,					index=snake_case__		,					**snake_case__						)
         return num_rows or len(snake_case__						)







       def 	lowercase__							(						self		,					snake_case__		,					**snake_case__						):






         """simple docstring"""





         lowerCAmelCase					:					Tuple								=							0

         if self.num_proc is None or self.num_proc == 1:
           for offset in logging.tqdm(
               range(0		,					len(self.dataset						)		,					self.batch_size						)		,					unit="ba"		,					disable=not logging.is_progress_bar_enabled()		,					desc="Creating SQL from Arrow format"		,					):
             written += self._batch_sql((offset, index, to_sql_kwargs)						)
         else:
           lowerCAmelCase						,  lowerCAmelCase					:					Dict								=							len(self.dataset						), self.batch_size
           with multiprocessing.Pool(self.num_proc						) as pool:
             for num_rows in logging.tqdm(
                 pool.imap(
                     self._batch_sql		,					[(offset, index, to_sql_kwargs) for offset in range(0		,					snake_case__		,					snake_case__						)]		,					)		,					total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size		,					unit="ba"		,					disable=not logging.is_progress_bar_enabled()		,					desc="Creating SQL from Arrow format"		,					):
               written += num_rows

         return written





