

from unittest.mock import patch

import pyspark

from datasets.packaged_modules.spark.spark import (
    Spark,
    SparkExamplesIterable,
    _generate_iterable_examples,
)

from ..utils import (
    require_dill_gt_0_3_2,
    require_not_windows,
)






def        SCREAMING_SNAKE_CASE__	(  lowerCamelCase__     ,			lowerCamelCase__      )  ->					Tuple:
	__lowerCamelCase    :    List[str] 	=			[]
	for part_id in partition_order:
		__lowerCamelCase    :    str 	=			df.where(F"SPARK_PARTITION_ID() = {part_id}"      ).collect()
		for row_idx, row in enumerate(lowerCamelCase__      ):
			expected_row_ids_and_row_dicts.append((F"{part_id}_{row_idx}", row.asDict())      )
	return expected_row_ids_and_row_dicts






@require_not_windows
@require_dill_gt_0_3_2
def        SCREAMING_SNAKE_CASE__	(  )  ->					str:
	__lowerCamelCase    :    Any 	=			pyspark.sql.SparkSession.builder.master('local[*]'      ).appName('pyspark'      ).getOrCreate()
	__lowerCamelCase    :    Dict 	=			spark.range(1_0_0      ).repartition(1      )
	__lowerCamelCase    :    str 	=			Spark(lowerCamelCase__      )
	# The id ints will be converted to Pyarrow int64s, so each row will be 8 bytes. Setting a max_shard_size of 16 means
	# that each partition can hold 2 rows.
	spark_builder._repartition_df_if_needed(max_shard_size=1_6      )
	# Given that the dataframe has 100 rows and each partition has 2 rows, we expect 50 partitions.
	assert spark_builder.df.rdd.getNumPartitions() == 5_0






@require_not_windows
@require_dill_gt_0_3_2
def        SCREAMING_SNAKE_CASE__	(  )  ->					Dict:
	__lowerCamelCase    :    List[str] 	=			pyspark.sql.SparkSession.builder.master('local[*]'      ).appName('pyspark'      ).getOrCreate()
	__lowerCamelCase    :    Optional[Any] 	=			spark.range(1_0      ).repartition(2      )
	__lowerCamelCase    :    Optional[int] 	=			[1, 0]
	__lowerCamelCase    :    Dict 	=			_generate_iterable_examples(lowerCamelCase__     ,			lowerCamelCase__      )  # Reverse the partitions.
	__lowerCamelCase    :    Dict 	=			_get_expected_row_ids_and_row_dicts_for_partition_order(lowerCamelCase__     ,			lowerCamelCase__      )

	for i, (row_id, row_dict) in enumerate(generate_fn()      ):
		__lowerCamelCase    :    Any 	=			expected_row_ids_and_row_dicts[i]
		assert row_id == expected_row_id
		assert row_dict == expected_row_dict






@require_not_windows
@require_dill_gt_0_3_2
def        SCREAMING_SNAKE_CASE__	(  )  ->					List[str]:
	__lowerCamelCase    :    str 	=			pyspark.sql.SparkSession.builder.master('local[*]'      ).appName('pyspark'      ).getOrCreate()
	__lowerCamelCase    :    List[str] 	=			spark.range(1_0      ).repartition(1      )
	__lowerCamelCase    :    Dict 	=			SparkExamplesIterable(lowerCamelCase__      )
	assert it.n_shards == 1
	for i, (row_id, row_dict) in enumerate(lowerCamelCase__      ):
		assert row_id == F"0_{i}"
		assert row_dict == {"id": i}






@require_not_windows
@require_dill_gt_0_3_2
def        SCREAMING_SNAKE_CASE__	(  )  ->					List[str]:
	__lowerCamelCase    :    Union[str, Any] 	=			pyspark.sql.SparkSession.builder.master('local[*]'      ).appName('pyspark'      ).getOrCreate()
	__lowerCamelCase    :    Any 	=			spark.range(3_0      ).repartition(3      )
	# Mock the generator so that shuffle reverses the partition indices.
	with patch('numpy.random.Generator'      ) as generator_mock:
		__lowerCamelCase    :    int 	=			lambda lowerCamelCase__      : x.reverse()
		__lowerCamelCase    :    List[Any] 	=			_get_expected_row_ids_and_row_dicts_for_partition_order(lowerCamelCase__     ,			[2, 1, 0]      )

		__lowerCamelCase    :    Dict 	=			SparkExamplesIterable(lowerCamelCase__      ).shuffle_data_sources(lowerCamelCase__      )
		assert shuffled_it.n_shards == 3
		for i, (row_id, row_dict) in enumerate(lowerCamelCase__      ):
			__lowerCamelCase    :    Optional[Any] 	=			expected_row_ids_and_row_dicts[i]
			assert row_id == expected_row_id
			assert row_dict == expected_row_dict






@require_not_windows
@require_dill_gt_0_3_2
def        SCREAMING_SNAKE_CASE__	(  )  ->					Dict:
	__lowerCamelCase    :    Optional[Any] 	=			pyspark.sql.SparkSession.builder.master('local[*]'      ).appName('pyspark'      ).getOrCreate()
	__lowerCamelCase    :    int 	=			spark.range(2_0      ).repartition(4      )

	# Partitions 0 and 2
	__lowerCamelCase    :    Optional[Any] 	=			SparkExamplesIterable(lowerCamelCase__      ).shard_data_sources(worker_id=0     ,			num_workers=2      )
	assert shard_it_a.n_shards == 2
	__lowerCamelCase    :    Optional[Any] 	=			_get_expected_row_ids_and_row_dicts_for_partition_order(lowerCamelCase__     ,			[0, 2]      )
	for i, (row_id, row_dict) in enumerate(lowerCamelCase__      ):
		__lowerCamelCase    :    Dict 	=			expected_row_ids_and_row_dicts_a[i]
		assert row_id == expected_row_id
		assert row_dict == expected_row_dict

	# Partitions 1 and 3
	__lowerCamelCase    :    Any 	=			SparkExamplesIterable(lowerCamelCase__      ).shard_data_sources(worker_id=1     ,			num_workers=2      )
	assert shard_it_a.n_shards == 2
	__lowerCamelCase    :    Union[str, Any] 	=			_get_expected_row_ids_and_row_dicts_for_partition_order(lowerCamelCase__     ,			[1, 3]      )
	for i, (row_id, row_dict) in enumerate(lowerCamelCase__      ):
		__lowerCamelCase    :    Any 	=			expected_row_ids_and_row_dicts_a[i]
		assert row_id == expected_row_id
		assert row_dict == expected_row_dict






@require_not_windows
@require_dill_gt_0_3_2
def        SCREAMING_SNAKE_CASE__	(  )  ->					Dict:
	__lowerCamelCase    :    List[str] 	=			pyspark.sql.SparkSession.builder.master('local[*]'      ).appName('pyspark'      ).getOrCreate()
	__lowerCamelCase    :    Optional[int] 	=			spark.range(1_0_0      ).repartition(1      )
	__lowerCamelCase    :    Tuple 	=			Spark(lowerCamelCase__      )
	# Choose a small max_shard_size for maximum partitioning.
	spark_builder._repartition_df_if_needed(max_shard_size=1      )
	# The new number of partitions should not be greater than the number of rows.
	assert spark_builder.df.rdd.getNumPartitions() == 1_0_0







from __future__ import annotations

import unittest

from transformers import is_tf_available, is_torch_available
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, SMALL_MODEL_IDENTIFIER, is_pt_tf_cross_test, slow


if is_tf_available():
					from transformers import (
					    AutoConfig,
					    BertConfig,
					    GPTaConfig,
					    TaConfig,
					    TFAutoModel,
					    TFAutoModelForCausalLM,
					    TFAutoModelForMaskedLM,
					    TFAutoModelForPreTraining,
					    TFAutoModelForQuestionAnswering,
					    TFAutoModelForSeqaSeqLM,
					    TFAutoModelForSequenceClassification,
					    TFAutoModelWithLMHead,
					    TFBertForMaskedLM,
					    TFBertForPreTraining,
					    TFBertForQuestionAnswering,
					    TFBertForSequenceClassification,
					    TFBertModel,
					    TFGPTaLMHeadModel,
					    TFRobertaForMaskedLM,
					    TFTaForConditionalGeneration,
					)
					from transformers.models.bert.modeling_tf_bert import TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST
					from transformers.models.gpta.modeling_tf_gpta import TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST
					from transformers.models.ta.modeling_tf_ta import TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST

if is_torch_available():
					from transformers import (
					    AutoModel,
					    AutoModelForCausalLM,
					    AutoModelForMaskedLM,
					    AutoModelForPreTraining,
					    AutoModelForQuestionAnswering,
					    AutoModelForSeqaSeqLM,
					    AutoModelForSequenceClassification,
					    AutoModelWithLMHead,
					    BertForMaskedLM,
					    BertForPreTraining,
					    BertForQuestionAnswering,
					    BertForSequenceClassification,
					    BertModel,
					    GPTaLMHeadModel,
					    RobertaForMaskedLM,
					    TaForConditionalGeneration,
					)






@is_pt_tf_cross_test
class 	A_		(	unittest.TestCase    ):
							@slow
							def    lowerCAmelCase					(		self		:      List[str]):
								# for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
								for model_name in ["bert-base-uncased"]:
									__lowerCamelCase    :    Optional[int] 	=			AutoConfig.from_pretrained(SCREAMING_SNAKE_CASE__)
									self.assertIsNotNone(SCREAMING_SNAKE_CASE__)
									self.assertIsInstance(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__)

									__lowerCamelCase    :    Tuple 	=			TFAutoModel.from_pretrained(SCREAMING_SNAKE_CASE__       ,from_pt=SCREAMING_SNAKE_CASE__)
									self.assertIsNotNone(SCREAMING_SNAKE_CASE__)
									self.assertIsInstance(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__)

									__lowerCamelCase    :    str 	=			AutoModel.from_pretrained(SCREAMING_SNAKE_CASE__       ,from_tf=SCREAMING_SNAKE_CASE__)
									self.assertIsNotNone(SCREAMING_SNAKE_CASE__)
									self.assertIsInstance(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__)
							@slow
							def    lowerCAmelCase					(		self		:      Optional[int]):
								# for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
								for model_name in ["bert-base-uncased"]:
									__lowerCamelCase    :    str 	=			AutoConfig.from_pretrained(SCREAMING_SNAKE_CASE__)
									self.assertIsNotNone(SCREAMING_SNAKE_CASE__)
									self.assertIsInstance(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__)

									__lowerCamelCase    :    Tuple 	=			TFAutoModelForPreTraining.from_pretrained(SCREAMING_SNAKE_CASE__       ,from_pt=SCREAMING_SNAKE_CASE__)
									self.assertIsNotNone(SCREAMING_SNAKE_CASE__)
									self.assertIsInstance(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__)

									__lowerCamelCase    :    str 	=			AutoModelForPreTraining.from_pretrained(SCREAMING_SNAKE_CASE__       ,from_tf=SCREAMING_SNAKE_CASE__)
									self.assertIsNotNone(SCREAMING_SNAKE_CASE__)
									self.assertIsInstance(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__)
							@slow
							def    lowerCAmelCase					(		self		:      Optional[int]):
								for model_name in TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
									__lowerCamelCase    :    Union[str, Any] 	=			AutoConfig.from_pretrained(SCREAMING_SNAKE_CASE__)
									self.assertIsNotNone(SCREAMING_SNAKE_CASE__)
									self.assertIsInstance(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__)

									__lowerCamelCase    :    Union[str, Any] 	=			TFAutoModelForCausalLM.from_pretrained(SCREAMING_SNAKE_CASE__       ,from_pt=SCREAMING_SNAKE_CASE__)
									__lowerCamelCase	,			__lowerCamelCase    :    Union[str, Any] 	=			TFAutoModelForCausalLM.from_pretrained(
									    SCREAMING_SNAKE_CASE__       ,output_loading_info=SCREAMING_SNAKE_CASE__       ,from_pt=SCREAMING_SNAKE_CASE__)
									self.assertIsNotNone(SCREAMING_SNAKE_CASE__)
									self.assertIsInstance(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__)

									__lowerCamelCase    :    List[Any] 	=			AutoModelForCausalLM.from_pretrained(SCREAMING_SNAKE_CASE__       ,from_tf=SCREAMING_SNAKE_CASE__)
									__lowerCamelCase	,			__lowerCamelCase    :    Any 	=			AutoModelForCausalLM.from_pretrained(
									    SCREAMING_SNAKE_CASE__       ,output_loading_info=SCREAMING_SNAKE_CASE__       ,from_tf=SCREAMING_SNAKE_CASE__)
									self.assertIsNotNone(SCREAMING_SNAKE_CASE__)
									self.assertIsInstance(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__)
							@slow
							def    lowerCAmelCase					(		self		:      List[Any]):
								for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
									__lowerCamelCase    :    List[str] 	=			AutoConfig.from_pretrained(SCREAMING_SNAKE_CASE__)
									self.assertIsNotNone(SCREAMING_SNAKE_CASE__)
									self.assertIsInstance(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__)

									__lowerCamelCase    :    List[str] 	=			TFAutoModelWithLMHead.from_pretrained(SCREAMING_SNAKE_CASE__       ,from_pt=SCREAMING_SNAKE_CASE__)
									self.assertIsNotNone(SCREAMING_SNAKE_CASE__)
									self.assertIsInstance(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__)

									__lowerCamelCase    :    Optional[Any] 	=			AutoModelWithLMHead.from_pretrained(SCREAMING_SNAKE_CASE__       ,from_tf=SCREAMING_SNAKE_CASE__)
									self.assertIsNotNone(SCREAMING_SNAKE_CASE__)
									self.assertIsInstance(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__)
							@slow
							def    lowerCAmelCase					(		self		:      Tuple):
								for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
									__lowerCamelCase    :    Optional[Any] 	=			AutoConfig.from_pretrained(SCREAMING_SNAKE_CASE__)
									self.assertIsNotNone(SCREAMING_SNAKE_CASE__)
									self.assertIsInstance(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__)

									__lowerCamelCase    :    Optional[int] 	=			TFAutoModelForMaskedLM.from_pretrained(SCREAMING_SNAKE_CASE__       ,from_pt=SCREAMING_SNAKE_CASE__)
									__lowerCamelCase	,			__lowerCamelCase    :    Dict 	=			TFAutoModelForMaskedLM.from_pretrained(
									    SCREAMING_SNAKE_CASE__       ,output_loading_info=SCREAMING_SNAKE_CASE__       ,from_pt=SCREAMING_SNAKE_CASE__)
									self.assertIsNotNone(SCREAMING_SNAKE_CASE__)
									self.assertIsInstance(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__)

									__lowerCamelCase    :    List[Any] 	=			AutoModelForMaskedLM.from_pretrained(SCREAMING_SNAKE_CASE__       ,from_tf=SCREAMING_SNAKE_CASE__)
									__lowerCamelCase	,			__lowerCamelCase    :    Optional[Any] 	=			AutoModelForMaskedLM.from_pretrained(
									    SCREAMING_SNAKE_CASE__       ,output_loading_info=SCREAMING_SNAKE_CASE__       ,from_tf=SCREAMING_SNAKE_CASE__)
									self.assertIsNotNone(SCREAMING_SNAKE_CASE__)
									self.assertIsInstance(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__)
							@slow
							def    lowerCAmelCase					(		self		:      str):
								for model_name in TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
									__lowerCamelCase    :    Dict 	=			AutoConfig.from_pretrained(SCREAMING_SNAKE_CASE__)
									self.assertIsNotNone(SCREAMING_SNAKE_CASE__)
									self.assertIsInstance(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__)

									__lowerCamelCase    :    Optional[Any] 	=			TFAutoModelForSeqaSeqLM.from_pretrained(SCREAMING_SNAKE_CASE__       ,from_pt=SCREAMING_SNAKE_CASE__)
									__lowerCamelCase	,			__lowerCamelCase    :    List[str] 	=			TFAutoModelForSeqaSeqLM.from_pretrained(
									    SCREAMING_SNAKE_CASE__       ,output_loading_info=SCREAMING_SNAKE_CASE__       ,from_pt=SCREAMING_SNAKE_CASE__)
									self.assertIsNotNone(SCREAMING_SNAKE_CASE__)
									self.assertIsInstance(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__)

									__lowerCamelCase    :    Any 	=			AutoModelForSeqaSeqLM.from_pretrained(SCREAMING_SNAKE_CASE__       ,from_tf=SCREAMING_SNAKE_CASE__)
									__lowerCamelCase	,			__lowerCamelCase    :    List[str] 	=			AutoModelForSeqaSeqLM.from_pretrained(
									    SCREAMING_SNAKE_CASE__       ,output_loading_info=SCREAMING_SNAKE_CASE__       ,from_tf=SCREAMING_SNAKE_CASE__)
									self.assertIsNotNone(SCREAMING_SNAKE_CASE__)
									self.assertIsInstance(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__)
							@slow
							def    lowerCAmelCase					(		self		:      Optional[Any]):
								# for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
								for model_name in ["bert-base-uncased"]:
									__lowerCamelCase    :    List[Any] 	=			AutoConfig.from_pretrained(SCREAMING_SNAKE_CASE__)
									self.assertIsNotNone(SCREAMING_SNAKE_CASE__)
									self.assertIsInstance(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__)

									__lowerCamelCase    :    Optional[int] 	=			TFAutoModelForSequenceClassification.from_pretrained(SCREAMING_SNAKE_CASE__       ,from_pt=SCREAMING_SNAKE_CASE__)
									self.assertIsNotNone(SCREAMING_SNAKE_CASE__)
									self.assertIsInstance(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__)

									__lowerCamelCase    :    int 	=			AutoModelForSequenceClassification.from_pretrained(SCREAMING_SNAKE_CASE__       ,from_tf=SCREAMING_SNAKE_CASE__)
									self.assertIsNotNone(SCREAMING_SNAKE_CASE__)
									self.assertIsInstance(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__)
							@slow
							def    lowerCAmelCase					(		self		:      Any):
								# for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
								for model_name in ["bert-base-uncased"]:
									__lowerCamelCase    :    List[Any] 	=			AutoConfig.from_pretrained(SCREAMING_SNAKE_CASE__)
									self.assertIsNotNone(SCREAMING_SNAKE_CASE__)
									self.assertIsInstance(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__)

									__lowerCamelCase    :    str 	=			TFAutoModelForQuestionAnswering.from_pretrained(SCREAMING_SNAKE_CASE__       ,from_pt=SCREAMING_SNAKE_CASE__)
									self.assertIsNotNone(SCREAMING_SNAKE_CASE__)
									self.assertIsInstance(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__)

									__lowerCamelCase    :    Optional[int] 	=			AutoModelForQuestionAnswering.from_pretrained(SCREAMING_SNAKE_CASE__       ,from_tf=SCREAMING_SNAKE_CASE__)
									self.assertIsNotNone(SCREAMING_SNAKE_CASE__)
									self.assertIsInstance(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__)
							def    lowerCAmelCase					(		self		:      int):
								__lowerCamelCase    :    List[Any] 	=			TFAutoModelWithLMHead.from_pretrained(SCREAMING_SNAKE_CASE__       ,from_pt=SCREAMING_SNAKE_CASE__)
								self.assertIsInstance(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__)
								self.assertEqual(model.num_parameters()       ,1_4_4_1_0)
								self.assertEqual(model.num_parameters(only_trainable=SCREAMING_SNAKE_CASE__)       ,1_4_4_1_0)

								__lowerCamelCase    :    Union[str, Any] 	=			AutoModelWithLMHead.from_pretrained(SCREAMING_SNAKE_CASE__       ,from_tf=SCREAMING_SNAKE_CASE__)
								self.assertIsInstance(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__)
								self.assertEqual(model.num_parameters()       ,1_4_4_1_0)
								self.assertEqual(model.num_parameters(only_trainable=SCREAMING_SNAKE_CASE__)       ,1_4_4_1_0)


							def    lowerCAmelCase					(		self		:      Tuple):
								__lowerCamelCase    :    str 	=			TFAutoModelWithLMHead.from_pretrained(SCREAMING_SNAKE_CASE__       ,from_pt=SCREAMING_SNAKE_CASE__)
								self.assertIsInstance(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__)
								self.assertEqual(model.num_parameters()       ,1_4_4_1_0)
								self.assertEqual(model.num_parameters(only_trainable=SCREAMING_SNAKE_CASE__)       ,1_4_4_1_0)

								__lowerCamelCase    :    Optional[int] 	=			AutoModelWithLMHead.from_pretrained(SCREAMING_SNAKE_CASE__       ,from_tf=SCREAMING_SNAKE_CASE__)
								self.assertIsInstance(SCREAMING_SNAKE_CASE__       ,SCREAMING_SNAKE_CASE__)
								self.assertEqual(model.num_parameters()       ,1_4_4_1_0)
								self.assertEqual(model.num_parameters(only_trainable=SCREAMING_SNAKE_CASE__)       ,1_4_4_1_0)




