



from __future__ import annotations

import copy
import tempfile
import unittest

from transformers import CONFIG_MAPPING, AutoConfig, BertConfig, GPTaConfig, TaConfig, TapasConfig, is_tf_available
from transformers.testing_utils import (
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tensorflow_probability,
    require_tf,
    slow,
)

from ..bert.test_modeling_bert import BertModelTester


if is_tf_available():
			from transformers import (
			    TFAutoModel,
			    TFAutoModelForCausalLM,
			    TFAutoModelForMaskedLM,
			    TFAutoModelForPreTraining,
			    TFAutoModelForQuestionAnswering,
			    TFAutoModelForSeqaSeqLM,
			    TFAutoModelForSequenceClassification,
			    TFAutoModelForTableQuestionAnswering,
			    TFAutoModelForTokenClassification,
			    TFAutoModelWithLMHead,
			    TFBertForMaskedLM,
			    TFBertForPreTraining,
			    TFBertForQuestionAnswering,
			    TFBertForSequenceClassification,
			    TFBertModel,
			    TFFunnelBaseModel,
			    TFFunnelModel,
			    TFGPTaLMHeadModel,
			    TFRobertaForMaskedLM,
			    TFTaForConditionalGeneration,
			    TFTapasForQuestionAnswering,
			)
			from transformers.models.auto.modeling_tf_auto import (
			    TF_MODEL_FOR_CAUSAL_LM_MAPPING,
			    TF_MODEL_FOR_MASKED_LM_MAPPING,
			    TF_MODEL_FOR_PRETRAINING_MAPPING,
			    TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
			    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
			    TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
			    TF_MODEL_MAPPING,
			)
			from transformers.models.bert.modeling_tf_bert import TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST
			from transformers.models.gpta.modeling_tf_gpta import TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST
			from transformers.models.ta.modeling_tf_ta import TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST
			from transformers.models.tapas.modeling_tf_tapas import TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST

class        UpperCAmelCase_  (       UpperCamelCase_	):

						'''simple docstring'''




						UpperCamelCase__      :			str      =  '''new-model'''


if is_tf_available():

			class        UpperCAmelCase_  (       UpperCamelCase_	):

									'''simple docstring'''




									UpperCamelCase__      :			Tuple      =  NewModelConfig

@require_tf
class        UpperCAmelCase_  (       unittest.TestCase	):

						'''simple docstring'''




						@slow
						def 	_A     (   self		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= 'bert-base-cased'
													__SCREAMING_SNAKE_CASE									= AutoConfig.from_pretrained(_A		)
													self.assertIsNotNone(_A		)
													self.assertIsInstance(_A				,							_A		)

													__SCREAMING_SNAKE_CASE									= TFAutoModel.from_pretrained(_A		)
													self.assertIsNotNone(_A		)
													self.assertIsInstance(_A				,							_A		)






						@slow
						def 	_A     (   self		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= 'bert-base-cased'
													__SCREAMING_SNAKE_CASE									= AutoConfig.from_pretrained(_A		)
													self.assertIsNotNone(_A		)
													self.assertIsInstance(_A				,							_A		)

													__SCREAMING_SNAKE_CASE									= TFAutoModelForPreTraining.from_pretrained(_A		)
													self.assertIsNotNone(_A		)
													self.assertIsInstance(_A				,							_A		)






						@slow
						def 	_A     (   self		):







													'''simple docstring'''

													for model_name in TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																				__SCREAMING_SNAKE_CASE									= AutoConfig.from_pretrained(_A		)
																				self.assertIsNotNone(_A		)
																				self.assertIsInstance(_A				,							_A		)

																				__SCREAMING_SNAKE_CASE									= TFAutoModelForCausalLM.from_pretrained(_A		)
																				__SCREAMING_SNAKE_CASE							,	__SCREAMING_SNAKE_CASE									= TFAutoModelForCausalLM.from_pretrained(_A				,							output_loading_info=_A		)
																				self.assertIsNotNone(_A		)
																				self.assertIsInstance(_A				,							_A		)






						@slow
						def 	_A     (   self		):







													'''simple docstring'''

													for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																				__SCREAMING_SNAKE_CASE									= AutoConfig.from_pretrained(_A		)
																				self.assertIsNotNone(_A		)
																				self.assertIsInstance(_A				,							_A		)

																				__SCREAMING_SNAKE_CASE									= TFAutoModelWithLMHead.from_pretrained(_A		)
																				self.assertIsNotNone(_A		)
																				self.assertIsInstance(_A				,							_A		)






						@slow
						def 	_A     (   self		):







													'''simple docstring'''

													for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																				__SCREAMING_SNAKE_CASE									= AutoConfig.from_pretrained(_A		)
																				self.assertIsNotNone(_A		)
																				self.assertIsInstance(_A				,							_A		)

																				__SCREAMING_SNAKE_CASE									= TFAutoModelForMaskedLM.from_pretrained(_A		)
																				__SCREAMING_SNAKE_CASE							,	__SCREAMING_SNAKE_CASE									= TFAutoModelForMaskedLM.from_pretrained(_A				,							output_loading_info=_A		)
																				self.assertIsNotNone(_A		)
																				self.assertIsInstance(_A				,							_A		)






						@slow
						def 	_A     (   self		):







													'''simple docstring'''

													for model_name in TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																				__SCREAMING_SNAKE_CASE									= AutoConfig.from_pretrained(_A		)
																				self.assertIsNotNone(_A		)
																				self.assertIsInstance(_A				,							_A		)

																				__SCREAMING_SNAKE_CASE									= TFAutoModelForSeqaSeqLM.from_pretrained(_A		)
																				__SCREAMING_SNAKE_CASE							,	__SCREAMING_SNAKE_CASE									= TFAutoModelForSeqaSeqLM.from_pretrained(_A				,							output_loading_info=_A		)
																				self.assertIsNotNone(_A		)
																				self.assertIsInstance(_A				,							_A		)






						@slow
						def 	_A     (   self		):







													'''simple docstring'''

													for model_name in ["bert-base-uncased"]:
																				__SCREAMING_SNAKE_CASE									= AutoConfig.from_pretrained(_A		)
																				self.assertIsNotNone(_A		)
																				self.assertIsInstance(_A				,							_A		)

																				__SCREAMING_SNAKE_CASE									= TFAutoModelForSequenceClassification.from_pretrained(_A		)
																				self.assertIsNotNone(_A		)
																				self.assertIsInstance(_A				,							_A		)






						@slow
						def 	_A     (   self		):







													'''simple docstring'''

													for model_name in ["bert-base-uncased"]:
																				__SCREAMING_SNAKE_CASE									= AutoConfig.from_pretrained(_A		)
																				self.assertIsNotNone(_A		)
																				self.assertIsInstance(_A				,							_A		)

																				__SCREAMING_SNAKE_CASE									= TFAutoModelForQuestionAnswering.from_pretrained(_A		)
																				self.assertIsNotNone(_A		)
																				self.assertIsInstance(_A				,							_A		)






						@slow
						@require_tensorflow_probability
						def 	_A     (   self		):







													'''simple docstring'''

													for model_name in TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST[5:6]:
																				__SCREAMING_SNAKE_CASE									= AutoConfig.from_pretrained(_A		)
																				self.assertIsNotNone(_A		)
																				self.assertIsInstance(_A				,							_A		)

																				__SCREAMING_SNAKE_CASE									= TFAutoModelForTableQuestionAnswering.from_pretrained(_A		)
																				__SCREAMING_SNAKE_CASE							,	__SCREAMING_SNAKE_CASE									= TFAutoModelForTableQuestionAnswering.from_pretrained(
																				    _A				,							output_loading_info=_A		)
																				self.assertIsNotNone(_A		)
																				self.assertIsInstance(_A				,							_A		)






						def 	_A     (   self		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= TFAutoModelWithLMHead.from_pretrained(_A		)
													self.assertIsInstance(_A				,							_A		)
													self.assertEqual(model.num_parameters()				,							14_410		)
													self.assertEqual(model.num_parameters(only_trainable=_A		)				,							14_410		)






						def 	_A     (   self		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= TFAutoModelWithLMHead.from_pretrained(_A		)
													self.assertIsInstance(_A				,							_A		)
													self.assertEqual(model.num_parameters()				,							14_410		)
													self.assertEqual(model.num_parameters(only_trainable=_A		)				,							14_410		)






						def 	_A     (   self		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= TFAutoModel.from_pretrained('sgugger/funnel-random-tiny'		)
													self.assertIsInstance(_A				,							_A		)

													__SCREAMING_SNAKE_CASE									= copy.deepcopy(model.config		)
													__SCREAMING_SNAKE_CASE									= ['FunnelBaseModel']
													__SCREAMING_SNAKE_CASE									= TFAutoModel.from_config(_A		)
													self.assertIsInstance(_A				,							_A		)

													with tempfile.TemporaryDirectory() as tmp_dir:
																				model.save_pretrained(_A		)
																				__SCREAMING_SNAKE_CASE									= TFAutoModel.from_pretrained(_A		)
																				self.assertIsInstance(_A				,							_A		)






						def 	_A     (   self		):







													'''simple docstring'''

													try:
																				AutoConfig.register('new-model'				,							_A		)

																				__SCREAMING_SNAKE_CASE									= [
																				    TFAutoModel,
																				    TFAutoModelForCausalLM,
																				    TFAutoModelForMaskedLM,
																				    TFAutoModelForPreTraining,
																				    TFAutoModelForQuestionAnswering,
																				    TFAutoModelForSequenceClassification,
																				    TFAutoModelForTokenClassification,
																				]

																				for auto_class in auto_classes:
																											with self.subTest(auto_class.__name__		):
																																		# Wrong config class will raise an error
																																		with self.assertRaises(_A		):
																																									auto_class.register(_A				,							_A		)
																																		auto_class.register(_A				,							_A		)
																																		# Trying to register something existing in the Transformers library will raise an error
																																		with self.assertRaises(_A		):
																																									auto_class.register(_A				,							_A		)

																																		# Now that the config is registered, it can be used as any other config with the auto-API
																																		__SCREAMING_SNAKE_CASE									= BertModelTester(self		).get_config()
																																		__SCREAMING_SNAKE_CASE									= NewModelConfig(**tiny_config.to_dict()		)
																																		__SCREAMING_SNAKE_CASE									= auto_class.from_config(_A		)
																																		self.assertIsInstance(_A				,							_A		)

																																		with tempfile.TemporaryDirectory() as tmp_dir:
																																									model.save_pretrained(_A		)
																																									__SCREAMING_SNAKE_CASE									= auto_class.from_pretrained(_A		)
																																									self.assertIsInstance(_A				,							_A		)

													finally:
																				if "new-model" in CONFIG_MAPPING._extra_content:
																											del CONFIG_MAPPING._extra_content["new-model"]
																				for mapping in (
																				    TF_MODEL_MAPPING,
																				    TF_MODEL_FOR_PRETRAINING_MAPPING,
																				    TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
																				    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
																				    TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
																				    TF_MODEL_FOR_CAUSAL_LM_MAPPING,
																				    TF_MODEL_FOR_MASKED_LM_MAPPING,
																				):
																											if NewModelConfig in mapping._extra_content:
																																		del mapping._extra_content[NewModelConfig]






						def 	_A     (   self		):







													'''simple docstring'''

													with self.assertRaisesRegex(
													    _A				,							'bert-base is not a local folder and is not a valid model identifier'		):
																				__SCREAMING_SNAKE_CASE									= TFAutoModel.from_pretrained('bert-base'		)






						def 	_A     (   self		):







													'''simple docstring'''

													with self.assertRaisesRegex(
													    _A				,							r'aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'		):
																				__SCREAMING_SNAKE_CASE									= TFAutoModel.from_pretrained(_A				,							revision='aaaaaa'		)






						def 	_A     (   self		):







													'''simple docstring'''

													with self.assertRaisesRegex(
													    _A				,							'hf-internal-testing/config-no-model does not appear to have a file named pytorch_model.bin'				,							):
																				__SCREAMING_SNAKE_CASE									= TFAutoModel.from_pretrained('hf-internal-testing/config-no-model'		)






						def 	_A     (   self		):







													'''simple docstring'''

													with self.assertRaisesRegex(_A				,							'Use `from_pt=True` to load this model'		):
																				__SCREAMING_SNAKE_CASE									= TFAutoModel.from_pretrained('hf-internal-testing/tiny-bert-pt-only'		)






						def 	_A     (   self		):







													'''simple docstring'''

													__SCREAMING_SNAKE_CASE									= TFAutoModel.from_pretrained('hf-internal-testing/tiny-random-bert'		)
													with RequestCounter() as counter:
																				__SCREAMING_SNAKE_CASE									= TFAutoModel.from_pretrained('hf-internal-testing/tiny-random-bert'		)
																				self.assertEqual(counter.get_request_count				,							0		)
																				self.assertEqual(counter.head_request_count				,							1		)
																				self.assertEqual(counter.other_request_count				,							0		)

													# With a sharded checkpoint
													__SCREAMING_SNAKE_CASE									= TFAutoModel.from_pretrained('ArthurZ/tiny-random-bert-sharded'		)
													with RequestCounter() as counter:
																				__SCREAMING_SNAKE_CASE									= TFAutoModel.from_pretrained('ArthurZ/tiny-random-bert-sharded'		)
																				self.assertEqual(counter.get_request_count				,							0		)
																				self.assertEqual(counter.head_request_count				,							1		)
																				self.assertEqual(counter.other_request_count				,							0		)











import pytest

from datasets.parallel import ParallelBackendConfig, parallel_backend
from datasets.utils.py_utils import map_nested

from .utils import require_dill_gt_0_3_2, require_joblibspark, require_not_windows






def      __lowercase       (			a__       )    ->			List[Any]:  # picklable for multiprocessing
							return i + 1






@require_dill_gt_0_3_2
@require_joblibspark
@require_not_windows
def      __lowercase       (			)    ->			Any:
							with parallel_backend('spark'       ):
														assert ParallelBackendConfig.backend_name == "spark"

							__SCREAMING_SNAKE_CASE									= [1, 2, 3]
							with pytest.raises(a__       ):
														with parallel_backend('unsupported backend'       ):
																					map_nested(a__							,       a__							,       num_proc=2       )

							with pytest.raises(a__       ):
														with parallel_backend('unsupported backend'       ):
																					map_nested(a__							,       a__							,       num_proc=-1       )






@require_dill_gt_0_3_2
@require_joblibspark
@require_not_windows
@pytest.mark.parametrize('num_proc'							,       [2, -1]       )
def      __lowercase       (			a__       )    ->			Union[str, Any]:
							__SCREAMING_SNAKE_CASE									= [1, 2]
							__SCREAMING_SNAKE_CASE									= {'a': 1, 'b': 2}
							__SCREAMING_SNAKE_CASE									= {'a': [1, 2], 'b': [3, 4]}
							__SCREAMING_SNAKE_CASE									= {'a': {'1': 1}, 'b': 2}
							__SCREAMING_SNAKE_CASE									= {'a': 1, 'b': 2, 'c': 3, 'd': 4}
							__SCREAMING_SNAKE_CASE									= [2, 3]
							__SCREAMING_SNAKE_CASE									= {'a': 2, 'b': 3}
							__SCREAMING_SNAKE_CASE									= {'a': [2, 3], 'b': [4, 5]}
							__SCREAMING_SNAKE_CASE									= {'a': {'1': 2}, 'b': 3}
							__SCREAMING_SNAKE_CASE									= {'a': 2, 'b': 3, 'c': 4, 'd': 5}

							with parallel_backend('spark'       ):
														assert map_nested(a__							,       a__							,       num_proc=a__       ) == expected_map_nested_sa
														assert map_nested(a__							,       a__							,       num_proc=a__       ) == expected_map_nested_sa
														assert map_nested(a__							,       a__							,       num_proc=a__       ) == expected_map_nested_sa
														assert map_nested(a__							,       a__							,       num_proc=a__       ) == expected_map_nested_sa
														assert map_nested(a__							,       a__							,       num_proc=a__       ) == expected_map_nested_sa






