


from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
	import numpy
	import tensorflow as tf

	from transformers import (
	    TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
	    TF_DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
	    TF_DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST,
	    BertConfig,
	    DPRConfig,
	    TFDPRContextEncoder,
	    TFDPRQuestionEncoder,
	    TFDPRReader,
	)



class 							A   :
					'''simple docstring'''






					def __init__(self   :						str				, _UpperCAmelCase   :						Dict				, _UpperCAmelCase   :						Optional[int]=13				, _UpperCAmelCase   :						List[Any]=7				, _UpperCAmelCase   :						Union[str, Any]=True				, _UpperCAmelCase   :						Union[str, Any]=True				, _UpperCAmelCase   :						Dict=True				, _UpperCAmelCase   :						Any=True				, _UpperCAmelCase   :						Optional[Any]=99				, _UpperCAmelCase   :						Any=32				, _UpperCAmelCase   :						Optional[int]=2				, _UpperCAmelCase   :						Optional[int]=4				, _UpperCAmelCase   :						Any=37				, _UpperCAmelCase   :						int="gelu"				, _UpperCAmelCase   :						int=0.1				, _UpperCAmelCase   :						Optional[Any]=0.1				, _UpperCAmelCase   :						int=512				, _UpperCAmelCase   :						Any=16				, _UpperCAmelCase   :						Any=2				, _UpperCAmelCase   :						Union[str, Any]=0.02				, _UpperCAmelCase   :						Tuple=3				, _UpperCAmelCase   :						Tuple=4				, _UpperCAmelCase   :						List[str]=None				, _UpperCAmelCase   :						Optional[Any]=0				, )						->       Tuple:




							"""simple docstring"""


							lowercase__					      =		parent
							lowercase__					      =		batch_size
							lowercase__					      =		seq_length
							lowercase__					      =		is_training
							lowercase__					      =		use_input_mask
							lowercase__					      =		use_token_type_ids
							lowercase__					      =		use_labels
							lowercase__					      =		vocab_size
							lowercase__					      =		hidden_size
							lowercase__					      =		num_hidden_layers
							lowercase__					      =		num_attention_heads
							lowercase__					      =		intermediate_size
							lowercase__					      =		hidden_act
							lowercase__					      =		hidden_dropout_prob
							lowercase__					      =		attention_probs_dropout_prob
							lowercase__					      =		max_position_embeddings
							lowercase__					      =		type_vocab_size
							lowercase__					      =		type_sequence_label_size
							lowercase__					      =		initializer_range
							lowercase__					      =		num_labels
							lowercase__					      =		num_choices
							lowercase__					      =		scope
							lowercase__					      =		projection_dim

					def   lowerCamelCase__					(self   :						List[Any]		)						->       List[str]:




							"""simple docstring"""


							lowercase__					      =		ids_tensor([self.batch_size, self.seq_length]				, self.vocab_size		)

							lowercase__					      =		None
							if self.use_input_mask:
									# follow test_modeling_tf_ctrl.py
									lowercase__					      =		random_attention_mask([self.batch_size, self.seq_length]		)

							lowercase__					      =		None
							if self.use_token_type_ids:
									lowercase__					      =		ids_tensor([self.batch_size, self.seq_length]				, self.type_vocab_size		)

							lowercase__					      =		None
							lowercase__					      =		None
							lowercase__					      =		None
							if self.use_labels:
									lowercase__					      =		ids_tensor([self.batch_size]				, self.type_sequence_label_size		)
									lowercase__					      =		ids_tensor([self.batch_size, self.seq_length]				, self.num_labels		)
									lowercase__					      =		ids_tensor([self.batch_size]				, self.num_choices		)

							lowercase__					      =		BertConfig(
							    vocab_size=self.vocab_size				, hidden_size=self.hidden_size				, num_hidden_layers=self.num_hidden_layers				, num_attention_heads=self.num_attention_heads				, intermediate_size=self.intermediate_size				, hidden_act=self.hidden_act				, hidden_dropout_prob=self.hidden_dropout_prob				, attention_probs_dropout_prob=self.attention_probs_dropout_prob				, max_position_embeddings=self.max_position_embeddings				, type_vocab_size=self.type_vocab_size				, is_decoder=_UpperCAmelCase				, initializer_range=self.initializer_range				, )
							lowercase__					      =		DPRConfig(projection_dim=self.projection_dim				, **config.to_dict()		)

							return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels

					def   lowerCamelCase__					(self   :						List[str]				, _UpperCAmelCase   :						Optional[Any]				, _UpperCAmelCase   :						Tuple				, _UpperCAmelCase   :						int				, _UpperCAmelCase   :						Dict				, _UpperCAmelCase   :						Optional[int]				, _UpperCAmelCase   :						int				, _UpperCAmelCase   :						List[str]		)						->       Optional[Any]:




							"""simple docstring"""


							lowercase__					      =		TFDPRContextEncoder(config=_UpperCAmelCase		)
							lowercase__					      =		model(_UpperCAmelCase				, attention_mask=_UpperCAmelCase				, token_type_ids=_UpperCAmelCase		)
							lowercase__					      =		model(_UpperCAmelCase				, token_type_ids=_UpperCAmelCase		)
							lowercase__					      =		model(_UpperCAmelCase		)
							self.parent.assertEqual(result.pooler_output.shape				, (self.batch_size, self.projection_dim or self.hidden_size)		)

					def   lowerCamelCase__					(self   :						List[str]				, _UpperCAmelCase   :						Dict				, _UpperCAmelCase   :						Tuple				, _UpperCAmelCase   :						List[str]				, _UpperCAmelCase   :						Tuple				, _UpperCAmelCase   :						Optional[int]				, _UpperCAmelCase   :						Union[str, Any]				, _UpperCAmelCase   :						List[Any]		)						->       Optional[int]:




							"""simple docstring"""


							lowercase__					      =		TFDPRQuestionEncoder(config=_UpperCAmelCase		)
							lowercase__					      =		model(_UpperCAmelCase				, attention_mask=_UpperCAmelCase				, token_type_ids=_UpperCAmelCase		)
							lowercase__					      =		model(_UpperCAmelCase				, token_type_ids=_UpperCAmelCase		)
							lowercase__					      =		model(_UpperCAmelCase		)
							self.parent.assertEqual(result.pooler_output.shape				, (self.batch_size, self.projection_dim or self.hidden_size)		)

					def   lowerCamelCase__					(self   :						Union[str, Any]				, _UpperCAmelCase   :						Optional[int]				, _UpperCAmelCase   :						Optional[int]				, _UpperCAmelCase   :						List[str]				, _UpperCAmelCase   :						List[Any]				, _UpperCAmelCase   :						str				, _UpperCAmelCase   :						Dict				, _UpperCAmelCase   :						Any		)						->       Tuple:




							"""simple docstring"""


							lowercase__					      =		TFDPRReader(config=_UpperCAmelCase		)
							lowercase__					      =		model(_UpperCAmelCase				, attention_mask=_UpperCAmelCase		)

							self.parent.assertEqual(result.start_logits.shape				, (self.batch_size, self.seq_length)		)
							self.parent.assertEqual(result.end_logits.shape				, (self.batch_size, self.seq_length)		)
							self.parent.assertEqual(result.relevance_logits.shape				, (self.batch_size,)		)


					def   lowerCamelCase__					(self   :						Any		)						->       Tuple:




							"""simple docstring"""


							lowercase__					      =		self.prepare_config_and_inputs()
							(
							    (
							    lowercase__
							)      ,     (
							    lowercase__
							)      ,     (
							    lowercase__
							)      ,     (
							    lowercase__
							)      ,     (
							    lowercase__
							)      ,     (
							    lowercase__
							)      ,     (
							    lowercase__
							)      ,     
							)					      =		config_and_inputs
							lowercase__					      =		{"""input_ids""": input_ids}
							return config, inputs_dict






@require_tf
class 							A   (	UpperCAmelCase__			,  UpperCAmelCase__			,  unittest.TestCase							):
					'''simple docstring'''






					A__      =			(
					    (
					        TFDPRContextEncoder,
					        TFDPRQuestionEncoder,
					        TFDPRReader,
					    )
					    if is_tf_available()
					    else ()
					)
					A__      =			{'''feature-extraction''': TFDPRQuestionEncoder} if is_tf_available() else {}

					A__      =			False
					A__      =			False
					A__      =			False
					A__      =			False
					A__      =			False

					def   lowerCamelCase__					(self   :						Optional[Any]		)						->       int:




							"""simple docstring"""


							lowercase__					      =		TFDPRModelTester(self		)
							lowercase__					      =		ConfigTester(self				, config_class=_UpperCAmelCase				, hidden_size=37		)

					def   lowerCamelCase__					(self   :						str		)						->       str:




							"""simple docstring"""


							self.config_tester.run_common_tests()

					def   lowerCamelCase__					(self   :						Optional[int]		)						->       Any:




							"""simple docstring"""


							lowercase__					      =		self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_dpr_context_encoder(*_UpperCAmelCase		)

					def   lowerCamelCase__					(self   :						Dict		)						->       Optional[Any]:




							"""simple docstring"""


							lowercase__					      =		self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_dpr_question_encoder(*_UpperCAmelCase		)

					def   lowerCamelCase__					(self   :						int		)						->       List[str]:




							"""simple docstring"""


							lowercase__					      =		self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_dpr_reader(*_UpperCAmelCase		)


					@slow
					def   lowerCamelCase__					(self   :						Union[str, Any]		)						->       str:




							"""simple docstring"""


							for model_name in TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
									lowercase__					      =		TFDPRContextEncoder.from_pretrained(_UpperCAmelCase		)
									self.assertIsNotNone(_UpperCAmelCase		)

							for model_name in TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
									lowercase__					      =		TFDPRContextEncoder.from_pretrained(_UpperCAmelCase		)
									self.assertIsNotNone(_UpperCAmelCase		)

							for model_name in TF_DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
									lowercase__					      =		TFDPRQuestionEncoder.from_pretrained(_UpperCAmelCase		)
									self.assertIsNotNone(_UpperCAmelCase		)

							for model_name in TF_DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
									lowercase__					      =		TFDPRReader.from_pretrained(_UpperCAmelCase		)
									self.assertIsNotNone(_UpperCAmelCase		)






@require_tf
class 							A   (	unittest.TestCase							):
					'''simple docstring'''






					@slow
					def   lowerCamelCase__					(self   :						int		)						->       str:




							"""simple docstring"""


							lowercase__					      =		TFDPRQuestionEncoder.from_pretrained("""facebook/dpr-question_encoder-single-nq-base"""		)

							lowercase__					      =		tf.constant(
							    [[101, 7592, 1010, 2003, 2026, 3899, 1_0140, 1029, 102]]		)  # [CLS] hello, is my dog cute? [SEP]
							lowercase__					      =		model(_UpperCAmelCase		)[0]  # embedding shape = (1, 768)
							# compare the actual values for a slice.
							lowercase__					      =		tf.constant(
							    [
							        [
							            0.03_236_253,
							            0.12_753_335,
							            0.16_818_509,
							            0.00_279_786,
							            0.3_896_933,
							            0.24_264_945,
							            0.2_178_971,
							            -0.02_335_227,
							            -0.08_481_959,
							            -0.14_324_117,
							        ]
							    ]		)
							self.assertTrue(numpy.allclose(output[:, :10].numpy()				, expected_slice.numpy()				, atol=1E-4		)		)





import json
import os
import unittest

from transformers import OpenAIGPTTokenizer, OpenAIGPTTokenizerFast
from transformers.models.openai.tokenization_openai import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_spacy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class 							A   (	UpperCAmelCase__			,  unittest.TestCase							):
					'''simple docstring'''






					A__      =			OpenAIGPTTokenizer
					A__      =			OpenAIGPTTokenizerFast
					A__      =			True
					A__      =			False

					def   lowerCamelCase__					(self   :						List[str]		)						->       int:




							"""simple docstring"""


							super().setUp()

							# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
							lowercase__					      =		[
							    """l""",
							    """o""",
							    """w""",
							    """e""",
							    """r""",
							    """s""",
							    """t""",
							    """i""",
							    """d""",
							    """n""",
							    """w</w>""",
							    """r</w>""",
							    """t</w>""",
							    """lo""",
							    """low""",
							    """er</w>""",
							    """low</w>""",
							    """lowest</w>""",
							    """newer</w>""",
							    """wider</w>""",
							    """<unk>""",
							]
							lowercase__					      =		dict(zip(_UpperCAmelCase				, range(len(_UpperCAmelCase		)		)		)		)
							lowercase__					      =		["""#version: 0.2""", """l o""", """lo w""", """e r</w>""", """"""]

							lowercase__					      =		os.path.join(self.tmpdirname				, VOCAB_FILES_NAMES["""vocab_file"""]		)
							lowercase__					      =		os.path.join(self.tmpdirname				, VOCAB_FILES_NAMES["""merges_file"""]		)
							with open(self.vocab_file				, """w"""		) as fp:
									fp.write(json.dumps(_UpperCAmelCase		)		)
							with open(self.merges_file				, """w"""		) as fp:
									fp.write("""\n""".join(_UpperCAmelCase		)		)

					def   lowerCamelCase__					(self   :						Dict				, _UpperCAmelCase   :						List[Any]		)						->       Any:




							"""simple docstring"""


							return "lower newer", "lower newer"

					def   lowerCamelCase__					(self   :						Dict		)						->       Tuple:




							"""simple docstring"""


							lowercase__					      =		OpenAIGPTTokenizer(self.vocab_file				, self.merges_file		)

							lowercase__					      =		"""lower"""
							lowercase__					      =		["""low""", """er</w>"""]
							lowercase__					      =		tokenizer.tokenize(_UpperCAmelCase		)
							self.assertListEqual(_UpperCAmelCase				, _UpperCAmelCase		)

							lowercase__					      =		tokens + ["""<unk>"""]
							lowercase__					      =		[14, 15, 20]
							self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCAmelCase		)				, _UpperCAmelCase		)

					def   lowerCamelCase__					(self   :						Optional[int]				, _UpperCAmelCase   :						Union[str, Any]=15		)						->       Optional[int]:




							"""simple docstring"""


							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''		):
											lowercase__					      =		self.rust_tokenizer_class.from_pretrained(_UpperCAmelCase				, **_UpperCAmelCase		)

											# Simple input
											lowercase__					      =		"""This is a simple input"""
											lowercase__					      =		["""This is a simple input 1""", """This is a simple input 2"""]
											lowercase__					      =		("""This is a simple input""", """This is a pair""")
											lowercase__					      =		[
											    ("""This is a simple input 1""", """This is a simple input 2"""),
											    ("""This is a simple pair 1""", """This is a simple pair 2"""),
											]

											# Simple input tests
											self.assertRaises(_UpperCAmelCase				, tokenizer_r.encode				, _UpperCAmelCase				, max_length=_UpperCAmelCase				, padding="""max_length"""		)

											# Simple input
											self.assertRaises(_UpperCAmelCase				, tokenizer_r.encode_plus				, _UpperCAmelCase				, max_length=_UpperCAmelCase				, padding="""max_length"""		)

											# Simple input
											self.assertRaises(
											    _UpperCAmelCase				, tokenizer_r.batch_encode_plus				, _UpperCAmelCase				, max_length=_UpperCAmelCase				, padding="""max_length"""				, )

											# Pair input
											self.assertRaises(_UpperCAmelCase				, tokenizer_r.encode				, _UpperCAmelCase				, max_length=_UpperCAmelCase				, padding="""max_length"""		)

											# Pair input
											self.assertRaises(_UpperCAmelCase				, tokenizer_r.encode_plus				, _UpperCAmelCase				, max_length=_UpperCAmelCase				, padding="""max_length"""		)

											# Pair input
											self.assertRaises(
											    _UpperCAmelCase				, tokenizer_r.batch_encode_plus				, _UpperCAmelCase				, max_length=_UpperCAmelCase				, padding="""max_length"""				, )


					def   lowerCamelCase__					(self   :						Tuple		)						->       Any:




							"""simple docstring"""


							pass






@require_ftfy
@require_spacy
@require_tokenizers
class 							A   (	UpperCAmelCase__							):
					'''simple docstring'''






					pass

