





from __future__ import annotations

import os
import tempfile
import unittest

from transformers import ConvBertConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
						import tensorflow as tf

						from transformers import (
						    TFConvBertForMaskedLM,
						    TFConvBertForMultipleChoice,
						    TFConvBertForQuestionAnswering,
						    TFConvBertForSequenceClassification,
						    TFConvBertForTokenClassification,
						    TFConvBertModel,
						)
class SCREAMING_SNAKE_CASE_  :






						def __init__( self : Optional[Any]						,				_A : List[str]						,				_A : Any=13						,				_A : Tuple=7						,				_A : Dict=True						,				_A : Optional[int]=True						,				_A : List[Any]=True						,				_A : Optional[int]=True						,				_A : Union[str, Any]=99						,				_A : Dict=32						,				_A : Any=2						,				_A : Any=4						,				_A : Optional[Any]=37						,				_A : Dict="gelu"						,				_A : int=0.1						,				_A : Any=0.1						,				_A : List[str]=512						,				_A : List[Any]=16						,				_A : Dict=2						,				_A : Any=0.0_2						,				_A : List[str]=3						,				_A : int=4						,				_A : str=None						,				)							->      int:







							"""simple docstring"""


							snake_case_						:     Optional[Any]     =		parent
							snake_case_						:     List[str]     =		13
							snake_case_						:     List[Any]     =		7
							snake_case_						:     Dict     =		True
							snake_case_						:     str     =		True
							snake_case_						:     int     =		True
							snake_case_						:     Optional[int]     =		True
							snake_case_						:     Optional[int]     =		99
							snake_case_						:     Optional[Any]     =		384
							snake_case_						:     str     =		2
							snake_case_						:     Optional[Any]     =		4
							snake_case_						:     Any     =		37
							snake_case_						:     Optional[Any]     =		'gelu'
							snake_case_						:     str     =		0.1
							snake_case_						:     str     =		0.1
							snake_case_						:     List[str]     =		512
							snake_case_						:     Union[str, Any]     =		16
							snake_case_						:     str     =		2
							snake_case_						:     Tuple     =		0.0_2
							snake_case_						:     List[Any]     =		3
							snake_case_						:     int     =		4
							snake_case_						:     List[Any]     =		128
							snake_case_						:     Union[str, Any]     =		2
							snake_case_						:     Optional[int]     =		9
							snake_case_						:     List[str]     =		1
							snake_case_						:     Union[str, Any]     =		None






						def UpperCAmelCase_			( self : Dict		)							->      Any:







							"""simple docstring"""


							snake_case_						:     Tuple     =		ids_tensor([self.batch_size, self.seq_length]						,				self.vocab_size		)

							snake_case_						:     Optional[int]     =		None
							if self.use_input_mask:
								snake_case_						:     List[Any]     =		random_attention_mask([self.batch_size, self.seq_length]		)

							snake_case_						:     str     =		None
							if self.use_token_type_ids:
								snake_case_						:     Optional[int]     =		ids_tensor([self.batch_size, self.seq_length]						,				self.type_vocab_size		)

							snake_case_						:     int     =		None
							snake_case_						:     Union[str, Any]     =		None
							snake_case_						:     str     =		None
							if self.use_labels:
								snake_case_						:     Optional[int]     =		ids_tensor([self.batch_size]						,				self.type_sequence_label_size		)
								snake_case_						:     Any     =		ids_tensor([self.batch_size, self.seq_length]						,				self.num_labels		)
								snake_case_						:     int     =		ids_tensor([self.batch_size]						,				self.num_choices		)

							snake_case_						:     int     =		ConvBertConfig(
							    vocab_size=self.vocab_size						,				hidden_size=self.hidden_size						,				num_hidden_layers=self.num_hidden_layers						,				num_attention_heads=self.num_attention_heads						,				intermediate_size=self.intermediate_size						,				hidden_act=self.hidden_act						,				hidden_dropout_prob=self.hidden_dropout_prob						,				attention_probs_dropout_prob=self.attention_probs_dropout_prob						,				max_position_embeddings=self.max_position_embeddings						,				type_vocab_size=self.type_vocab_size						,				initializer_range=self.initializer_range						,				return_dict=_A						,				)

							return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






						def UpperCAmelCase_			( self : str						,				_A : List[Any]						,				_A : List[str]						,				_A : Optional[int]						,				_A : Union[str, Any]						,				_A : Optional[int]						,				_A : Union[str, Any]						,				_A : str		)							->      Union[str, Any]:







							"""simple docstring"""


							snake_case_						:     List[str]     =		TFConvBertModel(config=_A		)
							snake_case_						:     Optional[int]     =		{'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}

							snake_case_						:     Tuple     =		[input_ids, input_mask]
							snake_case_						:     List[str]     =		model(_A		)

							snake_case_						:     Optional[Any]     =		model(_A		)

							self.parent.assertEqual(result.last_hidden_state.shape						,				(self.batch_size, self.seq_length, self.hidden_size)		)






						def UpperCAmelCase_			( self : Union[str, Any]						,				_A : int						,				_A : Dict						,				_A : int						,				_A : Dict						,				_A : List[Any]						,				_A : List[Any]						,				_A : Dict		)							->      List[str]:







							"""simple docstring"""


							snake_case_						:     List[str]     =		TFConvBertForMaskedLM(config=_A		)
							snake_case_						:     Optional[int]     =		{
							    'input_ids': input_ids,
							    'attention_mask': input_mask,
							    'token_type_ids': token_type_ids,
							}
							snake_case_						:     int     =		model(_A		)
							self.parent.assertEqual(result.logits.shape						,				(self.batch_size, self.seq_length, self.vocab_size)		)






						def UpperCAmelCase_			( self : str						,				_A : Any						,				_A : Any						,				_A : List[Any]						,				_A : Any						,				_A : Tuple						,				_A : Dict						,				_A : List[str]		)							->      List[Any]:







							"""simple docstring"""


							snake_case_						:     Optional[Any]     =		self.num_labels
							snake_case_						:     Union[str, Any]     =		TFConvBertForSequenceClassification(config=_A		)
							snake_case_						:     Any     =		{
							    'input_ids': input_ids,
							    'attention_mask': input_mask,
							    'token_type_ids': token_type_ids,
							}

							snake_case_						:     int     =		model(_A		)
							self.parent.assertEqual(result.logits.shape						,				(self.batch_size, self.num_labels)		)






						def UpperCAmelCase_			( self : Optional[int]						,				_A : Dict						,				_A : Tuple						,				_A : Union[str, Any]						,				_A : int						,				_A : Optional[int]						,				_A : int						,				_A : int		)							->      Any:







							"""simple docstring"""


							snake_case_						:     Optional[int]     =		self.num_choices
							snake_case_						:     Tuple     =		TFConvBertForMultipleChoice(config=_A		)
							snake_case_						:     Union[str, Any]     =		tf.tile(tf.expand_dims(_A						,				1		)						,				(1, self.num_choices, 1)		)
							snake_case_						:     Optional[int]     =		tf.tile(tf.expand_dims(_A						,				1		)						,				(1, self.num_choices, 1)		)
							snake_case_						:     Tuple     =		tf.tile(tf.expand_dims(_A						,				1		)						,				(1, self.num_choices, 1)		)
							snake_case_						:     List[str]     =		{
							    'input_ids': multiple_choice_inputs_ids,
							    'attention_mask': multiple_choice_input_mask,
							    'token_type_ids': multiple_choice_token_type_ids,
							}
							snake_case_						:     Optional[Any]     =		model(_A		)
							self.parent.assertEqual(result.logits.shape						,				(self.batch_size, self.num_choices)		)






						def UpperCAmelCase_			( self : int						,				_A : List[str]						,				_A : List[Any]						,				_A : Tuple						,				_A : Optional[Any]						,				_A : List[Any]						,				_A : Any						,				_A : Optional[int]		)							->      int:







							"""simple docstring"""


							snake_case_						:     List[Any]     =		self.num_labels
							snake_case_						:     Dict     =		TFConvBertForTokenClassification(config=_A		)
							snake_case_						:     int     =		{
							    'input_ids': input_ids,
							    'attention_mask': input_mask,
							    'token_type_ids': token_type_ids,
							}
							snake_case_						:     Tuple     =		model(_A		)
							self.parent.assertEqual(result.logits.shape						,				(self.batch_size, self.seq_length, self.num_labels)		)






						def UpperCAmelCase_			( self : List[Any]						,				_A : List[Any]						,				_A : Tuple						,				_A : int						,				_A : List[Any]						,				_A : str						,				_A : List[str]						,				_A : Tuple		)							->      Optional[int]:







							"""simple docstring"""


							snake_case_						:     str     =		TFConvBertForQuestionAnswering(config=_A		)
							snake_case_						:     str     =		{
							    'input_ids': input_ids,
							    'attention_mask': input_mask,
							    'token_type_ids': token_type_ids,
							}

							snake_case_						:     Tuple     =		model(_A		)
							self.parent.assertEqual(result.start_logits.shape						,				(self.batch_size, self.seq_length)		)
							self.parent.assertEqual(result.end_logits.shape						,				(self.batch_size, self.seq_length)		)






						def UpperCAmelCase_			( self : Optional[Any]		)							->      str:







							"""simple docstring"""


							snake_case_						:     Optional[Any]     =		self.prepare_config_and_inputs()
							(
							    (
							    snake_case_
							)  ,(
							    snake_case_
							)  ,(
							    snake_case_
							)  ,(
							    snake_case_
							)  ,(
							    snake_case_
							)  ,(
							    snake_case_
							)  ,(
							    snake_case_
							)  ,
							)						:     Optional[Any]     =		config_and_inputs
							snake_case_						:     Union[str, Any]     =		{'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
							return config, inputs_dict
@require_tf
class SCREAMING_SNAKE_CASE_  (   snake_case_    ,			snake_case_    ,			unittest.TestCase   ):
						__magic_name__:     Union[str, Any]	 =    (
						    (
						        TFConvBertModel,
						        TFConvBertForMaskedLM,
						        TFConvBertForQuestionAnswering,
						        TFConvBertForSequenceClassification,
						        TFConvBertForTokenClassification,
						        TFConvBertForMultipleChoice,
						    )
						    if is_tf_available()
						    else ()
						)
						__magic_name__:     List[str]	 =    (
						    {
						        "feature-extraction": TFConvBertModel,
						        "fill-mask": TFConvBertForMaskedLM,
						        "question-answering": TFConvBertForQuestionAnswering,
						        "text-classification": TFConvBertForSequenceClassification,
						        "token-classification": TFConvBertForTokenClassification,
						        "zero-shot": TFConvBertForSequenceClassification,
						    }
						    if is_tf_available()
						    else {}
						)
						__magic_name__:     Optional[int]	 =    False
						__magic_name__:     List[str]	 =    False
						__magic_name__:     List[str]	 =    False






						def UpperCAmelCase_			( self : Optional[Any]		)							->      Union[str, Any]:







							"""simple docstring"""


							snake_case_						:     Tuple     =		TFConvBertModelTester(self		)
							snake_case_						:     List[str]     =		ConfigTester(self						,				config_class=_A						,				hidden_size=37		)






						def UpperCAmelCase_			( self : str		)							->      List[str]:







							"""simple docstring"""


							self.config_tester.run_common_tests()






						def UpperCAmelCase_			( self : int		)							->      List[Any]:







							"""simple docstring"""


							snake_case_						:     Union[str, Any]     =		self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_model(*_A		)






						def UpperCAmelCase_			( self : List[str]		)							->      Union[str, Any]:







							"""simple docstring"""


							snake_case_						:     Dict     =		self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_masked_lm(*_A		)






						def UpperCAmelCase_			( self : List[Any]		)							->      int:







							"""simple docstring"""


							snake_case_						:     List[Any]     =		self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_multiple_choice(*_A		)






						def UpperCAmelCase_			( self : int		)							->      List[Any]:







							"""simple docstring"""


							snake_case_						:     Tuple     =		self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_question_answering(*_A		)






						def UpperCAmelCase_			( self : List[str]		)							->      Optional[Any]:







							"""simple docstring"""


							snake_case_						:     int     =		self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_sequence_classification(*_A		)






						def UpperCAmelCase_			( self : List[Any]		)							->      Tuple:







							"""simple docstring"""


							snake_case_						:     Union[str, Any]     =		self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_token_classification(*_A		)






						@slow
						def UpperCAmelCase_			( self : Optional[int]		)							->      Any:







							"""simple docstring"""


							snake_case_  ,snake_case_						:     int     =		self.model_tester.prepare_config_and_inputs_for_common()
							snake_case_						:     Dict     =		True
							snake_case_						:     Optional[Any]     =		True

							if hasattr(_A						,				'use_cache'		):
								snake_case_						:     Optional[Any]     =		True

							snake_case_						:     Any     =		getattr(self.model_tester						,				'encoder_seq_length'						,				self.model_tester.seq_length		)
							snake_case_						:     int     =		getattr(self.model_tester						,				'key_length'						,				_A		)

							for model_class in self.all_model_classes:
								snake_case_						:     Optional[Any]     =		self._prepare_for_class(_A						,				_A		)
								snake_case_						:     Tuple     =		model_class(_A		)
								snake_case_						:     Any     =		len(model(_A		)		)

								with tempfile.TemporaryDirectory() as tmpdirname:
									model.save_pretrained(_A						,				saved_model=_A		)
									snake_case_						:     int     =		os.path.join(_A						,				'saved_model'						,				'1'		)
									snake_case_						:     int     =		tf.keras.models.load_model(_A		)
									snake_case_						:     Any     =		model(_A		)

									if self.is_encoder_decoder:
										snake_case_						:     Optional[Any]     =		outputs['encoder_hidden_states']
										snake_case_						:     List[str]     =		outputs['encoder_attentions']
									else:
										snake_case_						:     str     =		outputs['hidden_states']
										snake_case_						:     Any     =		outputs['attentions']

									self.assertEqual(len(_A		)						,				_A		)

									snake_case_						:     Dict     =		getattr(
									    self.model_tester						,				'expected_num_hidden_layers'						,				self.model_tester.num_hidden_layers + 1		)

									self.assertEqual(len(_A		)						,				_A		)
									self.assertListEqual(
									    list(output_hidden_states[0].shape[-2:]		)						,				[self.model_tester.seq_length, self.model_tester.hidden_size]						,				)

									self.assertEqual(len(_A		)						,				self.model_tester.num_hidden_layers		)
									self.assertListEqual(
									    list(output_attentions[0].shape[-3:]		)						,				[self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length]						,				)






						@slow
						def UpperCAmelCase_			( self : Any		)							->      Union[str, Any]:







							"""simple docstring"""


							snake_case_						:     int     =		TFConvBertModel.from_pretrained('YituTech/conv-bert-base'		)
							self.assertIsNotNone(_A		)






						def UpperCAmelCase_			( self : List[Any]		)							->      List[str]:







							"""simple docstring"""


							snake_case_  ,snake_case_						:     int     =		self.model_tester.prepare_config_and_inputs_for_common()
							snake_case_						:     Optional[int]     =		True
							snake_case_						:     Dict     =		getattr(self.model_tester						,				'decoder_seq_length'						,				self.model_tester.seq_length		)
							snake_case_						:     Dict     =		getattr(self.model_tester						,				'encoder_seq_length'						,				self.model_tester.seq_length		)
							snake_case_						:     Tuple     =		getattr(self.model_tester						,				'key_length'						,				_A		)
							snake_case_						:     Tuple     =		getattr(self.model_tester						,				'key_length'						,				_A		)

							def check_decoder_attentions_output(_A : Optional[int]		):
								snake_case_						:     str     =		len(_A		)
								self.assertEqual(out_len % 2						,				0		)
								snake_case_						:     List[Any]     =		outputs.decoder_attentions
								self.assertEqual(len(_A		)						,				self.model_tester.num_hidden_layers		)
								self.assertListEqual(
								    list(decoder_attentions[0].shape[-3:]		)						,				[self.model_tester.num_attention_heads / 2, decoder_seq_length, decoder_key_length]						,				)

							def check_encoder_attentions_output(_A : List[Any]		):
								snake_case_						:     Any     =		[
								    t.numpy() for t in (outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions)
								]
								self.assertEqual(len(_A		)						,				self.model_tester.num_hidden_layers		)
								self.assertListEqual(
								    list(attentions[0].shape[-3:]		)						,				[self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length]						,				)

							for model_class in self.all_model_classes:
								snake_case_						:     Tuple     =		True
								snake_case_						:     List[Any]     =		False
								snake_case_						:     Tuple     =		model_class(_A		)
								snake_case_						:     List[Any]     =		model(self._prepare_for_class(_A						,				_A		)		)
								snake_case_						:     Union[str, Any]     =		len(_A		)
								self.assertEqual(config.output_hidden_states						,				_A		)
								check_encoder_attentions_output(_A		)

								if self.is_encoder_decoder:
									snake_case_						:     Union[str, Any]     =		model_class(_A		)
									snake_case_						:     str     =		model(self._prepare_for_class(_A						,				_A		)		)
									self.assertEqual(config.output_hidden_states						,				_A		)
									check_decoder_attentions_output(_A		)

								# Check that output attentions can also be changed via the config
								del inputs_dict["output_attentions"]
								snake_case_						:     Optional[Any]     =		True
								snake_case_						:     Tuple     =		model_class(_A		)
								snake_case_						:     Dict     =		model(self._prepare_for_class(_A						,				_A		)		)
								self.assertEqual(config.output_hidden_states						,				_A		)
								check_encoder_attentions_output(_A		)

								# Check attention is always last and order is fine
								snake_case_						:     Dict     =		True
								snake_case_						:     str     =		True
								snake_case_						:     int     =		model_class(_A		)
								snake_case_						:     str     =		model(self._prepare_for_class(_A						,				_A		)		)

								self.assertEqual(out_len + (2 if self.is_encoder_decoder else 1)						,				len(_A		)		)
								self.assertEqual(model.config.output_hidden_states						,				_A		)
								check_encoder_attentions_output(_A		)
@require_tf
class SCREAMING_SNAKE_CASE_  (   unittest.TestCase   ):






						@slow
						def UpperCAmelCase_			( self : List[Any]		)							->      Tuple:







							"""simple docstring"""


							snake_case_						:     Optional[int]     =		TFConvBertModel.from_pretrained('YituTech/conv-bert-base'		)
							snake_case_						:     Optional[Any]     =		tf.constant([[0, 1, 2, 3, 4, 5]]		)
							snake_case_						:     str     =		model(_A		)[0]

							snake_case_						:     str     =		[1, 6, 768]
							self.assertEqual(output.shape						,				_A		)

							snake_case_						:     int     =		tf.constant(
							    [
							        [
							            [-0.0_3_4_7_5_4_9_3, -0.4_6_8_6_0_3_4, -0.3_0_6_3_8_8_3_2],
							            [0.2_2_6_3_7_2_4_8, -0.2_6_9_8_8_6_4_6, -0.7_4_2_3_4_2_4],
							            [0.1_0_3_2_4_8_6_8, -0.4_5_0_1_3_5_0_8, -0.5_8_2_8_0_7_8_4],
							        ]
							    ]		)
							tf.debugging.assert_near(output[:, :3, :3]						,				_A						,				atol=1E-4		)








import torch
from torch import nn

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin
class SCREAMING_SNAKE_CASE_  (   snake_case_    ,			snake_case_   ):






						@register_to_config
						def __init__( self : Optional[Any]						,				*,
						    _A : int = 4						,				_A : int = 768						,				_A : int						,				_A : Tuple						,				)							->      Dict:







							"""simple docstring"""


							super().__init__()

							snake_case_						:     int     =		nn.Parameter(torch.zeros(_A		)		)

							# parameters for additional clip time embeddings
							snake_case_						:     Tuple     =		nn.Linear(_A						,				_A		)
							snake_case_						:     List[Any]     =		nn.Linear(_A						,				_A		)

							# parameters for encoder hidden states
							snake_case_						:     Union[str, Any]     =		clip_extra_context_tokens
							snake_case_						:     str     =		nn.Linear(
							    _A						,				self.clip_extra_context_tokens * cross_attention_dim		)
							snake_case_						:     Any     =		nn.Linear(_A						,				_A		)
							snake_case_						:     Tuple     =		nn.LayerNorm(_A		)






						def UpperCAmelCase_			( self : List[str]						,				*, _A : Tuple						,				_A : List[Any]						,				_A : str						,				_A : Optional[Any]		)							->      List[Any]:







							"""simple docstring"""


							if do_classifier_free_guidance:
								# Add the classifier free guidance embeddings to the image embeddings
								snake_case_						:     Optional[int]     =		image_embeddings.shape[0]
								snake_case_						:     Optional[Any]     =		self.learned_classifier_free_guidance_embeddings.unsqueeze(0		)
								snake_case_						:     Optional[Any]     =		classifier_free_guidance_embeddings.expand(
								    _A						,				-1		)
								snake_case_						:     Any     =		torch.cat([classifier_free_guidance_embeddings, image_embeddings]						,				dim=0		)

							# The image embeddings batch size and the text embeddings batch size are equal
							assert image_embeddings.shape[0] == prompt_embeds.shape[0]

							snake_case_						:     str     =		prompt_embeds.shape[0]

							# "Specifically, we modify the architecture described in Nichol et al. (2021) by projecting and
							# adding CLIP embeddings to the existing timestep embedding, ...
							snake_case_						:     str     =		self.embedding_proj(_A		)
							snake_case_						:     Dict     =		self.clip_image_embeddings_project_to_time_embeddings(_A		)
							snake_case_						:     Tuple     =		time_projected_image_embeddings + time_projected_prompt_embeds

							# ... and by projecting CLIP embeddings into four
							# extra tokens of context that are concatenated to the sequence of outputs from the GLIDE text encoder"
							snake_case_						:     List[str]     =		self.clip_extra_context_tokens_proj(_A		)
							snake_case_						:     Optional[Any]     =		clip_extra_context_tokens.reshape(_A						,				-1						,				self.clip_extra_context_tokens		)
							snake_case_						:     int     =		clip_extra_context_tokens.permute(0						,				2						,				1		)

							snake_case_						:     Optional[int]     =		self.encoder_hidden_states_proj(_A		)
							snake_case_						:     Any     =		self.text_encoder_hidden_states_norm(_A		)
							snake_case_						:     Dict     =		torch.cat([clip_extra_context_tokens, text_encoder_hidden_states]						,				dim=1		)

							return text_encoder_hidden_states, additive_clip_time_embeddings

