





"""simple docstring"""




import unittest

from transformers import AutoTokenizer, FalconConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
						import torch

						from transformers import (
						    FalconForCausalLM,
						    FalconForQuestionAnswering,
						    FalconForSequenceClassification,
						    FalconForTokenClassification,
						    FalconModel,
						)




class 			_a						:






				def __init__(			self		:   str,			lowerCAmelCase__		:   Dict,			lowerCAmelCase__		:   str=3,			lowerCAmelCase__		:   List[str]=7,			lowerCAmelCase__		:   int=True,			lowerCAmelCase__		:   Optional[Any]=True,			lowerCAmelCase__		:   int=False,			lowerCAmelCase__		:   Dict=True,			lowerCAmelCase__		:   int=9_9,			lowerCAmelCase__		:   Tuple=3_2,			lowerCAmelCase__		:   Any=5,			lowerCAmelCase__		:   str=4,			lowerCAmelCase__		:   int=3_7,			lowerCAmelCase__		:   Union[str, Any]="gelu",			lowerCAmelCase__		:   Dict=0.1,			lowerCAmelCase__		:   str=0.1,			lowerCAmelCase__		:   str=5_1_2,			lowerCAmelCase__		:   Optional[Any]=1_6,			lowerCAmelCase__		:   Union[str, Any]=2,			lowerCAmelCase__		:   Any=0.02,			lowerCAmelCase__		:   int=3,			lowerCAmelCase__		:   Tuple=4,			lowerCAmelCase__		:   List[str]=None,			)     ->				List[Any]:




										'''simple docstring'''



										_UpperCamelCase      : Optional[int]         =   parent
										_UpperCamelCase      : Any         =   batch_size
										_UpperCamelCase      : str         =   seq_length
										_UpperCamelCase      : Dict         =   is_training
										_UpperCamelCase      : Dict         =   use_input_mask
										_UpperCamelCase      : int         =   use_token_type_ids
										_UpperCamelCase      : str         =   use_labels
										_UpperCamelCase      : List[str]         =   vocab_size
										_UpperCamelCase      : str         =   hidden_size
										_UpperCamelCase      : Dict         =   num_hidden_layers
										_UpperCamelCase      : int         =   num_attention_heads
										_UpperCamelCase      : Dict         =   intermediate_size
										_UpperCamelCase      : Dict         =   hidden_act
										_UpperCamelCase      : Optional[Any]         =   hidden_dropout_prob
										_UpperCamelCase      : List[str]         =   attention_probs_dropout_prob
										_UpperCamelCase      : Tuple         =   max_position_embeddings
										_UpperCamelCase      : Union[str, Any]         =   type_vocab_size
										_UpperCamelCase      : str         =   type_sequence_label_size
										_UpperCamelCase      : Dict         =   initializer_range
										_UpperCamelCase      : List[Any]         =   num_labels
										_UpperCamelCase      : Any         =   num_choices
										_UpperCamelCase      : str         =   scope






				def     snake_case     (			self		:   Union[str, Any]							)     ->				Any:




										'''simple docstring'''



										_UpperCamelCase      : int         =   ids_tensor([self.batch_size, self.seq_length],			self.vocab_size							)

										_UpperCamelCase      : int         =   None
										if self.use_input_mask:
																_UpperCamelCase      : Tuple         =   random_attention_mask([self.batch_size, self.seq_length]							)

										_UpperCamelCase      : int         =   None

										_UpperCamelCase      : List[Any]         =   None
										_UpperCamelCase      : Any         =   None
										_UpperCamelCase      : Dict         =   None
										if self.use_labels:
																_UpperCamelCase      : Optional[Any]         =   ids_tensor([self.batch_size],			self.type_sequence_label_size							)
																_UpperCamelCase      : List[str]         =   ids_tensor([self.batch_size, self.seq_length],			self.num_labels							)
																_UpperCamelCase      : Optional[int]         =   ids_tensor([self.batch_size],			self.num_choices							)

										_UpperCamelCase      : str         =   self.get_config()

										return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






				def     snake_case     (			self		:   str							)     ->				List[Any]:




										'''simple docstring'''



										return FalconConfig(
										    vocab_size=self.vocab_size,			hidden_size=self.hidden_size,			num_hidden_layers=self.num_hidden_layers,			num_attention_heads=self.num_attention_heads,			intermediate_size=self.intermediate_size,			hidden_act=self.hidden_act,			hidden_dropout_prob=self.hidden_dropout_prob,			attention_probs_dropout_prob=self.attention_probs_dropout_prob,			max_position_embeddings=self.max_position_embeddings,			type_vocab_size=self.type_vocab_size,			is_decoder=lowerCAmelCase__,			initializer_range=self.initializer_range,			pad_token_id=1,			new_decoder_architecture=lowerCAmelCase__,			)






				def     snake_case     (			self		:   Union[str, Any],			lowerCAmelCase__		:   Dict,			lowerCAmelCase__		:   List[Any],			lowerCAmelCase__		:   List[Any],			lowerCAmelCase__		:   Optional[Any],			lowerCAmelCase__		:   Optional[Any],			lowerCAmelCase__		:   Optional[Any],			lowerCAmelCase__		:   Optional[Any]							)     ->				Union[str, Any]:




										'''simple docstring'''



										_UpperCamelCase      : Dict         =   FalconModel(config=lowerCAmelCase__							)
										model.to(lowerCAmelCase__							)
										model.eval()
										_UpperCamelCase      : Union[str, Any]         =   model(lowerCAmelCase__,			attention_mask=lowerCAmelCase__							)
										_UpperCamelCase      : Union[str, Any]         =   model(lowerCAmelCase__							)
										self.parent.assertEqual(result.last_hidden_state.shape,			(self.batch_size, self.seq_length, self.hidden_size)							)






				def     snake_case     (			self		:   str,			lowerCAmelCase__		:   Any,			lowerCAmelCase__		:   List[Any],			lowerCAmelCase__		:   Union[str, Any],			lowerCAmelCase__		:   List[Any],			lowerCAmelCase__		:   Tuple,			lowerCAmelCase__		:   str,			lowerCAmelCase__		:   Dict,			lowerCAmelCase__		:   Tuple,			lowerCAmelCase__		:   Optional[Any],			)     ->				Optional[int]:




										'''simple docstring'''



										_UpperCamelCase      : Any         =   True
										_UpperCamelCase      : Dict         =   FalconModel(lowerCAmelCase__							)
										model.to(lowerCAmelCase__							)
										model.eval()
										_UpperCamelCase      : Union[str, Any]         =   model(
										    lowerCAmelCase__,			attention_mask=lowerCAmelCase__,			encoder_hidden_states=lowerCAmelCase__,			encoder_attention_mask=lowerCAmelCase__,			)
										_UpperCamelCase      : Union[str, Any]         =   model(
										    lowerCAmelCase__,			attention_mask=lowerCAmelCase__,			encoder_hidden_states=lowerCAmelCase__,			)
										_UpperCamelCase      : Tuple         =   model(lowerCAmelCase__,			attention_mask=lowerCAmelCase__							)
										self.parent.assertEqual(result.last_hidden_state.shape,			(self.batch_size, self.seq_length, self.hidden_size)							)






				def     snake_case     (			self		:   Optional[Any],			lowerCAmelCase__		:   int,			lowerCAmelCase__		:   Dict,			lowerCAmelCase__		:   Optional[Any],			lowerCAmelCase__		:   Dict,			lowerCAmelCase__		:   str,			lowerCAmelCase__		:   List[Any],			lowerCAmelCase__		:   List[str],			lowerCAmelCase__		:   Dict,			lowerCAmelCase__		:   Optional[Any],			)     ->				Optional[int]:




										'''simple docstring'''



										_UpperCamelCase      : List[Any]         =   FalconForCausalLM(config=lowerCAmelCase__							)
										model.to(lowerCAmelCase__							)
										model.eval()
										_UpperCamelCase      : int         =   model(lowerCAmelCase__,			attention_mask=lowerCAmelCase__,			labels=lowerCAmelCase__							)
										self.parent.assertEqual(result.logits.shape,			(self.batch_size, self.seq_length, self.vocab_size)							)






				def     snake_case     (			self		:   Any,			lowerCAmelCase__		:   Any,			lowerCAmelCase__		:   List[str],			lowerCAmelCase__		:   str,			lowerCAmelCase__		:   int,			lowerCAmelCase__		:   List[Any],			lowerCAmelCase__		:   Optional[Any],			lowerCAmelCase__		:   List[Any],			lowerCAmelCase__		:   Optional[Any],			lowerCAmelCase__		:   Optional[int],			)     ->				int:




										'''simple docstring'''



										_UpperCamelCase      : int         =   True
										_UpperCamelCase      : Optional[int]         =   True
										_UpperCamelCase      : Union[str, Any]         =   FalconForCausalLM(config=lowerCAmelCase__							)
										model.to(lowerCAmelCase__							)
										model.eval()

										# first forward pass
										_UpperCamelCase      : Union[str, Any]         =   model(
										    lowerCAmelCase__,			attention_mask=lowerCAmelCase__,			encoder_hidden_states=lowerCAmelCase__,			encoder_attention_mask=lowerCAmelCase__,			use_cache=lowerCAmelCase__,			)
										_UpperCamelCase      : int         =   outputs.past_key_values

										# create hypothetical multiple next token and extent to next_input_ids
										_UpperCamelCase      : Dict         =   ids_tensor((self.batch_size, 3),			config.vocab_size							)
										_UpperCamelCase      : Dict         =   ids_tensor((self.batch_size, 3),			vocab_size=2							)

										# append to next input_ids and
										_UpperCamelCase      : Dict         =   torch.cat([input_ids, next_tokens],			dim=-1							)
										_UpperCamelCase      : List[str]         =   torch.cat([input_mask, next_mask],			dim=-1							)

										_UpperCamelCase      : Dict         =   model(
										    lowerCAmelCase__,			attention_mask=lowerCAmelCase__,			encoder_hidden_states=lowerCAmelCase__,			encoder_attention_mask=lowerCAmelCase__,			output_hidden_states=lowerCAmelCase__,			)['''hidden_states'''][0]
										_UpperCamelCase      : Tuple         =   model(
										    lowerCAmelCase__,			attention_mask=lowerCAmelCase__,			encoder_hidden_states=lowerCAmelCase__,			encoder_attention_mask=lowerCAmelCase__,			past_key_values=lowerCAmelCase__,			output_hidden_states=lowerCAmelCase__,			)['''hidden_states'''][0]

										# select random slice
										_UpperCamelCase      : Dict         =   ids_tensor((1,),			output_from_past.shape[-1]							).item()
										_UpperCamelCase      : Optional[int]         =   output_from_no_past[:, -3:, random_slice_idx].detach()
										_UpperCamelCase      : List[str]         =   output_from_past[:, :, random_slice_idx].detach()

										self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]							)

										# test that outputs are equal for slice
										self.parent.assertTrue(torch.allclose(lowerCAmelCase__,			lowerCAmelCase__,			atol=1e-3							)							)






				def     snake_case     (			self		:   List[str]							)     ->				Union[str, Any]:




										'''simple docstring'''



										_UpperCamelCase      : Optional[int]         =   self.prepare_config_and_inputs()
										(
										    (
										    _UpperCamelCase
										)       ,							(
										    _UpperCamelCase
										)       ,							(
										    _UpperCamelCase
										)       ,							(
										    _UpperCamelCase
										)       ,							(
										    _UpperCamelCase
										)       ,							(
										    _UpperCamelCase
										)       ,							(
										    _UpperCamelCase
										)       ,							
										)      : Optional[Any]         =   config_and_inputs
										_UpperCamelCase      : List[Any]         =   {'''input_ids''': input_ids, '''attention_mask''': input_mask}
										return config, inputs_dict




@require_torch
class 			_a						(				_lowerCAmelCase     ,    _lowerCAmelCase     ,    _lowerCAmelCase     ,    unittest.TestCase    ):
				UpperCamelCase								=   (
				    (
				        FalconModel,
				        FalconForCausalLM,
				        FalconForSequenceClassification,
				        FalconForTokenClassification,
				        FalconForQuestionAnswering,
				    )
				    if is_torch_available()
				    else ()
				)
				UpperCamelCase								=   (FalconForCausalLM,) if is_torch_available() else ()
				UpperCamelCase								=   (
				    {
				        '''feature-extraction''': FalconModel,
				        '''text-classification''': FalconForSequenceClassification,
				        '''text-generation''': FalconForCausalLM,
				        '''question-answering''': FalconForQuestionAnswering,
				        '''token-classification''': FalconForTokenClassification,
				        '''zero-shot''': FalconForSequenceClassification,
				    }
				    if is_torch_available()
				    else {}
				)
				UpperCamelCase								=   False
				UpperCamelCase								=   False






				def     snake_case     (			self		:   Dict							)     ->				str:




										'''simple docstring'''



										_UpperCamelCase      : str         =   FalconModelTester(self							)
										_UpperCamelCase      : Optional[int]         =   ConfigTester(self,			config_class=lowerCAmelCase__,			hidden_size=3_7							)






				def     snake_case     (			self		:   List[str]							)     ->				Dict:




										'''simple docstring'''



										self.config_tester.run_common_tests()






				def     snake_case     (			self		:   Optional[Any]							)     ->				Dict:




										'''simple docstring'''



										_UpperCamelCase      : str         =   self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_model(*lowerCAmelCase__							)






				def     snake_case     (			self		:   int							)     ->				Optional[int]:




										'''simple docstring'''



										_UpperCamelCase       ,							*_UpperCamelCase      : Dict         =   self.model_tester.prepare_config_and_inputs()
										for alibi in [True, False]:
																_UpperCamelCase      : List[str]         =   alibi
																self.model_tester.create_and_check_model(lowerCAmelCase__,			*lowerCAmelCase__							)






				def     snake_case     (			self		:   Any							)     ->				List[str]:




										'''simple docstring'''



										_UpperCamelCase       ,							_UpperCamelCase      : Tuple         =   self.model_tester.prepare_config_and_inputs_for_common()
										_UpperCamelCase      : Optional[int]         =   3
										_UpperCamelCase      : List[Any]         =   input_dict['''input_ids''']
										_UpperCamelCase      : List[str]         =   input_ids.ne(1							).to(lowerCAmelCase__							)
										_UpperCamelCase      : Tuple         =   ids_tensor([self.model_tester.batch_size],			self.model_tester.type_sequence_label_size							)
										_UpperCamelCase      : Any         =   FalconForSequenceClassification(lowerCAmelCase__							)
										model.to(lowerCAmelCase__							)
										model.eval()
										_UpperCamelCase      : int         =   model(lowerCAmelCase__,			attention_mask=lowerCAmelCase__,			labels=lowerCAmelCase__							)
										self.assertEqual(result.logits.shape,			(self.model_tester.batch_size, self.model_tester.num_labels)							)






				def     snake_case     (			self		:   Dict							)     ->				List[Any]:




										'''simple docstring'''



										_UpperCamelCase       ,							_UpperCamelCase      : str         =   self.model_tester.prepare_config_and_inputs_for_common()
										_UpperCamelCase      : Optional[int]         =   3
										_UpperCamelCase      : Union[str, Any]         =   '''single_label_classification'''
										_UpperCamelCase      : Optional[Any]         =   input_dict['''input_ids''']
										_UpperCamelCase      : List[Any]         =   input_ids.ne(1							).to(lowerCAmelCase__							)
										_UpperCamelCase      : Tuple         =   ids_tensor([self.model_tester.batch_size],			self.model_tester.type_sequence_label_size							)
										_UpperCamelCase      : Dict         =   FalconForSequenceClassification(lowerCAmelCase__							)
										model.to(lowerCAmelCase__							)
										model.eval()
										_UpperCamelCase      : Dict         =   model(lowerCAmelCase__,			attention_mask=lowerCAmelCase__,			labels=lowerCAmelCase__							)
										self.assertEqual(result.logits.shape,			(self.model_tester.batch_size, self.model_tester.num_labels)							)






				def     snake_case     (			self		:   int							)     ->				List[str]:




										'''simple docstring'''



										_UpperCamelCase       ,							_UpperCamelCase      : Union[str, Any]         =   self.model_tester.prepare_config_and_inputs_for_common()
										_UpperCamelCase      : Optional[int]         =   input_dict['''input_ids''']
										_UpperCamelCase      : str         =   FalconForCausalLM(lowerCAmelCase__							)
										model.to(lowerCAmelCase__							)
										model.eval()
										_UpperCamelCase      : int         =   model(lowerCAmelCase__,			use_cache=lowerCAmelCase__							)
										_UpperCamelCase      : List[str]         =   input_ids.shape[0]
										_UpperCamelCase      : Optional[Any]         =   model._convert_to_rw_cache(result.past_key_values							)
										_UpperCamelCase      : List[str]         =   model._convert_cache_to_standard_format(lowerCAmelCase__,			lowerCAmelCase__							)
										for layer in range(len(lowerCAmelCase__							)							):
																for tensor_idx in range(2							):
																						self.assertTrue(rw_cache[layer][tensor_idx].ndim == 3							)
																						self.assertTrue(result.past_key_values[layer][tensor_idx].ndim == 4							)
																						self.assertTrue(
																						    torch.all(result.past_key_values[layer][tensor_idx] == standard_cache[layer][tensor_idx]							)							)






				def     snake_case     (			self		:   List[str]							)     ->				Dict:




										'''simple docstring'''



										_UpperCamelCase       ,							_UpperCamelCase      : Optional[Any]         =   self.model_tester.prepare_config_and_inputs_for_common()
										_UpperCamelCase      : Optional[Any]         =   3
										_UpperCamelCase      : List[str]         =   '''multi_label_classification'''
										_UpperCamelCase      : Any         =   input_dict['''input_ids''']
										_UpperCamelCase      : Optional[Any]         =   input_ids.ne(1							).to(lowerCAmelCase__							)
										_UpperCamelCase      : Optional[int]         =   ids_tensor(
										    [self.model_tester.batch_size, config.num_labels],			self.model_tester.type_sequence_label_size							).to(torch.float							)
										_UpperCamelCase      : Optional[Any]         =   FalconForSequenceClassification(lowerCAmelCase__							)
										model.to(lowerCAmelCase__							)
										model.eval()
										_UpperCamelCase      : Any         =   model(lowerCAmelCase__,			attention_mask=lowerCAmelCase__,			labels=lowerCAmelCase__							)
										self.assertEqual(result.logits.shape,			(self.model_tester.batch_size, self.model_tester.num_labels)							)






				def     snake_case     (			self		:   Optional[int]							)     ->				str:




										'''simple docstring'''



										for model_class in self.all_generative_model_classes:
																_UpperCamelCase       ,							_UpperCamelCase      : Optional[Any]         =   self.model_tester.prepare_config_and_inputs_for_common()

																# If it doesn't support cache, pass the test
																if not hasattr(lowerCAmelCase__,			'''use_cache'''							):
																						return

																_UpperCamelCase      : Optional[Any]         =   model_class(lowerCAmelCase__							).to(lowerCAmelCase__							)
																if "use_cache" not in inputs:
																						_UpperCamelCase      : Dict         =   True
																_UpperCamelCase      : str         =   model(**lowerCAmelCase__							)

																# If "past_key_values" is not returned, pass the test (e.g. RWKV uses a different cache name and format)
																if "past_key_values" not in outputs:
																						return

																_UpperCamelCase      : int         =   (
																    getattr(lowerCAmelCase__,			'''decoder_layers''',			lowerCAmelCase__							)
																    or getattr(lowerCAmelCase__,			'''num_decoder_layers''',			lowerCAmelCase__							)
																    or config.num_hidden_layers
																)
																_UpperCamelCase      : List[str]         =   getattr(lowerCAmelCase__,			'''num_kv_heads''',			config.num_attention_heads							)
																_UpperCamelCase      : int         =   getattr(lowerCAmelCase__,			'''d_model''',			config.hidden_size							)
																_UpperCamelCase      : Optional[int]         =   embed_dim // num_attention_heads

																_UpperCamelCase      : int         =   outputs['''past_key_values''']
																self.assertEqual(len(lowerCAmelCase__							),			lowerCAmelCase__							)

																_UpperCamelCase       ,							_UpperCamelCase      : Optional[Any]         =   inputs['''input_ids'''].shape
																for i in range(lowerCAmelCase__							):
																						if config.new_decoder_architecture:
																												_UpperCamelCase      : List[str]         =   config.num_attention_heads
																						elif config.multi_query:
																												_UpperCamelCase      : Optional[Any]         =   1
																						self.assertEqual(len(past_kv[0]							),			2							)  # K V for the decoder = 2
																						self.assertEqual(
																						    past_kv[i][0].shape,			(batch_size, num_attention_heads, seq_length, per_head_embed_dim)							)
																						self.assertEqual(
																						    past_kv[i][1].shape,			(batch_size, num_attention_heads, seq_length, per_head_embed_dim)							)




@require_torch
class 			_a						(				unittest.TestCase    ):






				@slow
				def     snake_case     (			self		:   Any							)     ->				Any:




										'''simple docstring'''



										_UpperCamelCase      : Optional[Any]         =   AutoTokenizer.from_pretrained('''Rocketknight1/falcon-rw-1b'''							)
										_UpperCamelCase      : Union[str, Any]         =   FalconForCausalLM.from_pretrained('''Rocketknight1/falcon-rw-1b'''							)
										model.eval()
										model.to(lowerCAmelCase__							)
										_UpperCamelCase      : Dict         =   tokenizer('''My favorite food is''',			return_tensors='''pt'''							).to(lowerCAmelCase__							)

										_UpperCamelCase      : Dict         =   (
										    '''My favorite food is pizza. I love it so much that I have a pizza party every year for my birthday.'''
										)

										_UpperCamelCase      : Any         =   model.generate(**lowerCAmelCase__,			do_sample=lowerCAmelCase__,			max_new_tokens=1_9							)
										_UpperCamelCase      : List[str]         =   tokenizer.batch_decode(lowerCAmelCase__							)[0]

										self.assertEqual(lowerCAmelCase__,			lowerCAmelCase__							)






				@slow
				def     snake_case     (			self		:   List[str]							)     ->				Optional[Any]:




										'''simple docstring'''



										for repo in ["Rocketknight1/tiny-random-falcon-7b", "Rocketknight1/tiny-random-falcon-40b"]:
																_UpperCamelCase      : int         =   AutoTokenizer.from_pretrained(lowerCAmelCase__							)
																_UpperCamelCase      : List[str]         =   FalconForCausalLM.from_pretrained(lowerCAmelCase__							)
																model.eval()
																model.to(lowerCAmelCase__							)
																_UpperCamelCase      : Tuple         =   tokenizer('''My favorite food is''',			return_tensors='''pt'''							).to(lowerCAmelCase__							)

																# We just test that these run without errors - the models are randomly initialized
																# and so the actual text outputs will be garbage
																model.generate(**lowerCAmelCase__,			do_sample=lowerCAmelCase__,			max_new_tokens=4							)
																model.generate(**lowerCAmelCase__,			do_sample=lowerCAmelCase__,			max_new_tokens=4							)
																model.generate(**lowerCAmelCase__,			num_beams=2,			max_new_tokens=4							)






				@slow
				def     snake_case     (			self		:   str							)     ->				List[Any]:




										'''simple docstring'''



										with torch.no_grad():
																for repo in [
																    "Rocketknight1/falcon-rw-1b",
																    "Rocketknight1/tiny-random-falcon-7b",
																    "Rocketknight1/tiny-random-falcon-40b",
																]:
																						_UpperCamelCase      : Union[str, Any]         =   AutoTokenizer.from_pretrained(lowerCAmelCase__							)
																						_UpperCamelCase      : List[str]         =   FalconForCausalLM.from_pretrained(lowerCAmelCase__							)
																						model.eval()
																						model.to(device=lowerCAmelCase__							)
																						_UpperCamelCase      : Dict         =   tokenizer('''My favorite food is''',			return_tensors='''pt'''							).to(lowerCAmelCase__							)

																						# Test results are the same with and without cache
																						_UpperCamelCase      : Optional[Any]         =   model.generate(**lowerCAmelCase__,			do_sample=lowerCAmelCase__,			max_new_tokens=2_0,			use_cache=lowerCAmelCase__							)
																						_UpperCamelCase      : str         =   model.generate(**lowerCAmelCase__,			do_sample=lowerCAmelCase__,			max_new_tokens=2_0,			use_cache=lowerCAmelCase__							)
																						self.assertTrue((outputs_cache - outputs_no_cache).sum().item() == 0							)










"""simple docstring"""




import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
						import torch

						from transformers import (
						    OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
						    OpenAIGPTConfig,
						    OpenAIGPTDoubleHeadsModel,
						    OpenAIGPTForSequenceClassification,
						    OpenAIGPTLMHeadModel,
						    OpenAIGPTModel,
						)




class 			_a						:






				def __init__(			self		:   Dict,			lowerCAmelCase__		:   Optional[Any],			lowerCAmelCase__		:   Optional[int]=1_3,			lowerCAmelCase__		:   Optional[Any]=7,			lowerCAmelCase__		:   Optional[Any]=True,			lowerCAmelCase__		:   Any=True,			lowerCAmelCase__		:   str=True,			lowerCAmelCase__		:   Any=9_9,			lowerCAmelCase__		:   Dict=3_2,			lowerCAmelCase__		:   List[Any]=5,			lowerCAmelCase__		:   Tuple=4,			lowerCAmelCase__		:   List[Any]=3_7,			lowerCAmelCase__		:   Tuple="gelu",			lowerCAmelCase__		:   Any=0.1,			lowerCAmelCase__		:   Optional[Any]=0.1,			lowerCAmelCase__		:   Dict=5_1_2,			lowerCAmelCase__		:   List[str]=1_6,			lowerCAmelCase__		:   Tuple=2,			lowerCAmelCase__		:   int=0.02,			lowerCAmelCase__		:   int=3,			lowerCAmelCase__		:   Optional[Any]=4,			lowerCAmelCase__		:   Dict=None,			)     ->				int:




										'''simple docstring'''



										_UpperCamelCase      : Tuple         =   parent
										_UpperCamelCase      : Union[str, Any]         =   batch_size
										_UpperCamelCase      : Union[str, Any]         =   seq_length
										_UpperCamelCase      : Tuple         =   is_training
										_UpperCamelCase      : Tuple         =   use_token_type_ids
										_UpperCamelCase      : Optional[int]         =   use_labels
										_UpperCamelCase      : Dict         =   vocab_size
										_UpperCamelCase      : int         =   hidden_size
										_UpperCamelCase      : Optional[Any]         =   num_hidden_layers
										_UpperCamelCase      : str         =   num_attention_heads
										_UpperCamelCase      : Union[str, Any]         =   intermediate_size
										_UpperCamelCase      : List[str]         =   hidden_act
										_UpperCamelCase      : Optional[Any]         =   hidden_dropout_prob
										_UpperCamelCase      : int         =   attention_probs_dropout_prob
										_UpperCamelCase      : Union[str, Any]         =   max_position_embeddings
										_UpperCamelCase      : int         =   type_vocab_size
										_UpperCamelCase      : List[str]         =   type_sequence_label_size
										_UpperCamelCase      : List[str]         =   initializer_range
										_UpperCamelCase      : int         =   num_labels
										_UpperCamelCase      : List[str]         =   num_choices
										_UpperCamelCase      : str         =   scope
										_UpperCamelCase      : Optional[int]         =   self.vocab_size - 1






				def     snake_case     (			self		:   int							)     ->				Optional[Any]:




										'''simple docstring'''



										_UpperCamelCase      : Dict         =   ids_tensor([self.batch_size, self.seq_length],			self.vocab_size							)

										_UpperCamelCase      : List[str]         =   None
										if self.use_token_type_ids:
																_UpperCamelCase      : Tuple         =   ids_tensor([self.batch_size, self.seq_length],			self.type_vocab_size							)

										_UpperCamelCase      : Optional[int]         =   None
										_UpperCamelCase      : str         =   None
										_UpperCamelCase      : List[str]         =   None
										if self.use_labels:
																_UpperCamelCase      : int         =   ids_tensor([self.batch_size],			self.type_sequence_label_size							)
																_UpperCamelCase      : Dict         =   ids_tensor([self.batch_size, self.seq_length],			self.num_labels							)
																_UpperCamelCase      : Dict         =   ids_tensor([self.batch_size],			self.num_choices							)

										_UpperCamelCase      : str         =   OpenAIGPTConfig(
										    vocab_size=self.vocab_size,			n_embd=self.hidden_size,			n_layer=self.num_hidden_layers,			n_head=self.num_attention_heads,			n_positions=self.max_position_embeddings,			pad_token_id=self.pad_token_id,			)

										_UpperCamelCase      : List[Any]         =   ids_tensor([self.num_hidden_layers, self.num_attention_heads],			2							)

										return (
										    config,
										    input_ids,
										    head_mask,
										    token_type_ids,
										    sequence_labels,
										    token_labels,
										    choice_labels,
										)






				def     snake_case     (			self		:   Union[str, Any],			lowerCAmelCase__		:   Optional[int],			lowerCAmelCase__		:   List[str],			lowerCAmelCase__		:   List[str],			lowerCAmelCase__		:   List[str],			*lowerCAmelCase__		:   List[Any]							)     ->				Optional[Any]:




										'''simple docstring'''



										_UpperCamelCase      : Dict         =   OpenAIGPTModel(config=lowerCAmelCase__							)
										model.to(lowerCAmelCase__							)
										model.eval()

										_UpperCamelCase      : List[str]         =   model(lowerCAmelCase__,			token_type_ids=lowerCAmelCase__,			head_mask=lowerCAmelCase__							)
										_UpperCamelCase      : Any         =   model(lowerCAmelCase__,			token_type_ids=lowerCAmelCase__							)
										_UpperCamelCase      : List[str]         =   model(lowerCAmelCase__							)

										self.parent.assertEqual(result.last_hidden_state.shape,			(self.batch_size, self.seq_length, self.hidden_size)							)






				def     snake_case     (			self		:   Any,			lowerCAmelCase__		:   Tuple,			lowerCAmelCase__		:   Union[str, Any],			lowerCAmelCase__		:   Any,			lowerCAmelCase__		:   Optional[Any],			*lowerCAmelCase__		:   Union[str, Any]							)     ->				Any:




										'''simple docstring'''



										_UpperCamelCase      : Any         =   OpenAIGPTLMHeadModel(lowerCAmelCase__							)
										model.to(lowerCAmelCase__							)
										model.eval()

										_UpperCamelCase      : Tuple         =   model(lowerCAmelCase__,			token_type_ids=lowerCAmelCase__,			labels=lowerCAmelCase__							)
										self.parent.assertEqual(result.loss.shape,			()							)
										self.parent.assertEqual(result.logits.shape,			(self.batch_size, self.seq_length, self.vocab_size)							)






				def     snake_case     (			self		:   Optional[int],			lowerCAmelCase__		:   str,			lowerCAmelCase__		:   List[str],			lowerCAmelCase__		:   Any,			lowerCAmelCase__		:   List[Any],			*lowerCAmelCase__		:   Any							)     ->				int:




										'''simple docstring'''



										_UpperCamelCase      : Tuple         =   OpenAIGPTDoubleHeadsModel(lowerCAmelCase__							)
										model.to(lowerCAmelCase__							)
										model.eval()

										_UpperCamelCase      : Optional[int]         =   model(lowerCAmelCase__,			token_type_ids=lowerCAmelCase__,			labels=lowerCAmelCase__							)
										self.parent.assertEqual(result.loss.shape,			()							)
										self.parent.assertEqual(result.logits.shape,			(self.batch_size, self.seq_length, self.vocab_size)							)






				def     snake_case     (			self		:   List[str],			lowerCAmelCase__		:   Dict,			lowerCAmelCase__		:   Dict,			lowerCAmelCase__		:   List[str],			lowerCAmelCase__		:   Optional[Any],			*lowerCAmelCase__		:   List[str]							)     ->				int:




										'''simple docstring'''



										_UpperCamelCase      : List[Any]         =   self.num_labels
										_UpperCamelCase      : Optional[int]         =   OpenAIGPTForSequenceClassification(lowerCAmelCase__							)
										model.to(lowerCAmelCase__							)
										model.eval()

										_UpperCamelCase      : str         =   ids_tensor([self.batch_size],			self.type_sequence_label_size							)
										_UpperCamelCase      : Union[str, Any]         =   model(lowerCAmelCase__,			token_type_ids=lowerCAmelCase__,			labels=lowerCAmelCase__							)
										self.parent.assertEqual(result.logits.shape,			(self.batch_size, self.num_labels)							)






				def     snake_case     (			self		:   str							)     ->				Optional[Any]:




										'''simple docstring'''



										_UpperCamelCase      : Any         =   self.prepare_config_and_inputs()
										(
										    (
										    _UpperCamelCase
										)       ,							(
										    _UpperCamelCase
										)       ,							(
										    _UpperCamelCase
										)       ,							(
										    _UpperCamelCase
										)       ,							(
										    _UpperCamelCase
										)       ,							(
										    _UpperCamelCase
										)       ,							(
										    _UpperCamelCase
										)       ,							
										)      : Tuple         =   config_and_inputs
										_UpperCamelCase      : Tuple         =   {
										    '''input_ids''': input_ids,
										    '''token_type_ids''': token_type_ids,
										    '''head_mask''': head_mask,
										}

										return config, inputs_dict




@require_torch
class 			_a						(				_lowerCAmelCase     ,    _lowerCAmelCase     ,    _lowerCAmelCase     ,    unittest.TestCase    ):
				UpperCamelCase								=   (
				    (OpenAIGPTModel, OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel, OpenAIGPTForSequenceClassification)
				    if is_torch_available()
				    else ()
				)
				UpperCamelCase								=   (
				    (OpenAIGPTLMHeadModel,) if is_torch_available() else ()
				)  # TODO (PVP): Add Double HeadsModel when generate() function is changed accordingly
				UpperCamelCase								=   (
				    {
				        '''feature-extraction''': OpenAIGPTModel,
				        '''text-classification''': OpenAIGPTForSequenceClassification,
				        '''text-generation''': OpenAIGPTLMHeadModel,
				        '''zero-shot''': OpenAIGPTForSequenceClassification,
				    }
				    if is_torch_available()
				    else {}
				)






				def     snake_case     (			self		:   Union[str, Any],			lowerCAmelCase__		:   Any,			lowerCAmelCase__		:   List[str],			lowerCAmelCase__		:   str,			lowerCAmelCase__		:   List[str],			lowerCAmelCase__		:   List[str]							)     ->				List[str]:




										'''simple docstring'''



										if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests":
																# Get `tokenizer does not have a padding token` error for both fast/slow tokenizers.
																# `OpenAIGPTConfig` was never used in pipeline tests, either because of a missing checkpoint or because a
																# tiny config could not be created.
																return True

										return False






				def     snake_case     (			self		:   str,			lowerCAmelCase__		:   Optional[int],			lowerCAmelCase__		:   List[str],			lowerCAmelCase__		:   Optional[int]=False							)     ->				Tuple:




										'''simple docstring'''



										_UpperCamelCase      : Optional[Any]         =   super()._prepare_for_class(lowerCAmelCase__,			lowerCAmelCase__,			return_labels=lowerCAmelCase__							)

										if return_labels:
																if model_class.__name__ == "OpenAIGPTDoubleHeadsModel":
																						_UpperCamelCase      : Union[str, Any]         =   torch.zeros(
																						    (self.model_tester.batch_size, self.model_tester.num_choices, self.model_tester.seq_length),			dtype=torch.long,			device=lowerCAmelCase__,			)
																						_UpperCamelCase      : Tuple         =   inputs_dict['''labels''']
																						_UpperCamelCase      : List[str]         =   inputs_dict['''labels''']
																						_UpperCamelCase      : Optional[Any]         =   torch.zeros(
																						    (self.model_tester.batch_size, self.model_tester.num_choices),			dtype=torch.long,			device=lowerCAmelCase__,			)
																						_UpperCamelCase      : Dict         =   torch.zeros(
																						    self.model_tester.batch_size,			dtype=torch.long,			device=lowerCAmelCase__							)
										return inputs_dict






				def     snake_case     (			self		:   List[str]							)     ->				List[str]:




										'''simple docstring'''



										_UpperCamelCase      : Optional[Any]         =   OpenAIGPTModelTester(self							)
										_UpperCamelCase      : int         =   ConfigTester(self,			config_class=lowerCAmelCase__,			n_embd=3_7							)






				def     snake_case     (			self		:   Optional[int]							)     ->				str:




										'''simple docstring'''



										self.config_tester.run_common_tests()






				def     snake_case     (			self		:   Optional[int]							)     ->				Any:




										'''simple docstring'''



										_UpperCamelCase      : List[Any]         =   self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_openai_gpt_model(*lowerCAmelCase__							)






				def     snake_case     (			self		:   Any							)     ->				Dict:




										'''simple docstring'''



										_UpperCamelCase      : Union[str, Any]         =   self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_lm_head_model(*lowerCAmelCase__							)






				def     snake_case     (			self		:   int							)     ->				Dict:




										'''simple docstring'''



										_UpperCamelCase      : Union[str, Any]         =   self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_double_lm_head_model(*lowerCAmelCase__							)






				def     snake_case     (			self		:   List[str]							)     ->				int:




										'''simple docstring'''



										_UpperCamelCase      : List[Any]         =   self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_openai_gpt_for_sequence_classification(*lowerCAmelCase__							)






				@slow
				def     snake_case     (			self		:   Optional[Any]							)     ->				Optional[Any]:




										'''simple docstring'''



										for model_name in OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																_UpperCamelCase      : int         =   OpenAIGPTModel.from_pretrained(lowerCAmelCase__							)
																self.assertIsNotNone(lowerCAmelCase__							)




@require_torch
class 			_a						(				unittest.TestCase    ):






				@slow
				def     snake_case     (			self		:   Union[str, Any]							)     ->				Optional[Any]:




										'''simple docstring'''



										_UpperCamelCase      : int         =   OpenAIGPTLMHeadModel.from_pretrained('''openai-gpt'''							)
										model.to(lowerCAmelCase__							)
										_UpperCamelCase      : str         =   torch.tensor([[4_8_1, 4_7_3_5, 5_4_4]],			dtype=torch.long,			device=lowerCAmelCase__							)  # the president is
										_UpperCamelCase      : Optional[int]         =   [
										    4_8_1,
										    4_7_3_5,
										    5_4_4,
										    2_4_6,
										    9_6_3,
										    8_7_0,
										    7_6_2,
										    2_3_9,
										    2_4_4,
										    4_0_4_7_7,
										    2_4_4,
										    2_4_9,
										    7_1_9,
										    8_8_1,
										    4_8_7,
										    5_4_4,
										    2_4_0,
										    2_4_4,
										    6_0_3,
										    4_8_1,
										]  # the president is a very good man. " \n " i\'m sure he is, " said the

										_UpperCamelCase      : Union[str, Any]         =   model.generate(lowerCAmelCase__,			do_sample=lowerCAmelCase__							)
										self.assertListEqual(output_ids[0].tolist(),			lowerCAmelCase__							)



