






import unittest

from transformers import AutoTokenizer, FalconConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import (
		    FalconForCausalLM,
		    FalconForQuestionAnswering,
		    FalconForSequenceClassification,
		    FalconForTokenClassification,
		    FalconModel,
		)



class 				__UpperCAmelCase		:







			def __init__(     self:  Optional[int]				,     UpperCAmelCase_:  List[Any]				,     UpperCAmelCase_:  List[str]=3				,     UpperCAmelCase_:  List[Any]=7				,     UpperCAmelCase_:  str=True				,     UpperCAmelCase_:  Optional[int]=True				,     UpperCAmelCase_:  List[Any]=False				,     UpperCAmelCase_:  str=True				,     UpperCAmelCase_:  Any=99				,     UpperCAmelCase_:  int=32				,     UpperCAmelCase_:  List[Any]=5				,     UpperCAmelCase_:  str=4				,     UpperCAmelCase_:  Optional[int]=37				,     UpperCAmelCase_:  int="gelu"				,     UpperCAmelCase_:  List[str]=0.1				,     UpperCAmelCase_:  str=0.1				,     UpperCAmelCase_:  int=512				,     UpperCAmelCase_:  int=16				,     UpperCAmelCase_:  Optional[int]=2				,     UpperCAmelCase_:  Optional[Any]=0.02				,     UpperCAmelCase_:  Any=3				,     UpperCAmelCase_:  Optional[int]=4				,     UpperCAmelCase_:  Union[str, Any]=None				,     ) ->    Any:







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  parent
									_SCREAMING_SNAKE_CASE     				=  batch_size
									_SCREAMING_SNAKE_CASE     				=  seq_length
									_SCREAMING_SNAKE_CASE     				=  is_training
									_SCREAMING_SNAKE_CASE     				=  use_input_mask
									_SCREAMING_SNAKE_CASE     				=  use_token_type_ids
									_SCREAMING_SNAKE_CASE     				=  use_labels
									_SCREAMING_SNAKE_CASE     				=  vocab_size
									_SCREAMING_SNAKE_CASE     				=  hidden_size
									_SCREAMING_SNAKE_CASE     				=  num_hidden_layers
									_SCREAMING_SNAKE_CASE     				=  num_attention_heads
									_SCREAMING_SNAKE_CASE     				=  intermediate_size
									_SCREAMING_SNAKE_CASE     				=  hidden_act
									_SCREAMING_SNAKE_CASE     				=  hidden_dropout_prob
									_SCREAMING_SNAKE_CASE     				=  attention_probs_dropout_prob
									_SCREAMING_SNAKE_CASE     				=  max_position_embeddings
									_SCREAMING_SNAKE_CASE     				=  type_vocab_size
									_SCREAMING_SNAKE_CASE     				=  type_sequence_label_size
									_SCREAMING_SNAKE_CASE     				=  initializer_range
									_SCREAMING_SNAKE_CASE     				=  num_labels
									_SCREAMING_SNAKE_CASE     				=  num_choices
									_SCREAMING_SNAKE_CASE     				=  scope







			def 							UpperCamelCase				(     self:  Dict					) ->    Any:







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  ids_tensor([self.batch_size, self.seq_length]				,     self.vocab_size					)

									_SCREAMING_SNAKE_CASE     				=  None
									if self.use_input_mask:
															_SCREAMING_SNAKE_CASE     				=  random_attention_mask([self.batch_size, self.seq_length]					)

									_SCREAMING_SNAKE_CASE     				=  None

									_SCREAMING_SNAKE_CASE     				=  None
									_SCREAMING_SNAKE_CASE     				=  None
									_SCREAMING_SNAKE_CASE     				=  None
									if self.use_labels:
															_SCREAMING_SNAKE_CASE     				=  ids_tensor([self.batch_size]				,     self.type_sequence_label_size					)
															_SCREAMING_SNAKE_CASE     				=  ids_tensor([self.batch_size, self.seq_length]				,     self.num_labels					)
															_SCREAMING_SNAKE_CASE     				=  ids_tensor([self.batch_size]				,     self.num_choices					)

									_SCREAMING_SNAKE_CASE     				=  self.get_config()

									return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels







			def 							UpperCamelCase				(     self:  Optional[int]					) ->    Optional[int]:







									'''simple docstring'''




									return FalconConfig(
									    vocab_size=self.vocab_size				,     hidden_size=self.hidden_size				,     num_hidden_layers=self.num_hidden_layers				,     num_attention_heads=self.num_attention_heads				,     intermediate_size=self.intermediate_size				,     hidden_act=self.hidden_act				,     hidden_dropout_prob=self.hidden_dropout_prob				,     attention_probs_dropout_prob=self.attention_probs_dropout_prob				,     max_position_embeddings=self.max_position_embeddings				,     type_vocab_size=self.type_vocab_size				,     is_decoder=__A				,     initializer_range=self.initializer_range				,     pad_token_id=1				,     new_decoder_architecture=__A				,     )







			def 							UpperCamelCase				(     self:  List[str]				,     UpperCAmelCase_:  Union[str, Any]				,     UpperCAmelCase_:  Optional[Any]				,     UpperCAmelCase_:  List[Any]				,     UpperCAmelCase_:  Any				,     UpperCAmelCase_:  Dict				,     UpperCAmelCase_:  List[str]				,     UpperCAmelCase_:  Dict					) ->    List[Any]:







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  FalconModel(config=__A					)
									model.to(__A					)
									model.eval()
									_SCREAMING_SNAKE_CASE     				=  model(__A				,     attention_mask=__A					)
									_SCREAMING_SNAKE_CASE     				=  model(__A					)
									self.parent.assertEqual(result.last_hidden_state.shape				,     (self.batch_size, self.seq_length, self.hidden_size)					)







			def 							UpperCamelCase				(     self:  int				,     UpperCAmelCase_:  Dict				,     UpperCAmelCase_:  Dict				,     UpperCAmelCase_:  List[Any]				,     UpperCAmelCase_:  int				,     UpperCAmelCase_:  str				,     UpperCAmelCase_:  str				,     UpperCAmelCase_:  Any				,     UpperCAmelCase_:  Optional[Any]				,     UpperCAmelCase_:  Union[str, Any]				,     ) ->    List[Any]:







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  True
									_SCREAMING_SNAKE_CASE     				=  FalconModel(__A					)
									model.to(__A					)
									model.eval()
									_SCREAMING_SNAKE_CASE     				=  model(
									    __A				,     attention_mask=__A				,     encoder_hidden_states=__A				,     encoder_attention_mask=__A				,     )
									_SCREAMING_SNAKE_CASE     				=  model(
									    __A				,     attention_mask=__A				,     encoder_hidden_states=__A				,     )
									_SCREAMING_SNAKE_CASE     				=  model(__A				,     attention_mask=__A					)
									self.parent.assertEqual(result.last_hidden_state.shape				,     (self.batch_size, self.seq_length, self.hidden_size)					)







			def 							UpperCamelCase				(     self:  List[Any]				,     UpperCAmelCase_:  List[Any]				,     UpperCAmelCase_:  List[Any]				,     UpperCAmelCase_:  Optional[int]				,     UpperCAmelCase_:  Union[str, Any]				,     UpperCAmelCase_:  Optional[Any]				,     UpperCAmelCase_:  Optional[Any]				,     UpperCAmelCase_:  List[Any]				,     UpperCAmelCase_:  Any				,     UpperCAmelCase_:  Union[str, Any]				,     ) ->    Optional[Any]:







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  FalconForCausalLM(config=__A					)
									model.to(__A					)
									model.eval()
									_SCREAMING_SNAKE_CASE     				=  model(__A				,     attention_mask=__A				,     labels=__A					)
									self.parent.assertEqual(result.logits.shape				,     (self.batch_size, self.seq_length, self.vocab_size)					)







			def 							UpperCamelCase				(     self:  List[str]				,     UpperCAmelCase_:  Optional[Any]				,     UpperCAmelCase_:  List[Any]				,     UpperCAmelCase_:  List[str]				,     UpperCAmelCase_:  str				,     UpperCAmelCase_:  Any				,     UpperCAmelCase_:  List[Any]				,     UpperCAmelCase_:  Optional[int]				,     UpperCAmelCase_:  Tuple				,     UpperCAmelCase_:  Tuple				,     ) ->    int:







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  True
									_SCREAMING_SNAKE_CASE     				=  True
									_SCREAMING_SNAKE_CASE     				=  FalconForCausalLM(config=__A					)
									model.to(__A					)
									model.eval()

									# first forward pass
									_SCREAMING_SNAKE_CASE     				=  model(
									    __A				,     attention_mask=__A				,     encoder_hidden_states=__A				,     encoder_attention_mask=__A				,     use_cache=__A				,     )
									_SCREAMING_SNAKE_CASE     				=  outputs.past_key_values

									# create hypothetical multiple next token and extent to next_input_ids
									_SCREAMING_SNAKE_CASE     				=  ids_tensor((self.batch_size, 3)				,     config.vocab_size					)
									_SCREAMING_SNAKE_CASE     				=  ids_tensor((self.batch_size, 3)				,     vocab_size=2					)

									# append to next input_ids and
									_SCREAMING_SNAKE_CASE     				=  torch.cat([input_ids, next_tokens]				,     dim=-1					)
									_SCREAMING_SNAKE_CASE     				=  torch.cat([input_mask, next_mask]				,     dim=-1					)

									_SCREAMING_SNAKE_CASE     				=  model(
									    __A				,     attention_mask=__A				,     encoder_hidden_states=__A				,     encoder_attention_mask=__A				,     output_hidden_states=__A				,     )['''hidden_states'''][0]
									_SCREAMING_SNAKE_CASE     				=  model(
									    __A				,     attention_mask=__A				,     encoder_hidden_states=__A				,     encoder_attention_mask=__A				,     past_key_values=__A				,     output_hidden_states=__A				,     )['''hidden_states'''][0]

									# select random slice
									_SCREAMING_SNAKE_CASE     				=  ids_tensor((1,)				,     output_from_past.shape[-1]					).item()
									_SCREAMING_SNAKE_CASE     				=  output_from_no_past[:, -3:, random_slice_idx].detach()
									_SCREAMING_SNAKE_CASE     				=  output_from_past[:, :, random_slice_idx].detach()

									self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]					)

									# test that outputs are equal for slice
									self.parent.assertTrue(torch.allclose(__A				,     __A				,     atol=1E-3					)					)







			def 							UpperCamelCase				(     self:  Union[str, Any]					) ->    Tuple:







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  self.prepare_config_and_inputs()
									(
									    _SCREAMING_SNAKE_CASE						
									)     				=  config_and_inputs
									_SCREAMING_SNAKE_CASE     				=  {'''input_ids''': input_ids, '''attention_mask''': input_mask}
									return config, inputs_dict



@require_torch
class 				__UpperCAmelCase		(lowerCamelCase__	,lowerCamelCase__	,lowerCamelCase__	,unittest.TestCase   ):
			__snake_case    :				Tuple									=							(
			    (
			        FalconModel,
			        FalconForCausalLM,
			        FalconForSequenceClassification,
			        FalconForTokenClassification,
			        FalconForQuestionAnswering,
			    )
			    if is_torch_available()
			    else ()
			)
			__snake_case    :				Dict									=							(FalconForCausalLM,) if is_torch_available() else ()
			__snake_case    :				List[str]									=							(
			    {
			        "feature-extraction": FalconModel,
			        "text-classification": FalconForSequenceClassification,
			        "text-generation": FalconForCausalLM,
			        "question-answering": FalconForQuestionAnswering,
			        "token-classification": FalconForTokenClassification,
			        "zero-shot": FalconForSequenceClassification,
			    }
			    if is_torch_available()
			    else {}
			)
			__snake_case    :				List[Any]									=							False
			__snake_case    :				Dict									=							False







			def 							UpperCamelCase				(     self:  Dict					) ->    str:







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  FalconModelTester(self					)
									_SCREAMING_SNAKE_CASE     				=  ConfigTester(self				,     config_class=__A				,     hidden_size=37					)







			def 							UpperCamelCase				(     self:  Optional[Any]					) ->    List[str]:







									'''simple docstring'''




									self.config_tester.run_common_tests()







			def 							UpperCamelCase				(     self:  Union[str, Any]					) ->    Any:







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_model(*__A					)







			def 							UpperCamelCase				(     self:  Optional[Any]					) ->    Dict:







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  self.model_tester.prepare_config_and_inputs()
									for alibi in [True, False]:
															_SCREAMING_SNAKE_CASE     				=  alibi
															self.model_tester.create_and_check_model(__A				,     *__A					)







			def 							UpperCamelCase				(     self:  List[str]					) ->    str:







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  self.model_tester.prepare_config_and_inputs_for_common()
									_SCREAMING_SNAKE_CASE     				=  3
									_SCREAMING_SNAKE_CASE     				=  input_dict['''input_ids''']
									_SCREAMING_SNAKE_CASE     				=  input_ids.ne(1					).to(__A					)
									_SCREAMING_SNAKE_CASE     				=  ids_tensor([self.model_tester.batch_size]				,     self.model_tester.type_sequence_label_size					)
									_SCREAMING_SNAKE_CASE     				=  FalconForSequenceClassification(__A					)
									model.to(__A					)
									model.eval()
									_SCREAMING_SNAKE_CASE     				=  model(__A				,     attention_mask=__A				,     labels=__A					)
									self.assertEqual(result.logits.shape				,     (self.model_tester.batch_size, self.model_tester.num_labels)					)







			def 							UpperCamelCase				(     self:  Optional[int]					) ->    List[Any]:







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  self.model_tester.prepare_config_and_inputs_for_common()
									_SCREAMING_SNAKE_CASE     				=  3
									_SCREAMING_SNAKE_CASE     				=  '''single_label_classification'''
									_SCREAMING_SNAKE_CASE     				=  input_dict['''input_ids''']
									_SCREAMING_SNAKE_CASE     				=  input_ids.ne(1					).to(__A					)
									_SCREAMING_SNAKE_CASE     				=  ids_tensor([self.model_tester.batch_size]				,     self.model_tester.type_sequence_label_size					)
									_SCREAMING_SNAKE_CASE     				=  FalconForSequenceClassification(__A					)
									model.to(__A					)
									model.eval()
									_SCREAMING_SNAKE_CASE     				=  model(__A				,     attention_mask=__A				,     labels=__A					)
									self.assertEqual(result.logits.shape				,     (self.model_tester.batch_size, self.model_tester.num_labels)					)







			def 							UpperCamelCase				(     self:  Any					) ->    Tuple:







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  self.model_tester.prepare_config_and_inputs_for_common()
									_SCREAMING_SNAKE_CASE     				=  input_dict['''input_ids''']
									_SCREAMING_SNAKE_CASE     				=  FalconForCausalLM(__A					)
									model.to(__A					)
									model.eval()
									_SCREAMING_SNAKE_CASE     				=  model(__A				,     use_cache=__A					)
									_SCREAMING_SNAKE_CASE     				=  input_ids.shape[0]
									_SCREAMING_SNAKE_CASE     				=  model._convert_to_rw_cache(result.past_key_values					)
									_SCREAMING_SNAKE_CASE     				=  model._convert_cache_to_standard_format(__A				,     __A					)
									for layer in range(len(__A					)					):
															for tensor_idx in range(2					):
																					self.assertTrue(rw_cache[layer][tensor_idx].ndim == 3					)
																					self.assertTrue(result.past_key_values[layer][tensor_idx].ndim == 4					)
																					self.assertTrue(
																					    torch.all(result.past_key_values[layer][tensor_idx] == standard_cache[layer][tensor_idx]					)					)







			def 							UpperCamelCase				(     self:  Union[str, Any]					) ->    List[str]:







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  self.model_tester.prepare_config_and_inputs_for_common()
									_SCREAMING_SNAKE_CASE     				=  3
									_SCREAMING_SNAKE_CASE     				=  '''multi_label_classification'''
									_SCREAMING_SNAKE_CASE     				=  input_dict['''input_ids''']
									_SCREAMING_SNAKE_CASE     				=  input_ids.ne(1					).to(__A					)
									_SCREAMING_SNAKE_CASE     				=  ids_tensor(
									    [self.model_tester.batch_size, config.num_labels]				,     self.model_tester.type_sequence_label_size					).to(torch.float					)
									_SCREAMING_SNAKE_CASE     				=  FalconForSequenceClassification(__A					)
									model.to(__A					)
									model.eval()
									_SCREAMING_SNAKE_CASE     				=  model(__A				,     attention_mask=__A				,     labels=__A					)
									self.assertEqual(result.logits.shape				,     (self.model_tester.batch_size, self.model_tester.num_labels)					)







			def 							UpperCamelCase				(     self:  Optional[Any]					) ->    Optional[int]:







									'''simple docstring'''




									for model_class in self.all_generative_model_classes:
															_SCREAMING_SNAKE_CASE     				=  self.model_tester.prepare_config_and_inputs_for_common()

															# If it doesn't support cache, pass the test
															if not hasattr(__A				,     """use_cache"""					):
																					return

															_SCREAMING_SNAKE_CASE     				=  model_class(__A					).to(__A					)
															if "use_cache" not in inputs:
																					_SCREAMING_SNAKE_CASE     				=  True
															_SCREAMING_SNAKE_CASE     				=  model(**__A					)

															# If "past_key_values" is not returned, pass the test (e.g. RWKV uses a different cache name and format)
															if "past_key_values" not in outputs:
																					return

															_SCREAMING_SNAKE_CASE     				=  (
															    getattr(__A				,     """decoder_layers"""				,     __A					)
															    or getattr(__A				,     """num_decoder_layers"""				,     __A					)
															    or config.num_hidden_layers
															)
															_SCREAMING_SNAKE_CASE     				=  getattr(__A				,     """num_kv_heads"""				,     config.num_attention_heads					)
															_SCREAMING_SNAKE_CASE     				=  getattr(__A				,     """d_model"""				,     config.hidden_size					)
															_SCREAMING_SNAKE_CASE     				=  embed_dim // num_attention_heads

															_SCREAMING_SNAKE_CASE     				=  outputs['''past_key_values''']
															self.assertEqual(len(__A					)				,     __A					)

															_SCREAMING_SNAKE_CASE     				=  inputs['''input_ids'''].shape
															for i in range(__A					):
																					if config.new_decoder_architecture:
																											_SCREAMING_SNAKE_CASE     				=  config.num_attention_heads
																					elif config.multi_query:
																											_SCREAMING_SNAKE_CASE     				=  1
																					self.assertEqual(len(past_kv[0]					)				,     2					)  # K V for the decoder = 2
																					self.assertEqual(
																					    past_kv[i][0].shape				,     (batch_size, num_attention_heads, seq_length, per_head_embed_dim)					)
																					self.assertEqual(
																					    past_kv[i][1].shape				,     (batch_size, num_attention_heads, seq_length, per_head_embed_dim)					)



@require_torch
class 				__UpperCAmelCase		(unittest.TestCase   ):







			@slow
			def 							UpperCamelCase				(     self:  Union[str, Any]					) ->    Any:







									'''simple docstring'''




									_SCREAMING_SNAKE_CASE     				=  AutoTokenizer.from_pretrained("""Rocketknight1/falcon-rw-1b"""					)
									_SCREAMING_SNAKE_CASE     				=  FalconForCausalLM.from_pretrained("""Rocketknight1/falcon-rw-1b"""					)
									model.eval()
									model.to(__A					)
									_SCREAMING_SNAKE_CASE     				=  tokenizer("""My favorite food is"""				,     return_tensors="""pt"""					).to(__A					)

									_SCREAMING_SNAKE_CASE     				=  (
									    '''My favorite food is pizza. I love it so much that I have a pizza party every year for my birthday.'''
									)

									_SCREAMING_SNAKE_CASE     				=  model.generate(**__A				,     do_sample=__A				,     max_new_tokens=19					)
									_SCREAMING_SNAKE_CASE     				=  tokenizer.batch_decode(__A					)[0]

									self.assertEqual(__A				,     __A					)







			@slow
			def 							UpperCamelCase				(     self:  List[Any]					) ->    List[Any]:







									'''simple docstring'''




									for repo in ["Rocketknight1/tiny-random-falcon-7b", "Rocketknight1/tiny-random-falcon-40b"]:
															_SCREAMING_SNAKE_CASE     				=  AutoTokenizer.from_pretrained(__A					)
															_SCREAMING_SNAKE_CASE     				=  FalconForCausalLM.from_pretrained(__A					)
															model.eval()
															model.to(__A					)
															_SCREAMING_SNAKE_CASE     				=  tokenizer("""My favorite food is"""				,     return_tensors="""pt"""					).to(__A					)

															# We just test that these run without errors - the models are randomly initialized
															# and so the actual text outputs will be garbage
															model.generate(**__A				,     do_sample=__A				,     max_new_tokens=4					)
															model.generate(**__A				,     do_sample=__A				,     max_new_tokens=4					)
															model.generate(**__A				,     num_beams=2				,     max_new_tokens=4					)







			@slow
			def 							UpperCamelCase				(     self:  Union[str, Any]					) ->    List[str]:







									'''simple docstring'''




									with torch.no_grad():
															for repo in [
															    "Rocketknight1/falcon-rw-1b",
															    "Rocketknight1/tiny-random-falcon-7b",
															    "Rocketknight1/tiny-random-falcon-40b",
															]:
																					_SCREAMING_SNAKE_CASE     				=  AutoTokenizer.from_pretrained(__A					)
																					_SCREAMING_SNAKE_CASE     				=  FalconForCausalLM.from_pretrained(__A					)
																					model.eval()
																					model.to(device=__A					)
																					_SCREAMING_SNAKE_CASE     				=  tokenizer("""My favorite food is"""				,     return_tensors="""pt"""					).to(__A					)

																					# Test results are the same with and without cache
																					_SCREAMING_SNAKE_CASE     				=  model.generate(**__A				,     do_sample=__A				,     max_new_tokens=20				,     use_cache=__A					)
																					_SCREAMING_SNAKE_CASE     				=  model.generate(**__A				,     do_sample=__A				,     max_new_tokens=20				,     use_cache=__A					)
																					self.assertTrue((outputs_cache - outputs_no_cache).sum().item() == 0					)









from typing import TYPE_CHECKING

from ....utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


UpperCamelCase			    =     {'''configuration_mmbt''': ['''MMBTConfig''']}

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		UpperCamelCase			    =     ['''MMBTForClassification''', '''MMBTModel''', '''ModalEmbeddings''']


if TYPE_CHECKING:
		from .configuration_mmbt import MMBTConfig

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_mmbt import MMBTForClassification, MMBTModel, ModalEmbeddings

else:
		import sys

		UpperCamelCase			    =     _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)

