

import unittest

from transformers import AutoTokenizer, FalconConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
						import torch

						from transformers import (
						    FalconForCausalLM,
						    FalconForQuestionAnswering,
						    FalconForSequenceClassification,
						    FalconForTokenClassification,
						    FalconModel,
						)



class   __lowercase    :




			"""simple docstring"""







			def __init__(			self  :					List[str]       ,						lowerCAmelCase__  :					int       ,						lowerCAmelCase__  :					str=3       ,						lowerCAmelCase__  :					List[str]=7       ,						lowerCAmelCase__  :					Tuple=True       ,						lowerCAmelCase__  :					Optional[int]=True       ,						lowerCAmelCase__  :					Union[str, Any]=False       ,						lowerCAmelCase__  :					Optional[int]=True       ,						lowerCAmelCase__  :					Optional[int]=99       ,						lowerCAmelCase__  :					Dict=32       ,						lowerCAmelCase__  :					Union[str, Any]=5       ,						lowerCAmelCase__  :					List[str]=4       ,						lowerCAmelCase__  :					int=37       ,						lowerCAmelCase__  :					Tuple="gelu"       ,						lowerCAmelCase__  :					Any=0.1       ,						lowerCAmelCase__  :					Union[str, Any]=0.1       ,						lowerCAmelCase__  :					Optional[int]=512       ,						lowerCAmelCase__  :					Tuple=16       ,						lowerCAmelCase__  :					Dict=2       ,						lowerCAmelCase__  :					Optional[int]=0.02       ,						lowerCAmelCase__  :					List[str]=3       ,						lowerCAmelCase__  :					Dict=4       ,						lowerCAmelCase__  :					List[str]=None       ,						):
									SCREAMING_SNAKE_CASE_:			int  		=	parent
									SCREAMING_SNAKE_CASE_:			Optional[int]  		=	batch_size
									SCREAMING_SNAKE_CASE_:			Dict  		=	seq_length
									SCREAMING_SNAKE_CASE_:			Any  		=	is_training
									SCREAMING_SNAKE_CASE_:			Tuple  		=	use_input_mask
									SCREAMING_SNAKE_CASE_:			List[str]  		=	use_token_type_ids
									SCREAMING_SNAKE_CASE_:			Tuple  		=	use_labels
									SCREAMING_SNAKE_CASE_:			int  		=	vocab_size
									SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	hidden_size
									SCREAMING_SNAKE_CASE_:			List[str]  		=	num_hidden_layers
									SCREAMING_SNAKE_CASE_:			int  		=	num_attention_heads
									SCREAMING_SNAKE_CASE_:			Dict  		=	intermediate_size
									SCREAMING_SNAKE_CASE_:			List[Any]  		=	hidden_act
									SCREAMING_SNAKE_CASE_:			Dict  		=	hidden_dropout_prob
									SCREAMING_SNAKE_CASE_:			List[Any]  		=	attention_probs_dropout_prob
									SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	max_position_embeddings
									SCREAMING_SNAKE_CASE_:			List[Any]  		=	type_vocab_size
									SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	type_sequence_label_size
									SCREAMING_SNAKE_CASE_:			Dict  		=	initializer_range
									SCREAMING_SNAKE_CASE_:			Tuple  		=	num_labels
									SCREAMING_SNAKE_CASE_:			int  		=	num_choices
									SCREAMING_SNAKE_CASE_:			Any  		=	scope



			def 	_SCREAMING_SNAKE_CASE      (			self  :					Optional[Any]):
									SCREAMING_SNAKE_CASE_:			List[str]  		=	ids_tensor([self.batch_size, self.seq_length]       ,						self.vocab_size)

									SCREAMING_SNAKE_CASE_:			Any  		=	None
									if self.use_input_mask:
															SCREAMING_SNAKE_CASE_:			Any  		=	random_attention_mask([self.batch_size, self.seq_length])

									SCREAMING_SNAKE_CASE_:			Any  		=	None

									SCREAMING_SNAKE_CASE_:			int  		=	None
									SCREAMING_SNAKE_CASE_:			int  		=	None
									SCREAMING_SNAKE_CASE_:			Any  		=	None
									if self.use_labels:
															SCREAMING_SNAKE_CASE_:			List[str]  		=	ids_tensor([self.batch_size]       ,						self.type_sequence_label_size)
															SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	ids_tensor([self.batch_size, self.seq_length]       ,						self.num_labels)
															SCREAMING_SNAKE_CASE_:			Optional[int]  		=	ids_tensor([self.batch_size]       ,						self.num_choices)

									SCREAMING_SNAKE_CASE_:			int  		=	self.get_config()

									return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels



			def 	_SCREAMING_SNAKE_CASE      (			self  :					Dict):
									return FalconConfig(
									    vocab_size=self.vocab_size       ,						hidden_size=self.hidden_size       ,						num_hidden_layers=self.num_hidden_layers       ,						num_attention_heads=self.num_attention_heads       ,						intermediate_size=self.intermediate_size       ,						hidden_act=self.hidden_act       ,						hidden_dropout_prob=self.hidden_dropout_prob       ,						attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,						max_position_embeddings=self.max_position_embeddings       ,						type_vocab_size=self.type_vocab_size       ,						is_decoder=__a       ,						initializer_range=self.initializer_range       ,						pad_token_id=1       ,						new_decoder_architecture=__a       ,						)



			def 	_SCREAMING_SNAKE_CASE      (			self  :					Tuple       ,						lowerCAmelCase__  :					Optional[int]       ,						lowerCAmelCase__  :					Optional[Any]       ,						lowerCAmelCase__  :					Dict       ,						lowerCAmelCase__  :					str       ,						lowerCAmelCase__  :					Optional[int]       ,						lowerCAmelCase__  :					Tuple       ,						lowerCAmelCase__  :					Optional[Any]):
									SCREAMING_SNAKE_CASE_:			Any  		=	FalconModel(config=__a)
									model.to(__a)
									model.eval()
									SCREAMING_SNAKE_CASE_:			Union[str, Any]  		=	model(__a       ,						attention_mask=__a)
									SCREAMING_SNAKE_CASE_:			str  		=	model(__a)
									self.parent.assertEqual(result.last_hidden_state.shape       ,						(self.batch_size, self.seq_length, self.hidden_size))



			def 	_SCREAMING_SNAKE_CASE      (			self  :					Dict       ,						lowerCAmelCase__  :					Dict       ,						lowerCAmelCase__  :					List[Any]       ,						lowerCAmelCase__  :					Tuple       ,						lowerCAmelCase__  :					Union[str, Any]       ,						lowerCAmelCase__  :					Any       ,						lowerCAmelCase__  :					List[Any]       ,						lowerCAmelCase__  :					Optional[int]       ,						lowerCAmelCase__  :					Tuple       ,						lowerCAmelCase__  :					List[str]       ,						):
									SCREAMING_SNAKE_CASE_:			List[str]  		=	True
									SCREAMING_SNAKE_CASE_:			Any  		=	FalconModel(__a)
									model.to(__a)
									model.eval()
									SCREAMING_SNAKE_CASE_:			Dict  		=	model(
									    __a       ,						attention_mask=__a       ,						encoder_hidden_states=__a       ,						encoder_attention_mask=__a       ,						)
									SCREAMING_SNAKE_CASE_:			Optional[int]  		=	model(
									    __a       ,						attention_mask=__a       ,						encoder_hidden_states=__a       ,						)
									SCREAMING_SNAKE_CASE_:			str  		=	model(__a       ,						attention_mask=__a)
									self.parent.assertEqual(result.last_hidden_state.shape       ,						(self.batch_size, self.seq_length, self.hidden_size))



			def 	_SCREAMING_SNAKE_CASE      (			self  :					str       ,						lowerCAmelCase__  :					Optional[Any]       ,						lowerCAmelCase__  :					Any       ,						lowerCAmelCase__  :					int       ,						lowerCAmelCase__  :					Union[str, Any]       ,						lowerCAmelCase__  :					Optional[int]       ,						lowerCAmelCase__  :					int       ,						lowerCAmelCase__  :					Optional[int]       ,						lowerCAmelCase__  :					Union[str, Any]       ,						lowerCAmelCase__  :					Union[str, Any]       ,						):
									SCREAMING_SNAKE_CASE_:			Dict  		=	FalconForCausalLM(config=__a)
									model.to(__a)
									model.eval()
									SCREAMING_SNAKE_CASE_:			str  		=	model(__a       ,						attention_mask=__a       ,						labels=__a)
									self.parent.assertEqual(result.logits.shape       ,						(self.batch_size, self.seq_length, self.vocab_size))



			def 	_SCREAMING_SNAKE_CASE      (			self  :					Union[str, Any]       ,						lowerCAmelCase__  :					Tuple       ,						lowerCAmelCase__  :					List[Any]       ,						lowerCAmelCase__  :					int       ,						lowerCAmelCase__  :					Union[str, Any]       ,						lowerCAmelCase__  :					Optional[Any]       ,						lowerCAmelCase__  :					Optional[Any]       ,						lowerCAmelCase__  :					Any       ,						lowerCAmelCase__  :					Any       ,						lowerCAmelCase__  :					Tuple       ,						):
									SCREAMING_SNAKE_CASE_:			Union[str, Any]  		=	True
									SCREAMING_SNAKE_CASE_:			int  		=	True
									SCREAMING_SNAKE_CASE_:			Optional[int]  		=	FalconForCausalLM(config=__a)
									model.to(__a)
									model.eval()

									# first forward pass
									SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	model(
									    __a       ,						attention_mask=__a       ,						encoder_hidden_states=__a       ,						encoder_attention_mask=__a       ,						use_cache=__a       ,						)
									SCREAMING_SNAKE_CASE_:			Any  		=	outputs.past_key_values

									# create hypothetical multiple next token and extent to next_input_ids
									SCREAMING_SNAKE_CASE_:			str  		=	ids_tensor((self.batch_size, 3)       ,						config.vocab_size)
									SCREAMING_SNAKE_CASE_:			str  		=	ids_tensor((self.batch_size, 3)       ,						vocab_size=2)

									# append to next input_ids and
									SCREAMING_SNAKE_CASE_:			Union[str, Any]  		=	torch.cat([input_ids, next_tokens]       ,						dim=-1)
									SCREAMING_SNAKE_CASE_:			List[str]  		=	torch.cat([input_mask, next_mask]       ,						dim=-1)

									SCREAMING_SNAKE_CASE_:			Dict  		=	model(
									    __a       ,						attention_mask=__a       ,						encoder_hidden_states=__a       ,						encoder_attention_mask=__a       ,						output_hidden_states=__a       ,						)['hidden_states'][0]
									SCREAMING_SNAKE_CASE_:			Union[str, Any]  		=	model(
									    __a       ,						attention_mask=__a       ,						encoder_hidden_states=__a       ,						encoder_attention_mask=__a       ,						past_key_values=__a       ,						output_hidden_states=__a       ,						)['hidden_states'][0]

									# select random slice
									SCREAMING_SNAKE_CASE_:			str  		=	ids_tensor((1,)       ,						output_from_past.shape[-1]).item()
									SCREAMING_SNAKE_CASE_:			List[str]  		=	output_from_no_past[:, -3:, random_slice_idx].detach()
									SCREAMING_SNAKE_CASE_:			Dict  		=	output_from_past[:, :, random_slice_idx].detach()

									self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1])

									# test that outputs are equal for slice
									self.parent.assertTrue(torch.allclose(__a       ,						__a       ,						atol=1E-3))




			def 	_SCREAMING_SNAKE_CASE      (			self  :					Optional[Any]):
									SCREAMING_SNAKE_CASE_:			Union[str, Any]  		=	self.prepare_config_and_inputs()
									(
									    SCREAMING_SNAKE_CASE_      
									):			List[Any]  		=	config_and_inputs
									SCREAMING_SNAKE_CASE_:			Any  		=	{'input_ids': input_ids, 'attention_mask': input_mask}
									return config, inputs_dict







@require_torch
class   __lowercase    (			SCREAMING_SNAKE_CASE_		,	SCREAMING_SNAKE_CASE_		,	SCREAMING_SNAKE_CASE_		,	unittest.TestCase     ):




			"""simple docstring"""







			_UpperCAmelCase       :							Dict      							=					(
			    (
			        FalconModel,
			        FalconForCausalLM,
			        FalconForSequenceClassification,
			        FalconForTokenClassification,
			        FalconForQuestionAnswering,
			    )
			    if is_torch_available()
			    else ()
			)
			_UpperCAmelCase       :							Union[str, Any]      							=					(FalconForCausalLM,) if is_torch_available() else ()
			_UpperCAmelCase       :							Any      							=					(
			    {
			        '''feature-extraction''': FalconModel,
			        '''text-classification''': FalconForSequenceClassification,
			        '''text-generation''': FalconForCausalLM,
			        '''question-answering''': FalconForQuestionAnswering,
			        '''token-classification''': FalconForTokenClassification,
			        '''zero-shot''': FalconForSequenceClassification,
			    }
			    if is_torch_available()
			    else {}
			)
			_UpperCAmelCase       :							List[Any]      							=					False
			_UpperCAmelCase       :							int      							=					False



			def 	_SCREAMING_SNAKE_CASE      (			self  :					Any):
									SCREAMING_SNAKE_CASE_:			Tuple  		=	FalconModelTester(self)
									SCREAMING_SNAKE_CASE_:			List[Any]  		=	ConfigTester(self       ,						config_class=__a       ,						hidden_size=37)



			def 	_SCREAMING_SNAKE_CASE      (			self  :					Any):
									self.config_tester.run_common_tests()



			def 	_SCREAMING_SNAKE_CASE      (			self  :					str):
									SCREAMING_SNAKE_CASE_:			Union[str, Any]  		=	self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_model(*__a)



			def 	_SCREAMING_SNAKE_CASE      (			self  :					Optional[Any]):
									SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	self.model_tester.prepare_config_and_inputs()
									for alibi in [True, False]:
															SCREAMING_SNAKE_CASE_:			List[Any]  		=	alibi
															self.model_tester.create_and_check_model(__a       ,						*__a)



			def 	_SCREAMING_SNAKE_CASE      (			self  :					Any):
									SCREAMING_SNAKE_CASE_:			int  		=	self.model_tester.prepare_config_and_inputs_for_common()
									SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	3
									SCREAMING_SNAKE_CASE_:			str  		=	input_dict['input_ids']
									SCREAMING_SNAKE_CASE_:			Dict  		=	input_ids.ne(1).to(__a)
									SCREAMING_SNAKE_CASE_:			Tuple  		=	ids_tensor([self.model_tester.batch_size]       ,						self.model_tester.type_sequence_label_size)
									SCREAMING_SNAKE_CASE_:			Any  		=	FalconForSequenceClassification(__a)
									model.to(__a)
									model.eval()
									SCREAMING_SNAKE_CASE_:			List[str]  		=	model(__a       ,						attention_mask=__a       ,						labels=__a)
									self.assertEqual(result.logits.shape       ,						(self.model_tester.batch_size, self.model_tester.num_labels))



			def 	_SCREAMING_SNAKE_CASE      (			self  :					Optional[Any]):
									SCREAMING_SNAKE_CASE_:			int  		=	self.model_tester.prepare_config_and_inputs_for_common()
									SCREAMING_SNAKE_CASE_:			str  		=	3
									SCREAMING_SNAKE_CASE_:			int  		=	'single_label_classification'
									SCREAMING_SNAKE_CASE_:			Tuple  		=	input_dict['input_ids']
									SCREAMING_SNAKE_CASE_:			Dict  		=	input_ids.ne(1).to(__a)
									SCREAMING_SNAKE_CASE_:			List[Any]  		=	ids_tensor([self.model_tester.batch_size]       ,						self.model_tester.type_sequence_label_size)
									SCREAMING_SNAKE_CASE_:			List[Any]  		=	FalconForSequenceClassification(__a)
									model.to(__a)
									model.eval()
									SCREAMING_SNAKE_CASE_:			Dict  		=	model(__a       ,						attention_mask=__a       ,						labels=__a)
									self.assertEqual(result.logits.shape       ,						(self.model_tester.batch_size, self.model_tester.num_labels))



			def 	_SCREAMING_SNAKE_CASE      (			self  :					int):
									SCREAMING_SNAKE_CASE_:			Dict  		=	self.model_tester.prepare_config_and_inputs_for_common()
									SCREAMING_SNAKE_CASE_:			List[Any]  		=	input_dict['input_ids']
									SCREAMING_SNAKE_CASE_:			Union[str, Any]  		=	FalconForCausalLM(__a)
									model.to(__a)
									model.eval()
									SCREAMING_SNAKE_CASE_:			List[Any]  		=	model(__a       ,						use_cache=__a)
									SCREAMING_SNAKE_CASE_:			int  		=	input_ids.shape[0]
									SCREAMING_SNAKE_CASE_:			Tuple  		=	model._convert_to_rw_cache(result.past_key_values)
									SCREAMING_SNAKE_CASE_:			Any  		=	model._convert_cache_to_standard_format(__a       ,						__a)
									for layer in range(len(__a)):
															for tensor_idx in range(2):
																					self.assertTrue(rw_cache[layer][tensor_idx].ndim == 3)
																					self.assertTrue(result.past_key_values[layer][tensor_idx].ndim == 4)
																					self.assertTrue(
																					    torch.all(result.past_key_values[layer][tensor_idx] == standard_cache[layer][tensor_idx]))



			def 	_SCREAMING_SNAKE_CASE      (			self  :					Optional[Any]):
									SCREAMING_SNAKE_CASE_:			Union[str, Any]  		=	self.model_tester.prepare_config_and_inputs_for_common()
									SCREAMING_SNAKE_CASE_:			Union[str, Any]  		=	3
									SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	'multi_label_classification'
									SCREAMING_SNAKE_CASE_:			str  		=	input_dict['input_ids']
									SCREAMING_SNAKE_CASE_:			str  		=	input_ids.ne(1).to(__a)
									SCREAMING_SNAKE_CASE_:			Union[str, Any]  		=	ids_tensor(
									    [self.model_tester.batch_size, config.num_labels]       ,						self.model_tester.type_sequence_label_size).to(torch.float)
									SCREAMING_SNAKE_CASE_:			str  		=	FalconForSequenceClassification(__a)
									model.to(__a)
									model.eval()
									SCREAMING_SNAKE_CASE_:			str  		=	model(__a       ,						attention_mask=__a       ,						labels=__a)
									self.assertEqual(result.logits.shape       ,						(self.model_tester.batch_size, self.model_tester.num_labels))




			def 	_SCREAMING_SNAKE_CASE      (			self  :					Tuple):
									# Falcon can have different numbers of KV-heads than the number of query heads, so we need
									# to override this test to use the right head counts.
									for model_class in self.all_generative_model_classes:
															SCREAMING_SNAKE_CASE_:			Union[str, Any]  		=	self.model_tester.prepare_config_and_inputs_for_common()

															# If it doesn't support cache, pass the test
															if not hasattr(__a       ,						"use_cache"):
																					return

															SCREAMING_SNAKE_CASE_:			Dict  		=	model_class(__a).to(__a)
															if "use_cache" not in inputs:
																					SCREAMING_SNAKE_CASE_:			str  		=	True
															SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	model(**__a)

															# If "past_key_values" is not returned, pass the test (e.g. RWKV uses a different cache name and format)
															if "past_key_values" not in outputs:
																					return

															SCREAMING_SNAKE_CASE_:			Tuple  		=	(
															    getattr(__a       ,						"decoder_layers"       ,						__a)
															    or getattr(__a       ,						"num_decoder_layers"       ,						__a)
															    or config.num_hidden_layers
															)
															SCREAMING_SNAKE_CASE_:			Optional[int]  		=	getattr(__a       ,						"num_kv_heads"       ,						config.num_attention_heads)
															SCREAMING_SNAKE_CASE_:			List[str]  		=	getattr(__a       ,						"d_model"       ,						config.hidden_size)
															SCREAMING_SNAKE_CASE_:			List[str]  		=	embed_dim // num_attention_heads

															SCREAMING_SNAKE_CASE_:			Optional[int]  		=	outputs['past_key_values']
															self.assertEqual(len(__a)       ,						__a)

															SCREAMING_SNAKE_CASE_:			int  		=	inputs['input_ids'].shape
															for i in range(__a):
																					if config.new_decoder_architecture:
																											SCREAMING_SNAKE_CASE_:			str  		=	config.num_attention_heads
																					elif config.multi_query:
																											SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	1
																					self.assertEqual(len(past_kv[0])       ,						2)  # K V for the decoder = 2
																					self.assertEqual(
																					    past_kv[i][0].shape       ,						(batch_size, num_attention_heads, seq_length, per_head_embed_dim))
																					self.assertEqual(
																					    past_kv[i][1].shape       ,						(batch_size, num_attention_heads, seq_length, per_head_embed_dim))







@require_torch
class   __lowercase    (			unittest.TestCase     ):




			"""simple docstring"""







			@slow
			def 	_SCREAMING_SNAKE_CASE      (			self  :					str):
									SCREAMING_SNAKE_CASE_:			Dict  		=	AutoTokenizer.from_pretrained("Rocketknight1/falcon-rw-1b")
									SCREAMING_SNAKE_CASE_:			Union[str, Any]  		=	FalconForCausalLM.from_pretrained("Rocketknight1/falcon-rw-1b")
									model.eval()
									model.to(__a)
									SCREAMING_SNAKE_CASE_:			int  		=	tokenizer("My favorite food is"       ,						return_tensors="pt").to(__a)

									SCREAMING_SNAKE_CASE_:			List[str]  		=	(
									    'My favorite food is pizza. I love it so much that I have a pizza party every year for my birthday.'
									)

									SCREAMING_SNAKE_CASE_:			int  		=	model.generate(**__a       ,						do_sample=__a       ,						max_new_tokens=19)
									SCREAMING_SNAKE_CASE_:			Union[str, Any]  		=	tokenizer.batch_decode(__a)[0]

									self.assertEqual(__a       ,						__a)



			@slow
			def 	_SCREAMING_SNAKE_CASE      (			self  :					int):
									# The big models are way too big for the CI, so we use tiny random models that resemble their
									# architectures but with much smaller and fewer layers
									for repo in ["Rocketknight1/tiny-random-falcon-7b", "Rocketknight1/tiny-random-falcon-40b"]:
															SCREAMING_SNAKE_CASE_:			Optional[int]  		=	AutoTokenizer.from_pretrained(__a)
															SCREAMING_SNAKE_CASE_:			str  		=	FalconForCausalLM.from_pretrained(__a)
															model.eval()
															model.to(__a)
															SCREAMING_SNAKE_CASE_:			int  		=	tokenizer("My favorite food is"       ,						return_tensors="pt").to(__a)

															# We just test that these run without errors - the models are randomly initialized
															# and so the actual text outputs will be garbage
															model.generate(**__a       ,						do_sample=__a       ,						max_new_tokens=4)
															model.generate(**__a       ,						do_sample=__a       ,						max_new_tokens=4)
															model.generate(**__a       ,						num_beams=2       ,						max_new_tokens=4)




			@slow
			def 	_SCREAMING_SNAKE_CASE      (			self  :					List[Any]):
									# The big models are way too big for the CI, so we use tiny random models that resemble their
									# architectures but with much smaller and fewer layers
									with torch.no_grad():
															for repo in [
															    "Rocketknight1/falcon-rw-1b",
															    "Rocketknight1/tiny-random-falcon-7b",
															    "Rocketknight1/tiny-random-falcon-40b",
															]:
																					SCREAMING_SNAKE_CASE_:			Union[str, Any]  		=	AutoTokenizer.from_pretrained(__a)
																					SCREAMING_SNAKE_CASE_:			List[str]  		=	FalconForCausalLM.from_pretrained(__a)
																					model.eval()
																					model.to(device=__a)
																					SCREAMING_SNAKE_CASE_:			Optional[int]  		=	tokenizer("My favorite food is"       ,						return_tensors="pt").to(__a)

																					# Test results are the same with and without cache
																					SCREAMING_SNAKE_CASE_:			Dict  		=	model.generate(**__a       ,						do_sample=__a       ,						max_new_tokens=20       ,						use_cache=__a)
																					SCREAMING_SNAKE_CASE_:			str  		=	model.generate(**__a       ,						do_sample=__a       ,						max_new_tokens=20       ,						use_cache=__a)
																					self.assertTrue((outputs_cache - outputs_no_cache).sum().item() == 0)









import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, PLBartTokenizer, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


lowerCAmelCase							:  Any            =						get_tests_dir("""fixtures/test_sentencepiece.model""")


if is_torch_available():
						from transformers.models.plbart.modeling_plbart import shift_tokens_right

lowerCAmelCase							:  Optional[Any]            =						50003
lowerCAmelCase							:  List[str]            =						50002



@require_sentencepiece
@require_tokenizers
class   __lowercase    (			UpperCAmelCase_		,	unittest.TestCase     ):




			"""simple docstring"""







			_UpperCAmelCase       :							int      							=					PLBartTokenizer
			_UpperCAmelCase       :							Any      							=					None
			_UpperCAmelCase       :							Optional[Any]      							=					False



			def 	_SCREAMING_SNAKE_CASE      (			self  :					str):
									super().setUp()

									# We have a SentencePiece fixture for testing
									SCREAMING_SNAKE_CASE_:			Tuple  		=	PLBartTokenizer(lowerCAmelCase__       ,						language_codes="base"       ,						keep_accents=lowerCAmelCase__)
									tokenizer.save_pretrained(self.tmpdirname)



			def 	_SCREAMING_SNAKE_CASE      (			self  :					str):
									SCREAMING_SNAKE_CASE_:			Optional[int]  		=	PLBartTokenizer(lowerCAmelCase__       ,						language_codes="base"       ,						keep_accents=lowerCAmelCase__)

									SCREAMING_SNAKE_CASE_:			Dict  		=	tokenizer.tokenize("This is a test")
									self.assertListEqual(lowerCAmelCase__       ,						["▁This", "▁is", "▁a", "▁t", "est"])

									self.assertListEqual(
									    tokenizer.convert_tokens_to_ids(lowerCAmelCase__)       ,						[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]       ,						)

									SCREAMING_SNAKE_CASE_:			List[Any]  		=	tokenizer.tokenize("I was born in 92000, and this is falsé.")
									self.assertListEqual(
									    lowerCAmelCase__       ,						[
									        SPIECE_UNDERLINE + "I",
									        SPIECE_UNDERLINE + "was",
									        SPIECE_UNDERLINE + "b",
									        "or",
									        "n",
									        SPIECE_UNDERLINE + "in",
									        SPIECE_UNDERLINE + "",
									        "9",
									        "2",
									        "0",
									        "0",
									        "0",
									        ",",
									        SPIECE_UNDERLINE + "and",
									        SPIECE_UNDERLINE + "this",
									        SPIECE_UNDERLINE + "is",
									        SPIECE_UNDERLINE + "f",
									        "al",
									        "s",
									        "é",
									        ".",
									    ]       ,						)
									SCREAMING_SNAKE_CASE_:			str  		=	tokenizer.convert_tokens_to_ids(lowerCAmelCase__)
									self.assertListEqual(
									    lowerCAmelCase__       ,						[
									        value + tokenizer.fairseq_offset
									        for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
									    ]       ,						)

									SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	tokenizer.convert_ids_to_tokens(lowerCAmelCase__)
									self.assertListEqual(
									    lowerCAmelCase__       ,						[
									        SPIECE_UNDERLINE + "I",
									        SPIECE_UNDERLINE + "was",
									        SPIECE_UNDERLINE + "b",
									        "or",
									        "n",
									        SPIECE_UNDERLINE + "in",
									        SPIECE_UNDERLINE + "",
									        "<unk>",
									        "2",
									        "0",
									        "0",
									        "0",
									        ",",
									        SPIECE_UNDERLINE + "and",
									        SPIECE_UNDERLINE + "this",
									        SPIECE_UNDERLINE + "is",
									        SPIECE_UNDERLINE + "f",
									        "al",
									        "s",
									        "<unk>",
									        ".",
									    ]       ,						)

									SCREAMING_SNAKE_CASE_:			int  		=	tokenizer.vocab_size
									SCREAMING_SNAKE_CASE_:			Tuple  		=	[tokenizer.convert_ids_to_tokens(lowerCAmelCase__) for x in range(end - 4       ,						lowerCAmelCase__)]

									self.assertListEqual(lowerCAmelCase__       ,						["__java__", "__python__", "__en_XX__", "<mask>"])

									SCREAMING_SNAKE_CASE_:			Optional[int]  		=	"java.lang.Exception, python.lang.Exception, javascript, php, ruby, go"
									SCREAMING_SNAKE_CASE_:			Optional[int]  		=	tokenizer(lowerCAmelCase__).input_ids
									self.assertEqual(
									    tokenizer.decode(lowerCAmelCase__       ,						skip_special_tokens=lowerCAmelCase__       ,						clean_up_tokenization_spaces=lowerCAmelCase__)       ,						lowerCAmelCase__       ,						)




			def 	_SCREAMING_SNAKE_CASE      (			self  :					Union[str, Any]):
									SCREAMING_SNAKE_CASE_:			Union[str, Any]  		=	PLBartTokenizer(lowerCAmelCase__       ,						language_codes="multi"       ,						keep_accents=lowerCAmelCase__)

									SCREAMING_SNAKE_CASE_:			str  		=	tokenizer.tokenize("This is a test")
									self.assertListEqual(lowerCAmelCase__       ,						["▁This", "▁is", "▁a", "▁t", "est"])

									self.assertListEqual(
									    tokenizer.convert_tokens_to_ids(lowerCAmelCase__)       ,						[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]       ,						)

									SCREAMING_SNAKE_CASE_:			Tuple  		=	tokenizer.tokenize("I was born in 92000, and this is falsé.")
									self.assertListEqual(
									    lowerCAmelCase__       ,						[
									        SPIECE_UNDERLINE + "I",
									        SPIECE_UNDERLINE + "was",
									        SPIECE_UNDERLINE + "b",
									        "or",
									        "n",
									        SPIECE_UNDERLINE + "in",
									        SPIECE_UNDERLINE + "",
									        "9",
									        "2",
									        "0",
									        "0",
									        "0",
									        ",",
									        SPIECE_UNDERLINE + "and",
									        SPIECE_UNDERLINE + "this",
									        SPIECE_UNDERLINE + "is",
									        SPIECE_UNDERLINE + "f",
									        "al",
									        "s",
									        "é",
									        ".",
									    ]       ,						)
									SCREAMING_SNAKE_CASE_:			Optional[int]  		=	tokenizer.convert_tokens_to_ids(lowerCAmelCase__)
									self.assertListEqual(
									    lowerCAmelCase__       ,						[
									        value + tokenizer.fairseq_offset
									        for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
									    ]       ,						)

									SCREAMING_SNAKE_CASE_:			Optional[int]  		=	tokenizer.convert_ids_to_tokens(lowerCAmelCase__)
									self.assertListEqual(
									    lowerCAmelCase__       ,						[
									        SPIECE_UNDERLINE + "I",
									        SPIECE_UNDERLINE + "was",
									        SPIECE_UNDERLINE + "b",
									        "or",
									        "n",
									        SPIECE_UNDERLINE + "in",
									        SPIECE_UNDERLINE + "",
									        "<unk>",
									        "2",
									        "0",
									        "0",
									        "0",
									        ",",
									        SPIECE_UNDERLINE + "and",
									        SPIECE_UNDERLINE + "this",
									        SPIECE_UNDERLINE + "is",
									        SPIECE_UNDERLINE + "f",
									        "al",
									        "s",
									        "<unk>",
									        ".",
									    ]       ,						)
									SCREAMING_SNAKE_CASE_:			Optional[int]  		=	tokenizer.vocab_size
									SCREAMING_SNAKE_CASE_:			int  		=	[tokenizer.convert_ids_to_tokens(lowerCAmelCase__) for x in range(end - 7       ,						lowerCAmelCase__)]

									self.assertListEqual(
									    lowerCAmelCase__       ,						["__java__", "__python__", "__en_XX__", "__javascript__", "__php__", "__ruby__", "__go__"])
									SCREAMING_SNAKE_CASE_:			str  		=	"java.lang.Exception, python.lang.Exception, javascript, php, ruby, go"
									SCREAMING_SNAKE_CASE_:			Tuple  		=	tokenizer(lowerCAmelCase__).input_ids
									self.assertEqual(
									    tokenizer.decode(lowerCAmelCase__       ,						skip_special_tokens=lowerCAmelCase__       ,						clean_up_tokenization_spaces=lowerCAmelCase__)       ,						lowerCAmelCase__       ,						)







@require_torch
@require_sentencepiece
@require_tokenizers
class   __lowercase    (			unittest.TestCase     ):




			"""simple docstring"""







			_UpperCAmelCase       :							Optional[Any]      							=					'''uclanlp/plbart-python-en_XX'''
			_UpperCAmelCase       :							List[str]      							=					[
			    '''def maximum(a,b,c):NEW_LINE_INDENTreturn max([a,b,c])''',
			    '''def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])''',
			]
			_UpperCAmelCase       :							int      							=					[
			    '''Returns the maximum value of a b c.''',
			    '''Sums the values of a b c.''',
			]
			_UpperCAmelCase       :							Optional[Any]      							=					[
			    134,
			    5452,
			    3_3460,
			    3_3441,
			    3_3463,
			    3_3465,
			    3_3463,
			    3_3449,
			    988,
			    20,
			    3_3456,
			    19,
			    3_3456,
			    771,
			    39,
			    4258,
			    889,
			    3318,
			    3_3441,
			    3_3463,
			    3_3465,
			    3_3463,
			    3_3449,
			    2471,
			    2,
			    PYTHON_CODE,
			]



			@classmethod
			def 	_SCREAMING_SNAKE_CASE      (			cls  :					Dict):
									SCREAMING_SNAKE_CASE_:			PLBartTokenizer  		=	PLBartTokenizer.from_pretrained(
									    cls.checkpoint_name       ,						language_codes="base"       ,						src_lang="python"       ,						tgt_lang="en_XX")
									SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	1
									return cls



			def 	_SCREAMING_SNAKE_CASE      (			self  :					List[str]):
									self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["__java__"]       ,						5_0001)
									self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["__python__"]       ,						5_0002)
									self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["__en_XX__"]       ,						5_0003)



			def 	_SCREAMING_SNAKE_CASE      (			self  :					Optional[Any]):
									SCREAMING_SNAKE_CASE_:			Dict  		=	self.tokenizer.batch_encode_plus(self.src_text).input_ids[0]
									self.assertListEqual(self.expected_src_tokens       ,						lowerCAmelCase__)



			def 	_SCREAMING_SNAKE_CASE      (			self  :					Dict):
									self.assertIn(lowerCAmelCase__       ,						self.tokenizer.all_special_ids)
									SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	[EN_CODE, 9037, 3_3442, 57, 752, 153, 14, 56, 18, 9, 2]
									SCREAMING_SNAKE_CASE_:			int  		=	self.tokenizer.decode(lowerCAmelCase__       ,						skip_special_tokens=lowerCAmelCase__)
									SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	self.tokenizer.decode(generated_ids[1:]       ,						skip_special_tokens=lowerCAmelCase__)
									self.assertEqual(lowerCAmelCase__       ,						lowerCAmelCase__)
									self.assertNotIn(self.tokenizer.eos_token       ,						lowerCAmelCase__)



			def 	_SCREAMING_SNAKE_CASE      (			self  :					str):
									SCREAMING_SNAKE_CASE_:			List[str]  		=	["def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])" * 20]
									self.assertIsInstance(src_text[0]       ,						lowerCAmelCase__)
									SCREAMING_SNAKE_CASE_:			List[Any]  		=	10
									SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	self.tokenizer(lowerCAmelCase__       ,						max_length=lowerCAmelCase__       ,						truncation=lowerCAmelCase__).input_ids[0]
									self.assertEqual(ids[-2]       ,						2)
									self.assertEqual(ids[-1]       ,						lowerCAmelCase__)
									self.assertEqual(len(lowerCAmelCase__)       ,						lowerCAmelCase__)



			def 	_SCREAMING_SNAKE_CASE      (			self  :					Tuple):
									self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["<mask>", "__java__"])       ,						[5_0004, 5_0001])



			def 	_SCREAMING_SNAKE_CASE      (			self  :					str):
									SCREAMING_SNAKE_CASE_:			List[Any]  		=	tempfile.mkdtemp()
									SCREAMING_SNAKE_CASE_:			Optional[int]  		=	self.tokenizer.fairseq_tokens_to_ids
									self.tokenizer.save_pretrained(lowerCAmelCase__)
									SCREAMING_SNAKE_CASE_:			int  		=	PLBartTokenizer.from_pretrained(lowerCAmelCase__)
									self.assertDictEqual(new_tok.fairseq_tokens_to_ids       ,						lowerCAmelCase__)



			@require_torch
			def 	_SCREAMING_SNAKE_CASE      (			self  :					int):
									SCREAMING_SNAKE_CASE_:			List[Any]  		=	self.tokenizer(self.src_text       ,						text_target=self.tgt_text       ,						padding=lowerCAmelCase__       ,						return_tensors="pt")
									SCREAMING_SNAKE_CASE_:			Any  		=	shift_tokens_right(batch["labels"]       ,						self.tokenizer.pad_token_id)

									# fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
									self.assertEqual(batch.input_ids[1][-2:].tolist()       ,						[2, PYTHON_CODE])
									self.assertEqual(batch.decoder_input_ids[1][0]       ,						lowerCAmelCase__)
									self.assertEqual(batch.decoder_input_ids[1][-1]       ,						2)
									self.assertEqual(batch.labels[1][-2:].tolist()       ,						[2, EN_CODE])



			@require_torch
			def 	_SCREAMING_SNAKE_CASE      (			self  :					str):
									SCREAMING_SNAKE_CASE_:			Optional[int]  		=	self.tokenizer(
									    self.src_text       ,						text_target=self.tgt_text       ,						padding=lowerCAmelCase__       ,						truncation=lowerCAmelCase__       ,						max_length=len(self.expected_src_tokens)       ,						return_tensors="pt"       ,						)
									SCREAMING_SNAKE_CASE_:			Union[str, Any]  		=	shift_tokens_right(batch["labels"]       ,						self.tokenizer.pad_token_id)

									self.assertIsInstance(lowerCAmelCase__       ,						lowerCAmelCase__)

									self.assertEqual((2, 26)       ,						batch.input_ids.shape)
									self.assertEqual((2, 26)       ,						batch.attention_mask.shape)
									SCREAMING_SNAKE_CASE_:			Optional[Any]  		=	batch.input_ids.tolist()[0]
									self.assertListEqual(self.expected_src_tokens       ,						lowerCAmelCase__)
									self.assertEqual(2       ,						batch.decoder_input_ids[0, -1])  # EOS
									# Test that special tokens are reset
									self.assertEqual(self.tokenizer.prefix_tokens       ,						[])
									self.assertEqual(self.tokenizer.suffix_tokens       ,						[self.tokenizer.eos_token_id, PYTHON_CODE])



			def 	_SCREAMING_SNAKE_CASE      (			self  :					Optional[Any]):
									SCREAMING_SNAKE_CASE_:			List[Any]  		=	self.tokenizer(self.src_text       ,						padding=lowerCAmelCase__       ,						truncation=lowerCAmelCase__       ,						max_length=3       ,						return_tensors="pt")
									SCREAMING_SNAKE_CASE_:			Any  		=	self.tokenizer(
									    text_target=self.tgt_text       ,						padding=lowerCAmelCase__       ,						truncation=lowerCAmelCase__       ,						max_length=10       ,						return_tensors="pt")
									SCREAMING_SNAKE_CASE_:			List[Any]  		=	targets["input_ids"]
									SCREAMING_SNAKE_CASE_:			List[Any]  		=	shift_tokens_right(lowerCAmelCase__       ,						self.tokenizer.pad_token_id)

									self.assertEqual(batch.input_ids.shape[1]       ,						3)
									self.assertEqual(batch.decoder_input_ids.shape[1]       ,						10)




			@require_torch
			def 	_SCREAMING_SNAKE_CASE      (			self  :					int):
									SCREAMING_SNAKE_CASE_:			List[str]  		=	self.tokenizer._build_translation_inputs(
									    "A test"       ,						return_tensors="pt"       ,						src_lang="en_XX"       ,						tgt_lang="java")

									self.assertEqual(
									    nested_simplify(lowerCAmelCase__)       ,						{
									        # A, test, EOS, en_XX
									        "input_ids": [[150, 242, 2, 5_0003]],
									        "attention_mask": [[1, 1, 1, 1]],
									        # java
									        "forced_bos_token_id": 5_0001,
									    }       ,						)






