



import unittest

from parameterized import parameterized

from transformers import OpenLlamaConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import OpenLlamaForCausalLM, OpenLlamaForSequenceClassification, OpenLlamaModel



class 		lowerCamelCase__      :




		'''simple docstring'''



		def __init__(self		,__lowerCamelCase		,__lowerCamelCase=13		,__lowerCamelCase=7		,__lowerCamelCase=True		,__lowerCamelCase=True		,__lowerCamelCase=False		,__lowerCamelCase=True		,__lowerCamelCase=99		,__lowerCamelCase=32		,__lowerCamelCase=5		,__lowerCamelCase=4		,__lowerCamelCase=37		,__lowerCamelCase="gelu"		,__lowerCamelCase=0.1		,__lowerCamelCase=0.1		,__lowerCamelCase=5_12		,__lowerCamelCase=16		,__lowerCamelCase=2		,__lowerCamelCase=0.02		,__lowerCamelCase=3		,__lowerCamelCase=4		,__lowerCamelCase=None		,)					->      Union[str, Any]:

						"""simple docstring"""






						lowerCAmelCase__					:    str												=					parent
						lowerCAmelCase__					:    Dict												=					batch_size
						lowerCAmelCase__					:    Optional[int]												=					seq_length
						lowerCAmelCase__					:    int												=					is_training
						lowerCAmelCase__					:    str												=					use_input_mask
						lowerCAmelCase__					:    List[str]												=					use_token_type_ids
						lowerCAmelCase__					:    Optional[int]												=					use_labels
						lowerCAmelCase__					:    Tuple												=					vocab_size
						lowerCAmelCase__					:    int												=					hidden_size
						lowerCAmelCase__					:    Optional[Any]												=					num_hidden_layers
						lowerCAmelCase__					:    Tuple												=					num_attention_heads
						lowerCAmelCase__					:    List[Any]												=					intermediate_size
						lowerCAmelCase__					:    Union[str, Any]												=					hidden_act
						lowerCAmelCase__					:    List[str]												=					hidden_dropout_prob
						lowerCAmelCase__					:    Union[str, Any]												=					attention_probs_dropout_prob
						lowerCAmelCase__					:    int												=					max_position_embeddings
						lowerCAmelCase__					:    int												=					type_vocab_size
						lowerCAmelCase__					:    Any												=					type_sequence_label_size
						lowerCAmelCase__					:    str												=					initializer_range
						lowerCAmelCase__					:    Optional[Any]												=					num_labels
						lowerCAmelCase__					:    List[str]												=					num_choices
						lowerCAmelCase__					:    int												=					scope



		def 		lowerCAmelCase__				(self			)					->      Union[str, Any]:

						"""simple docstring"""






						lowerCAmelCase__					:    Tuple												=					ids_tensor([self.batch_size, self.seq_length]		,self.vocab_size			)

						lowerCAmelCase__					:    Union[str, Any]												=					None
						if self.use_input_mask:
										lowerCAmelCase__					:    Optional[Any]												=					random_attention_mask([self.batch_size, self.seq_length]			)

						lowerCAmelCase__					:    int												=					None
						if self.use_token_type_ids:
										lowerCAmelCase__					:    int												=					ids_tensor([self.batch_size, self.seq_length]		,self.type_vocab_size			)

						lowerCAmelCase__					:    Optional[int]												=					None
						lowerCAmelCase__					:    Union[str, Any]												=					None
						lowerCAmelCase__					:    List[Any]												=					None
						if self.use_labels:
										lowerCAmelCase__					:    Optional[int]												=					ids_tensor([self.batch_size]		,self.type_sequence_label_size			)
										lowerCAmelCase__					:    List[str]												=					ids_tensor([self.batch_size, self.seq_length]		,self.num_labels			)
										lowerCAmelCase__					:    str												=					ids_tensor([self.batch_size]		,self.num_choices			)

						lowerCAmelCase__					:    Any												=					self.get_config()

						return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels



		def 		lowerCAmelCase__				(self			)					->      Dict:

						"""simple docstring"""






						return OpenLlamaConfig(
						    vocab_size=self.vocab_size		,hidden_size=self.hidden_size		,num_hidden_layers=self.num_hidden_layers		,num_attention_heads=self.num_attention_heads		,intermediate_size=self.intermediate_size		,hidden_act=self.hidden_act		,hidden_dropout_prob=self.hidden_dropout_prob		,attention_probs_dropout_prob=self.attention_probs_dropout_prob		,max_position_embeddings=self.max_position_embeddings		,type_vocab_size=self.type_vocab_size		,is_decoder=lowercase_		,initializer_range=self.initializer_range		,use_stable_embedding=lowercase_		,)



		def 		lowerCAmelCase__				(self		,__lowerCamelCase		,__lowerCamelCase		,__lowerCamelCase		,__lowerCamelCase		,__lowerCamelCase		,__lowerCamelCase		,__lowerCamelCase			)					->      Tuple:

						"""simple docstring"""






						lowerCAmelCase__					:    Tuple												=					OpenLlamaModel(config=lowercase_			)
						model.to(lowercase_			)
						model.eval()
						lowerCAmelCase__					:    Tuple												=					model(lowercase_		,attention_mask=lowercase_			)
						lowerCAmelCase__					:    int												=					model(lowercase_			)
						self.parent.assertEqual(result.last_hidden_state.shape		,(self.batch_size, self.seq_length, self.hidden_size)			)



		def 		lowerCAmelCase__				(self		,__lowerCamelCase		,__lowerCamelCase		,__lowerCamelCase		,__lowerCamelCase		,__lowerCamelCase		,__lowerCamelCase		,__lowerCamelCase		,__lowerCamelCase		,__lowerCamelCase		,)					->      Dict:

						"""simple docstring"""






						lowerCAmelCase__					:    List[Any]												=					True
						lowerCAmelCase__					:    Optional[int]												=					OpenLlamaModel(lowercase_			)
						model.to(lowercase_			)
						model.eval()
						lowerCAmelCase__					:    Optional[Any]												=					model(
						    lowercase_		,attention_mask=lowercase_		,encoder_hidden_states=lowercase_		,encoder_attention_mask=lowercase_		,)
						lowerCAmelCase__					:    Optional[int]												=					model(
						    lowercase_		,attention_mask=lowercase_		,encoder_hidden_states=lowercase_		,)
						lowerCAmelCase__					:    Tuple												=					model(lowercase_		,attention_mask=lowercase_			)
						self.parent.assertEqual(result.last_hidden_state.shape		,(self.batch_size, self.seq_length, self.hidden_size)			)



		def 		lowerCAmelCase__				(self		,__lowerCamelCase		,__lowerCamelCase		,__lowerCamelCase		,__lowerCamelCase		,__lowerCamelCase		,__lowerCamelCase		,__lowerCamelCase		,__lowerCamelCase		,__lowerCamelCase		,)					->      Union[str, Any]:

						"""simple docstring"""






						lowerCAmelCase__					:    str												=					OpenLlamaForCausalLM(config=lowercase_			)
						model.to(lowercase_			)
						model.eval()
						lowerCAmelCase__					:    List[Any]												=					model(lowercase_		,attention_mask=lowercase_		,labels=lowercase_			)
						self.parent.assertEqual(result.logits.shape		,(self.batch_size, self.seq_length, self.vocab_size)			)



		def 		lowerCAmelCase__				(self		,__lowerCamelCase		,__lowerCamelCase		,__lowerCamelCase		,__lowerCamelCase		,__lowerCamelCase		,__lowerCamelCase		,__lowerCamelCase		,__lowerCamelCase		,__lowerCamelCase		,)					->      Union[str, Any]:

						"""simple docstring"""






						lowerCAmelCase__					:    List[Any]												=					True
						lowerCAmelCase__					:    Dict												=					True
						lowerCAmelCase__					:    Optional[int]												=					OpenLlamaForCausalLM(config=lowercase_			)
						model.to(lowercase_			)
						model.eval()

						# first forward pass
						lowerCAmelCase__					:    int												=					model(
						    lowercase_		,attention_mask=lowercase_		,encoder_hidden_states=lowercase_		,encoder_attention_mask=lowercase_		,use_cache=lowercase_		,)
						lowerCAmelCase__					:    int												=					outputs.past_key_values

						# create hypothetical multiple next token and extent to next_input_ids
						lowerCAmelCase__					:    Dict												=					ids_tensor((self.batch_size, 3)		,config.vocab_size			)
						lowerCAmelCase__					:    Tuple												=					ids_tensor((self.batch_size, 3)		,vocab_size=2			)

						# append to next input_ids and
						lowerCAmelCase__					:    Any												=					torch.cat([input_ids, next_tokens]		,dim=-1			)
						lowerCAmelCase__					:    Tuple												=					torch.cat([input_mask, next_mask]		,dim=-1			)

						lowerCAmelCase__					:    Union[str, Any]												=					model(
						    lowercase_		,attention_mask=lowercase_		,encoder_hidden_states=lowercase_		,encoder_attention_mask=lowercase_		,output_hidden_states=lowercase_		,)["hidden_states"][0]
						lowerCAmelCase__					:    List[Any]												=					model(
						    lowercase_		,attention_mask=lowercase_		,encoder_hidden_states=lowercase_		,encoder_attention_mask=lowercase_		,past_key_values=lowercase_		,output_hidden_states=lowercase_		,)["hidden_states"][0]

						# select random slice
						lowerCAmelCase__					:    Tuple												=					ids_tensor((1,)		,output_from_past.shape[-1]			).item()
						lowerCAmelCase__					:    int												=					output_from_no_past[:, -3:, random_slice_idx].detach()
						lowerCAmelCase__					:    Optional[int]												=					output_from_past[:, :, random_slice_idx].detach()

						self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]			)

						# test that outputs are equal for slice
						self.parent.assertTrue(torch.allclose(lowercase_		,lowercase_		,atol=1e-3			)			)



		def 		lowerCAmelCase__				(self			)					->      Tuple:

						"""simple docstring"""






						lowerCAmelCase__					:    List[Any]												=					self.prepare_config_and_inputs()
						(
						    lowerCAmelCase__       
						)					:    Optional[Any]												=					config_and_inputs
						lowerCAmelCase__					:    Optional[Any]												=					{"input_ids": input_ids, "attention_mask": input_mask}
						return config, inputs_dict



@require_torch
class 		lowerCamelCase__      (							lowercase__				,    lowercase__				,    lowercase__				,    unittest.TestCase):




		'''simple docstring'''



		snake_case_     =(
		    (OpenLlamaModel, OpenLlamaForCausalLM, OpenLlamaForSequenceClassification) if is_torch_available() else ()
		)
		snake_case_     =(OpenLlamaForCausalLM,) if is_torch_available() else ()
		snake_case_     =(
		    {
		        """feature-extraction""": OpenLlamaModel,
		        """text-classification""": OpenLlamaForSequenceClassification,
		        """text-generation""": OpenLlamaForCausalLM,
		        """zero-shot""": OpenLlamaForSequenceClassification,
		    }
		    if is_torch_available()
		    else {}
		)
		snake_case_     =False
		snake_case_     =False



		def 		lowerCAmelCase__				(self			)					->      str:

						"""simple docstring"""






						lowerCAmelCase__					:    List[str]												=					OpenLlamaModelTester(self			)
						lowerCAmelCase__					:    Union[str, Any]												=					ConfigTester(self		,config_class=lowercase_		,hidden_size=37			)



		def 		lowerCAmelCase__				(self			)					->      Optional[Any]:

						"""simple docstring"""






						self.config_tester.run_common_tests()



		def 		lowerCAmelCase__				(self			)					->      Any:

						"""simple docstring"""






						lowerCAmelCase__					:    Optional[Any]												=					self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_model(*lowercase_			)



		def 		lowerCAmelCase__				(self			)					->      Optional[Any]:

						"""simple docstring"""






						lowerCAmelCase__					:    Dict												=					self.model_tester.prepare_config_and_inputs()
						for type in ["absolute", "relative_key", "relative_key_query"]:
										lowerCAmelCase__					:    Optional[Any]												=					type
										self.model_tester.create_and_check_model(*lowercase_			)



		def 		lowerCAmelCase__				(self			)					->      str:

						"""simple docstring"""






						lowerCAmelCase__					:    int												=					self.model_tester.prepare_config_and_inputs_for_common()
						lowerCAmelCase__					:    Optional[int]												=					3
						lowerCAmelCase__					:    Optional[Any]												=					input_dict["input_ids"]
						lowerCAmelCase__					:    Union[str, Any]												=					input_ids.ne(1			).to(lowercase_			)
						lowerCAmelCase__					:    Union[str, Any]												=					ids_tensor([self.model_tester.batch_size]		,self.model_tester.type_sequence_label_size			)
						lowerCAmelCase__					:    str												=					OpenLlamaForSequenceClassification(lowercase_			)
						model.to(lowercase_			)
						model.eval()
						lowerCAmelCase__					:    int												=					model(lowercase_		,attention_mask=lowercase_		,labels=lowercase_			)
						self.assertEqual(result.logits.shape		,(self.model_tester.batch_size, self.model_tester.num_labels)			)



		def 		lowerCAmelCase__				(self			)					->      str:

						"""simple docstring"""






						lowerCAmelCase__					:    str												=					self.model_tester.prepare_config_and_inputs_for_common()
						lowerCAmelCase__					:    str												=					3
						lowerCAmelCase__					:    str												=					"single_label_classification"
						lowerCAmelCase__					:    Any												=					input_dict["input_ids"]
						lowerCAmelCase__					:    int												=					input_ids.ne(1			).to(lowercase_			)
						lowerCAmelCase__					:    Union[str, Any]												=					ids_tensor([self.model_tester.batch_size]		,self.model_tester.type_sequence_label_size			)
						lowerCAmelCase__					:    Optional[Any]												=					OpenLlamaForSequenceClassification(lowercase_			)
						model.to(lowercase_			)
						model.eval()
						lowerCAmelCase__					:    List[Any]												=					model(lowercase_		,attention_mask=lowercase_		,labels=lowercase_			)
						self.assertEqual(result.logits.shape		,(self.model_tester.batch_size, self.model_tester.num_labels)			)



		def 		lowerCAmelCase__				(self			)					->      Optional[Any]:

						"""simple docstring"""






						lowerCAmelCase__					:    Any												=					self.model_tester.prepare_config_and_inputs_for_common()
						lowerCAmelCase__					:    int												=					3
						lowerCAmelCase__					:    Union[str, Any]												=					"multi_label_classification"
						lowerCAmelCase__					:    Dict												=					input_dict["input_ids"]
						lowerCAmelCase__					:    Tuple												=					input_ids.ne(1			).to(lowercase_			)
						lowerCAmelCase__					:    Tuple												=					ids_tensor(
						    [self.model_tester.batch_size, config.num_labels]		,self.model_tester.type_sequence_label_size			).to(torch.float			)
						lowerCAmelCase__					:    List[str]												=					OpenLlamaForSequenceClassification(lowercase_			)
						model.to(lowercase_			)
						model.eval()
						lowerCAmelCase__					:    List[Any]												=					model(lowercase_		,attention_mask=lowercase_		,labels=lowercase_			)
						self.assertEqual(result.logits.shape		,(self.model_tester.batch_size, self.model_tester.num_labels)			)



		@unittest.skip('''Open-Llama buffers include complex numbers, which breaks this test'''			)
		def 		lowerCAmelCase__				(self			)					->      Any:

						"""simple docstring"""






						pass



		@parameterized.expand([('''linear''',), ('''dynamic''',)]			)
		def 		lowerCAmelCase__				(self		,__lowerCamelCase			)					->      List[Any]:

						"""simple docstring"""






						lowerCAmelCase__					:    Optional[Any]												=					self.model_tester.prepare_config_and_inputs_for_common()
						lowerCAmelCase__					:    Dict												=					ids_tensor([1, 10]		,config.vocab_size			)
						lowerCAmelCase__					:    List[Any]												=					ids_tensor([1, int(config.max_position_embeddings * 1.5			)]		,config.vocab_size			)

						set_seed(42			)  # Fixed seed at init time so the two models get the same random weights
						lowerCAmelCase__					:    Union[str, Any]												=					OpenLlamaModel(lowercase_			)
						original_model.to(lowercase_			)
						original_model.eval()
						lowerCAmelCase__					:    Optional[int]												=					original_model(lowercase_			).last_hidden_state
						lowerCAmelCase__					:    Optional[int]												=					original_model(lowercase_			).last_hidden_state

						set_seed(42			)  # Fixed seed at init time so the two models get the same random weights
						lowerCAmelCase__					:    Optional[int]												=					{"type": scaling_type, "factor": 10.0}
						lowerCAmelCase__					:    List[Any]												=					OpenLlamaModel(lowercase_			)
						scaled_model.to(lowercase_			)
						scaled_model.eval()
						lowerCAmelCase__					:    Tuple												=					scaled_model(lowercase_			).last_hidden_state
						lowerCAmelCase__					:    Any												=					scaled_model(lowercase_			).last_hidden_state

						# Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
						# maximum sequence length, so the outputs for the short input should match.
						if scaling_type == "dynamic":
										self.assertTrue(torch.allclose(lowercase_		,lowercase_		,atol=1e-5			)			)
						else:
										self.assertFalse(torch.allclose(lowercase_		,lowercase_		,atol=1e-5			)			)

						# The output should be different for long inputs
						self.assertFalse(torch.allclose(lowercase_		,lowercase_		,atol=1e-5			)			)






def      lowerCAmelCase__		(   lowerCamelCase_		:		int = 1000000):





				'''simple docstring'''

				lowerCAmelCase__					:    int												=					set(range(3	,lowerCamelCase_	,2))
				primes.add(2)
				for p in range(3	,lowerCamelCase_	,2):
								if p not in primes:
												continue
								primes.difference_update(set(range(p * p	,lowerCamelCase_	,lowerCamelCase_)))

				lowerCAmelCase__					:    int												=					[float(lowerCamelCase_) for n in range(limit + 1)]

				for p in primes:
								for n in range(lowerCamelCase_	,limit + 1	,lowerCamelCase_):
												phi[n] *= 1 - 1 / p

				return int(sum(phi[2:]))


if __name__ == "__main__":
							print(f"""{solution() = }""")

