"""simple docstring"""




import unittest

from parameterized import parameterized

from transformers import OpenLlamaConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
     import torch

     from transformers import OpenLlamaForCausalLM, OpenLlamaForSequenceClassification, OpenLlamaModel
class 						__A							:






     def __init__(		self    ,					a__    ,					a__=13    ,					a__=7    ,					a__=True    ,					a__=True    ,					a__=False    ,					a__=True    ,					a__=99    ,					a__=32    ,					a__=5    ,					a__=4    ,					a__=37    ,					a__="gelu"    ,					a__=0.1    ,					a__=0.1    ,					a__=512    ,					a__=16    ,					a__=2    ,					a__=0.0_2    ,					a__=3    ,					a__=4    ,					a__=None    ,					):
       _lowerCAmelCase						:	List[Any]        =					parent
       _lowerCAmelCase						:	int        =					batch_size
       _lowerCAmelCase						:	Any        =					seq_length
       _lowerCAmelCase						:	Optional[int]        =					is_training
       _lowerCAmelCase						:	Dict        =					use_input_mask
       _lowerCAmelCase						:	Optional[int]        =					use_token_type_ids
       _lowerCAmelCase						:	Dict        =					use_labels
       _lowerCAmelCase						:	List[str]        =					vocab_size
       _lowerCAmelCase						:	Union[str, Any]        =					hidden_size
       _lowerCAmelCase						:	List[Any]        =					num_hidden_layers
       _lowerCAmelCase						:	Tuple        =					num_attention_heads
       _lowerCAmelCase						:	Optional[int]        =					intermediate_size
       _lowerCAmelCase						:	Tuple        =					hidden_act
       _lowerCAmelCase						:	Tuple        =					hidden_dropout_prob
       _lowerCAmelCase						:	List[str]        =					attention_probs_dropout_prob
       _lowerCAmelCase						:	Any        =					max_position_embeddings
       _lowerCAmelCase						:	List[Any]        =					type_vocab_size
       _lowerCAmelCase						:	List[str]        =					type_sequence_label_size
       _lowerCAmelCase						:	Tuple        =					initializer_range
       _lowerCAmelCase						:	Optional[int]        =					num_labels
       _lowerCAmelCase						:	Union[str, Any]        =					num_choices
       _lowerCAmelCase						:	Any        =					scope






     def    __A  (		self  ):
       _lowerCAmelCase						:	Tuple        =					ids_tensor([self.batch_size, self.seq_length]    ,					self.vocab_size  )

       _lowerCAmelCase						:	str        =					None
       if self.use_input_mask:
         _lowerCAmelCase						:	Dict        =					random_attention_mask([self.batch_size, self.seq_length]  )

       _lowerCAmelCase						:	int        =					None
       if self.use_token_type_ids:
         _lowerCAmelCase						:	int        =					ids_tensor([self.batch_size, self.seq_length]    ,					self.type_vocab_size  )

       _lowerCAmelCase						:	Dict        =					None
       _lowerCAmelCase						:	List[str]        =					None
       _lowerCAmelCase						:	Any        =					None
       if self.use_labels:
         _lowerCAmelCase						:	Tuple        =					ids_tensor([self.batch_size]    ,					self.type_sequence_label_size  )
         _lowerCAmelCase						:	Optional[int]        =					ids_tensor([self.batch_size, self.seq_length]    ,					self.num_labels  )
         _lowerCAmelCase						:	Optional[int]        =					ids_tensor([self.batch_size]    ,					self.num_choices  )

       _lowerCAmelCase						:	List[Any]        =					self.get_config()

       return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






     def    __A  (		self  ):
       return OpenLlamaConfig(
           vocab_size=self.vocab_size    ,					hidden_size=self.hidden_size    ,					num_hidden_layers=self.num_hidden_layers    ,					num_attention_heads=self.num_attention_heads    ,					intermediate_size=self.intermediate_size    ,					hidden_act=self.hidden_act    ,					hidden_dropout_prob=self.hidden_dropout_prob    ,					attention_probs_dropout_prob=self.attention_probs_dropout_prob    ,					max_position_embeddings=self.max_position_embeddings    ,					type_vocab_size=self.type_vocab_size    ,					is_decoder=SCREAMING_SNAKE_CASE_    ,					initializer_range=self.initializer_range    ,					use_stable_embedding=SCREAMING_SNAKE_CASE_    ,					)






     def    __A  (		self    ,					a__    ,					a__    ,					a__    ,					a__    ,					a__    ,					a__    ,					a__  ):
       _lowerCAmelCase						:	List[Any]        =					OpenLlamaModel(config=SCREAMING_SNAKE_CASE_  )
       model.to(SCREAMING_SNAKE_CASE_  )
       model.eval()
       _lowerCAmelCase						:	int        =					model(SCREAMING_SNAKE_CASE_    ,					attention_mask=SCREAMING_SNAKE_CASE_  )
       _lowerCAmelCase						:	List[str]        =					model(SCREAMING_SNAKE_CASE_  )
       self.parent.assertEqual(result.last_hidden_state.shape    ,					(self.batch_size, self.seq_length, self.hidden_size)  )






     def    __A  (		self    ,					a__    ,					a__    ,					a__    ,					a__    ,					a__    ,					a__    ,					a__    ,					a__    ,					a__    ,					):
       _lowerCAmelCase						:	Tuple        =					True
       _lowerCAmelCase						:	List[Any]        =					OpenLlamaModel(SCREAMING_SNAKE_CASE_  )
       model.to(SCREAMING_SNAKE_CASE_  )
       model.eval()
       _lowerCAmelCase						:	Any        =					model(
           SCREAMING_SNAKE_CASE_    ,					attention_mask=SCREAMING_SNAKE_CASE_    ,					encoder_hidden_states=SCREAMING_SNAKE_CASE_    ,					encoder_attention_mask=SCREAMING_SNAKE_CASE_    ,					)
       _lowerCAmelCase						:	Optional[int]        =					model(
           SCREAMING_SNAKE_CASE_    ,					attention_mask=SCREAMING_SNAKE_CASE_    ,					encoder_hidden_states=SCREAMING_SNAKE_CASE_    ,					)
       _lowerCAmelCase						:	str        =					model(SCREAMING_SNAKE_CASE_    ,					attention_mask=SCREAMING_SNAKE_CASE_  )
       self.parent.assertEqual(result.last_hidden_state.shape    ,					(self.batch_size, self.seq_length, self.hidden_size)  )






     def    __A  (		self    ,					a__    ,					a__    ,					a__    ,					a__    ,					a__    ,					a__    ,					a__    ,					a__    ,					a__    ,					):
       _lowerCAmelCase						:	Any        =					OpenLlamaForCausalLM(config=SCREAMING_SNAKE_CASE_  )
       model.to(SCREAMING_SNAKE_CASE_  )
       model.eval()
       _lowerCAmelCase						:	Union[str, Any]        =					model(SCREAMING_SNAKE_CASE_    ,					attention_mask=SCREAMING_SNAKE_CASE_    ,					labels=SCREAMING_SNAKE_CASE_  )
       self.parent.assertEqual(result.logits.shape    ,					(self.batch_size, self.seq_length, self.vocab_size)  )






     def    __A  (		self    ,					a__    ,					a__    ,					a__    ,					a__    ,					a__    ,					a__    ,					a__    ,					a__    ,					a__    ,					):
       _lowerCAmelCase						:	Tuple        =					True
       _lowerCAmelCase						:	Optional[int]        =					True
       _lowerCAmelCase						:	Dict        =					OpenLlamaForCausalLM(config=SCREAMING_SNAKE_CASE_  )
       model.to(SCREAMING_SNAKE_CASE_  )
       model.eval()

       # first forward pass
       _lowerCAmelCase						:	str        =					model(
           SCREAMING_SNAKE_CASE_    ,					attention_mask=SCREAMING_SNAKE_CASE_    ,					encoder_hidden_states=SCREAMING_SNAKE_CASE_    ,					encoder_attention_mask=SCREAMING_SNAKE_CASE_    ,					use_cache=SCREAMING_SNAKE_CASE_    ,					)
       _lowerCAmelCase						:	Tuple        =					outputs.past_key_values

       # create hypothetical multiple next token and extent to next_input_ids
       _lowerCAmelCase						:	Tuple        =					ids_tensor((self.batch_size, 3)    ,					config.vocab_size  )
       _lowerCAmelCase						:	Optional[Any]        =					ids_tensor((self.batch_size, 3)    ,					vocab_size=2  )

       # append to next input_ids and
       _lowerCAmelCase						:	str        =					torch.cat([input_ids, next_tokens]    ,					dim=-1  )
       _lowerCAmelCase						:	str        =					torch.cat([input_mask, next_mask]    ,					dim=-1  )

       _lowerCAmelCase						:	Dict        =					model(
           SCREAMING_SNAKE_CASE_    ,					attention_mask=SCREAMING_SNAKE_CASE_    ,					encoder_hidden_states=SCREAMING_SNAKE_CASE_    ,					encoder_attention_mask=SCREAMING_SNAKE_CASE_    ,					output_hidden_states=SCREAMING_SNAKE_CASE_    ,					)["""hidden_states"""][0]
       _lowerCAmelCase						:	Tuple        =					model(
           SCREAMING_SNAKE_CASE_    ,					attention_mask=SCREAMING_SNAKE_CASE_    ,					encoder_hidden_states=SCREAMING_SNAKE_CASE_    ,					encoder_attention_mask=SCREAMING_SNAKE_CASE_    ,					past_key_values=SCREAMING_SNAKE_CASE_    ,					output_hidden_states=SCREAMING_SNAKE_CASE_    ,					)["""hidden_states"""][0]

       # select random slice
       _lowerCAmelCase						:	str        =					ids_tensor((1,)    ,					output_from_past.shape[-1]  ).item()
       _lowerCAmelCase						:	str        =					output_from_no_past[:, -3:, random_slice_idx].detach()
       _lowerCAmelCase						:	Any        =					output_from_past[:, :, random_slice_idx].detach()

       self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]  )

       # test that outputs are equal for slice
       self.parent.assertTrue(torch.allclose(SCREAMING_SNAKE_CASE_    ,					SCREAMING_SNAKE_CASE_    ,					atol=1e-3  )  )






     def    __A  (		self  ):
       _lowerCAmelCase						:	List[str]        =					self.prepare_config_and_inputs()
       (
           _lowerCAmelCase    
       )						:	List[Any]        =					config_and_inputs
       _lowerCAmelCase						:	List[Any]        =					{"""input_ids""": input_ids, """attention_mask""": input_mask}
       return config, inputs_dict







@require_torch
class 						__A							(			_lowerCAmelCase   ,     _lowerCAmelCase   ,     _lowerCAmelCase   ,     unittest.TestCase		):
     _UpperCamelCase							:      Dict		 =		(
         (OpenLlamaModel, OpenLlamaForCausalLM, OpenLlamaForSequenceClassification) if is_torch_available() else ()
     )
     _UpperCamelCase							:      int		 =		(OpenLlamaForCausalLM,) if is_torch_available() else ()
     _UpperCamelCase							:      Optional[Any]		 =		(
         {
             "feature-extraction": OpenLlamaModel,
             "text-classification": OpenLlamaForSequenceClassification,
             "text-generation": OpenLlamaForCausalLM,
             "zero-shot": OpenLlamaForSequenceClassification,
         }
         if is_torch_available()
         else {}
     )
     _UpperCamelCase							:      Optional[int]		 =		False
     _UpperCamelCase							:      List[str]		 =		False






     def    __A  (		self  ):
       _lowerCAmelCase						:	str        =					OpenLlamaModelTester(self  )
       _lowerCAmelCase						:	Any        =					ConfigTester(self    ,					config_class=SCREAMING_SNAKE_CASE_    ,					hidden_size=37  )






     def    __A  (		self  ):
       self.config_tester.run_common_tests()






     def    __A  (		self  ):
       _lowerCAmelCase						:	int        =					self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_model(*SCREAMING_SNAKE_CASE_  )






     def    __A  (		self  ):
       _lowerCAmelCase						:	List[Any]        =					self.model_tester.prepare_config_and_inputs()
       for type in ["absolute", "relative_key", "relative_key_query"]:
         _lowerCAmelCase						:	Dict        =					type
         self.model_tester.create_and_check_model(*SCREAMING_SNAKE_CASE_  )






     def    __A  (		self  ):
       _lowerCAmelCase						:	Tuple        =					self.model_tester.prepare_config_and_inputs_for_common()
       _lowerCAmelCase						:	int        =					3
       _lowerCAmelCase						:	Tuple        =					input_dict["""input_ids"""]
       _lowerCAmelCase						:	Optional[int]        =					input_ids.ne(1  ).to(SCREAMING_SNAKE_CASE_  )
       _lowerCAmelCase						:	Tuple        =					ids_tensor([self.model_tester.batch_size]    ,					self.model_tester.type_sequence_label_size  )
       _lowerCAmelCase						:	Optional[int]        =					OpenLlamaForSequenceClassification(SCREAMING_SNAKE_CASE_  )
       model.to(SCREAMING_SNAKE_CASE_  )
       model.eval()
       _lowerCAmelCase						:	Union[str, Any]        =					model(SCREAMING_SNAKE_CASE_    ,					attention_mask=SCREAMING_SNAKE_CASE_    ,					labels=SCREAMING_SNAKE_CASE_  )
       self.assertEqual(result.logits.shape    ,					(self.model_tester.batch_size, self.model_tester.num_labels)  )






     def    __A  (		self  ):
       _lowerCAmelCase						:	List[str]        =					self.model_tester.prepare_config_and_inputs_for_common()
       _lowerCAmelCase						:	Dict        =					3
       _lowerCAmelCase						:	Optional[Any]        =					"""single_label_classification"""
       _lowerCAmelCase						:	Optional[int]        =					input_dict["""input_ids"""]
       _lowerCAmelCase						:	str        =					input_ids.ne(1  ).to(SCREAMING_SNAKE_CASE_  )
       _lowerCAmelCase						:	Optional[int]        =					ids_tensor([self.model_tester.batch_size]    ,					self.model_tester.type_sequence_label_size  )
       _lowerCAmelCase						:	List[str]        =					OpenLlamaForSequenceClassification(SCREAMING_SNAKE_CASE_  )
       model.to(SCREAMING_SNAKE_CASE_  )
       model.eval()
       _lowerCAmelCase						:	Optional[int]        =					model(SCREAMING_SNAKE_CASE_    ,					attention_mask=SCREAMING_SNAKE_CASE_    ,					labels=SCREAMING_SNAKE_CASE_  )
       self.assertEqual(result.logits.shape    ,					(self.model_tester.batch_size, self.model_tester.num_labels)  )






     def    __A  (		self  ):
       _lowerCAmelCase						:	Any        =					self.model_tester.prepare_config_and_inputs_for_common()
       _lowerCAmelCase						:	Optional[int]        =					3
       _lowerCAmelCase						:	int        =					"""multi_label_classification"""
       _lowerCAmelCase						:	Tuple        =					input_dict["""input_ids"""]
       _lowerCAmelCase						:	int        =					input_ids.ne(1  ).to(SCREAMING_SNAKE_CASE_  )
       _lowerCAmelCase						:	Union[str, Any]        =					ids_tensor(
           [self.model_tester.batch_size, config.num_labels]    ,					self.model_tester.type_sequence_label_size  ).to(torch.float  )
       _lowerCAmelCase						:	Optional[Any]        =					OpenLlamaForSequenceClassification(SCREAMING_SNAKE_CASE_  )
       model.to(SCREAMING_SNAKE_CASE_  )
       model.eval()
       _lowerCAmelCase						:	Any        =					model(SCREAMING_SNAKE_CASE_    ,					attention_mask=SCREAMING_SNAKE_CASE_    ,					labels=SCREAMING_SNAKE_CASE_  )
       self.assertEqual(result.logits.shape    ,					(self.model_tester.batch_size, self.model_tester.num_labels)  )






     @unittest.skip("""Open-Llama buffers include complex numbers, which breaks this test"""  )
     def    __A  (		self  ):
       pass






     @parameterized.expand([("""linear""",), ("""dynamic""",)]  )
     def    __A  (		self    ,					a__  ):
       _lowerCAmelCase						:	Any        =					self.model_tester.prepare_config_and_inputs_for_common()
       _lowerCAmelCase						:	Dict        =					ids_tensor([1, 10]    ,					config.vocab_size  )
       _lowerCAmelCase						:	Optional[Any]        =					ids_tensor([1, int(config.max_position_embeddings * 1.5  )]    ,					config.vocab_size  )

       set_seed(42  )  # Fixed seed at init time so the two models get the same random weights
       _lowerCAmelCase						:	Any        =					OpenLlamaModel(SCREAMING_SNAKE_CASE_  )
       original_model.to(SCREAMING_SNAKE_CASE_  )
       original_model.eval()
       _lowerCAmelCase						:	Any        =					original_model(SCREAMING_SNAKE_CASE_  ).last_hidden_state
       _lowerCAmelCase						:	Tuple        =					original_model(SCREAMING_SNAKE_CASE_  ).last_hidden_state

       set_seed(42  )  # Fixed seed at init time so the two models get the same random weights
       _lowerCAmelCase						:	Optional[Any]        =					{"""type""": scaling_type, """factor""": 10.0}
       _lowerCAmelCase						:	int        =					OpenLlamaModel(SCREAMING_SNAKE_CASE_  )
       scaled_model.to(SCREAMING_SNAKE_CASE_  )
       scaled_model.eval()
       _lowerCAmelCase						:	Union[str, Any]        =					scaled_model(SCREAMING_SNAKE_CASE_  ).last_hidden_state
       _lowerCAmelCase						:	Union[str, Any]        =					scaled_model(SCREAMING_SNAKE_CASE_  ).last_hidden_state

       # Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
       # maximum sequence length, so the outputs for the short input should match.
       if scaling_type == "dynamic":
         self.assertTrue(torch.allclose(SCREAMING_SNAKE_CASE_    ,					SCREAMING_SNAKE_CASE_    ,					atol=1e-5  )  )
       else:
         self.assertFalse(torch.allclose(SCREAMING_SNAKE_CASE_    ,					SCREAMING_SNAKE_CASE_    ,					atol=1e-5  )  )

       # The output should be different for long inputs
       self.assertFalse(torch.allclose(SCREAMING_SNAKE_CASE_    ,					SCREAMING_SNAKE_CASE_    ,					atol=1e-5  )  )






"""simple docstring"""




def 							SCREAMING_SNAKE_CASE					(      _lowerCamelCase						:	str      ,_lowerCamelCase						:	bool = False     )	->				str:

  if not isinstance(_lowerCamelCase      ,_lowerCamelCase     ):
    _lowerCAmelCase						:	Union[str, Any]        =					f"Expected string as input, found {type(_lowerCamelCase     )}"
    raise ValueError(_lowerCamelCase     )
  if not isinstance(_lowerCamelCase      ,_lowerCamelCase     ):
    _lowerCAmelCase						:	Union[str, Any]        =					f"Expected boolean as use_pascal parameter, found {type(_lowerCamelCase     )}"
    raise ValueError(_lowerCamelCase     )

  _lowerCAmelCase						:	str        =					input_str.split("""_"""     )

  _lowerCAmelCase						:	Optional[int]        =					0 if use_pascal else 1

  _lowerCAmelCase						:	List[Any]        =					words[start_index:]

  _lowerCAmelCase						:	Union[str, Any]        =					[word[0].upper() + word[1:] for word in words_to_capitalize]

  _lowerCAmelCase						:	List[Any]        =					"""""" if use_pascal else words[0]

  return "".join([initial_word, *capitalized_words]     )


if __name__ == "__main__":
     from doctest import testmod

     testmod()




