






'''simple docstring'''







from ...configuration_utils import PretrainedConfig
from ...utils import logging


_A     :      List[Any]   					=logging.get_logger(__name__)

_A     :      int   					={
    '''s-JoL/Open-Llama-V1''': '''https://huggingface.co/s-JoL/Open-Llama-V1/blob/main/config.json''',
}

class _lowercase     (    _lowercase		):
							a    				=	"""open-llama"""




							def __init__(							self:    Union[str, Any]      ,							UpperCamelCase__:    str=100_000      ,							UpperCamelCase__:    Dict=4_096      ,							UpperCamelCase__:    Optional[int]=11_008      ,							UpperCamelCase__:    Union[str, Any]=32      ,							UpperCamelCase__:    List[str]=32      ,							UpperCamelCase__:    Dict="silu"      ,							UpperCamelCase__:    Dict=2_048      ,							UpperCamelCase__:    str=0.02      ,							UpperCamelCase__:    str=1e-6      ,							UpperCamelCase__:    List[Any]=True      ,							UpperCamelCase__:    Any=0      ,							UpperCamelCase__:    str=1      ,							UpperCamelCase__:    Optional[int]=2      ,							UpperCamelCase__:    Any=False      ,							UpperCamelCase__:    Union[str, Any]=True      ,							UpperCamelCase__:    Union[str, Any]=0.1      ,							UpperCamelCase__:    Union[str, Any]=0.1      ,							UpperCamelCase__:    List[str]=True      ,							UpperCamelCase__:    List[Any]=True      ,							UpperCamelCase__:    Union[str, Any]=None      ,							**UpperCamelCase__:    str      ,							):
														lowerCamelCase__ :     Any	  =	vocab_size
														lowerCamelCase__ :     Optional[Any]	  =	max_position_embeddings
														lowerCamelCase__ :     Union[str, Any]	  =	hidden_size
														lowerCamelCase__ :     str	  =	intermediate_size
														lowerCamelCase__ :     Any	  =	num_hidden_layers
														lowerCamelCase__ :     List[str]	  =	num_attention_heads
														lowerCamelCase__ :     List[str]	  =	hidden_act
														lowerCamelCase__ :     Any	  =	initializer_range
														lowerCamelCase__ :     Union[str, Any]	  =	rms_norm_eps
														lowerCamelCase__ :     Any	  =	use_cache
														lowerCamelCase__ :     Any	  =	kwargs.pop(
														    """use_memorry_efficient_attention"""      ,							UpperCamelCase__					)
														lowerCamelCase__ :     List[str]	  =	hidden_dropout_prob
														lowerCamelCase__ :     Tuple	  =	attention_dropout_prob
														lowerCamelCase__ :     Optional[Any]	  =	use_stable_embedding
														lowerCamelCase__ :     List[str]	  =	shared_input_output_embedding
														lowerCamelCase__ :     Optional[int]	  =	rope_scaling
														self._rope_scaling_validation()

														super().__init__(
														    pad_token_id=UpperCamelCase__      ,							bos_token_id=UpperCamelCase__      ,							eos_token_id=UpperCamelCase__      ,							tie_word_embeddings=UpperCamelCase__      ,							**UpperCamelCase__      ,							)




							def 		lowerCamelCase_    (							self:    Union[str, Any]					):
														if self.rope_scaling is None:
																					return

														if not isinstance(self.rope_scaling      ,							UpperCamelCase__					) or len(self.rope_scaling					) != 2:
																					raise ValueError(
																					    """`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, """
																					    F'''got {self.rope_scaling}'''					)
														lowerCamelCase__ :     Optional[int]	  =	self.rope_scaling.get("""type"""      ,							UpperCamelCase__					)
														lowerCamelCase__ :     Tuple	  =	self.rope_scaling.get("""factor"""      ,							UpperCamelCase__					)
														if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
																					raise ValueError(
																					    F'''`rope_scaling`\'s name field must be one of [\'linear\', \'dynamic\'], got {rope_scaling_type}'''					)
														if rope_scaling_factor is None or not isinstance(UpperCamelCase__      ,							UpperCamelCase__					) or rope_scaling_factor <= 1.0:
																					raise ValueError(F'''`rope_scaling`\'s factor field must be an float > 1, got {rope_scaling_factor}'''					)









'''simple docstring'''







from torch import nn





def 			SCREAMING_SNAKE_CASE_	(UpperCamelCase				)				->	Dict:
							if act_fn in ["swish", "silu"]:
														return nn.SiLU()
							elif act_fn == "mish":
														return nn.Mish()
							elif act_fn == "gelu":
														return nn.GELU()
							else:
														raise ValueError(f'''Unsupported activation function: {act_fn}'''				)

