
import math

import tensorflow as tf
from packaging import version


def 				A	( lowercase			)       ->  str:
		'''simple docstring'''

		UpperCamelCase          =					tf.convert_to_tensor(lowercase			)
		UpperCamelCase          =					0.5 * (1.0 + tf.math.erf(x / tf.cast(tf.sqrt(2.0			)   ,	x.dtype			)			))

		return x * cdf


def 				A	( lowercase			)       ->  Union[str, Any]:
		'''simple docstring'''

		UpperCamelCase          =					tf.convert_to_tensor(lowercase			)
		UpperCamelCase          =					tf.cast(math.pi   ,	x.dtype			)
		UpperCamelCase          =					tf.cast(0.0_4_4_7_1_5   ,	x.dtype			)
		UpperCamelCase          =					0.5 * (1.0 + tf.tanh(tf.sqrt(2.0 / pi			) * (x + coeff * tf.pow(lowercase   ,	3			))			))

		return x * cdf


def 				A	( lowercase			)       ->  Optional[int]:
		'''simple docstring'''

		UpperCamelCase          =					tf.convert_to_tensor(lowercase			)

		return x * tf.tanh(tf.math.softplus(lowercase			)			)


def 				A	( lowercase			)       ->  Union[str, Any]:
		'''simple docstring'''

		UpperCamelCase          =					tf.convert_to_tensor(lowercase			)
		UpperCamelCase          =					tf.cast(0.0_4_4_7_1_5   ,	x.dtype			)
		UpperCamelCase          =					tf.cast(0.7_9_7_8_8_4_5_6_0_8   ,	x.dtype			)

		return 0.5 * x * (1.0 + tf.tanh(x * coeffa * (1.0 + coeffa * x * x)			))


def 				A	( lowercase			)       ->  Optional[int]:
		'''simple docstring'''

		UpperCamelCase          =					tf.convert_to_tensor(lowercase			)
		UpperCamelCase          =					tf.cast(1.7_0_2   ,	x.dtype			)
		return x * tf.math.sigmoid(coeff * x			)


def 				A	( lowercase			)       ->  Dict:
		'''simple docstring'''

		return tf.clip_by_value(_gelu(lowercase			)   ,	-10   ,	10			)


def 				A	( lowercase   ,	lowercase=-1			)       ->  Dict:
		'''simple docstring'''

		UpperCamelCase     ,  UpperCamelCase          =					tf.split(lowercase   ,	2   ,	axis=lowercase			)
		return a * tf.math.sigmoid(lowercase			)


if version.parse(tf.version.VERSION) >= version.parse("2.4"):


					def 				A	( lowercase			)       ->  Dict:
							'''simple docstring'''

							return tf.keras.activations.gelu(lowercase   ,	approximate=lowercase			)

					_UpperCAmelCase :			List[str]              =	tf.keras.activations.gelu


					_UpperCAmelCase :			List[Any]              =	approximate_gelu_wrap
else:
					_UpperCAmelCase :			List[Any]              =	_gelu
					_UpperCAmelCase :			Any              =	_gelu_new


_UpperCAmelCase :			Tuple              =	{
    "gelu": gelu,
    "gelu_10": gelu_aa,
    "gelu_fast": gelu_fast,
    "gelu_new": gelu_new,
    "glu": glu,
    "mish": mish,
    "quick_gelu": quick_gelu,
    "relu": tf.keras.activations.relu,
    "sigmoid": tf.keras.activations.sigmoid,
    "silu": tf.keras.activations.swish,
    "swish": tf.keras.activations.swish,
    "tanh": tf.keras.activations.tanh,
}


def 				A	( lowercase			)       ->  Optional[Any]:
		'''simple docstring'''

		if activation_string in ACTaFN:
				return ACTaFN[activation_string]
		else:
				raise KeyError(f'''function {activation_string} not found in ACT2FN mapping {list(ACTaFN.keys()			)}'''			)


from typing import Optional, Union

import torch
from torch import nn

from ...configuration_utils import ConfigMixin, register_to_config
from ...models.modeling_utils import ModelMixin


class     lowercase (		_SCREAMING_SNAKE_CASE  ,       _SCREAMING_SNAKE_CASE       ):



		@register_to_config
		def __init__(			self    ,				A_ = 768    ,				)  -> List[Any]:
				"""simple docstring"""
				super().__init__()

				UpperCamelCase          =					nn.Parameter(torch.zeros(1    ,				A_	)	)
				UpperCamelCase          =					nn.Parameter(torch.ones(1    ,				A_	)	)



		def __UpperCamelCase			(			self    ,				A_ = None    ,				A_ = None    ,				)  -> Any:
				"""simple docstring"""
				UpperCamelCase          =					nn.Parameter(self.mean.to(A_	).to(A_	)	)
				UpperCamelCase          =					nn.Parameter(self.std.to(A_	).to(A_	)	)
				return self



		def __UpperCamelCase			(			self    ,				A_	)  -> Dict:
				"""simple docstring"""
				UpperCamelCase          =					(embeds - self.mean) * 1.0 / self.std
				return embeds



		def __UpperCamelCase			(			self    ,				A_	)  -> Dict:
				"""simple docstring"""
				UpperCamelCase          =					(embeds * self.std) + self.mean
				return embeds
