

'''simple docstring'''



from ...configuration_utils import PretrainedConfig
from ...utils import logging


__snake_case			:     Dict										=      logging.get_logger(__name__)

__snake_case			:     Dict										=      {
    'facebook/nllb-moe-54B': 'https://huggingface.co/facebook/nllb-moe-54b/resolve/main/config.json',
}



class lowerCamelCase			(  lowercase_						):



			'''simple docstring'''






			__snake_case        =	'nllb-moe'
			__snake_case        =	['past_key_values']
			__snake_case        =	{'num_attention_heads': 'encoder_attention_heads', 'hidden_size': 'd_model'}


			def __init__(				self	:						Optional[Any]		,      lowerCAmelCase_	:						str=12_81_12		,      lowerCAmelCase_	:						List[str]=10_24		,      lowerCAmelCase_	:						str=12		,      lowerCAmelCase_	:						List[str]=40_96		,      lowerCAmelCase_	:						Optional[Any]=16		,      lowerCAmelCase_	:						int=12		,      lowerCAmelCase_	:						int=40_96		,      lowerCAmelCase_	:						Optional[Any]=16		,      lowerCAmelCase_	:						Optional[int]=0.05		,      lowerCAmelCase_	:						Tuple=0.05		,      lowerCAmelCase_	:						Tuple=True		,      lowerCAmelCase_	:						List[str]=True		,      lowerCAmelCase_	:						int="relu"		,      lowerCAmelCase_	:						List[str]=10_24		,      lowerCAmelCase_	:						int=0.1		,      lowerCAmelCase_	:						Any=0.1		,      lowerCAmelCase_	:						Any=0.0		,      lowerCAmelCase_	:						Dict=0.02		,      lowerCAmelCase_	:						Union[str, Any]=2		,      lowerCAmelCase_	:						Optional[int]=True		,      lowerCAmelCase_	:						List[Any]=False		,      lowerCAmelCase_	:						List[Any]="float32"		,      lowerCAmelCase_	:						Dict=False		,      lowerCAmelCase_	:						int=1_28		,      lowerCAmelCase_	:						List[Any]=64		,      lowerCAmelCase_	:						str=4		,      lowerCAmelCase_	:						Dict=4		,      lowerCAmelCase_	:						List[str]=0.001		,      lowerCAmelCase_	:						Any=0.001		,      lowerCAmelCase_	:						Union[str, Any]="all"		,      lowerCAmelCase_	:						Optional[Any]=False		,      lowerCAmelCase_	:						Any=False		,      lowerCAmelCase_	:						str=1.0		,      lowerCAmelCase_	:						Tuple=0.2		,      lowerCAmelCase_	:						Tuple=1		,      lowerCAmelCase_	:						int=0		,      lowerCAmelCase_	:						List[str]=2		,      lowerCAmelCase_	:						Dict=False		,      **lowerCAmelCase_	:						Optional[Any]		,      )      ->		Optional[Any]:





									'''simple docstring'''
									A__					:       int     =vocab_size
									A__					:       List[Any]     =max_position_embeddings
									A__					:       Dict     =d_model
									A__					:       Optional[int]     =encoder_ffn_dim
									A__					:       List[Any]     =encoder_layers
									A__					:       int     =encoder_attention_heads
									A__					:       List[Any]     =decoder_ffn_dim
									A__					:       Union[str, Any]     =decoder_layers
									A__					:       Union[str, Any]     =decoder_attention_heads
									A__					:       List[str]     =dropout
									A__					:       List[Any]     =attention_dropout
									A__					:       Any     =activation_dropout
									A__					:       Optional[int]     =activation_function
									A__					:       Dict     =init_std
									A__					:       Union[str, Any]     =encoder_layerdrop
									A__					:       Optional[int]     =decoder_layerdrop
									A__					:       Any     =use_cache
									A__					:       List[str]     =encoder_layers
									A__					:       List[Any]     =scale_embedding  # scale factor will be sqrt(d_model) if True
									A__					:       List[str]     =router_z_loss_coef
									A__					:       Dict     =router_aux_loss_coef
									A__					:       Optional[Any]     =decoder_sparse_step
									A__					:       List[str]     =encoder_sparse_step
									A__					:       List[Any]     =num_experts
									A__					:       Union[str, Any]     =expert_capacity
									A__					:       List[str]     =router_bias
									if router_dtype not in ["float32", "float16", "bfloat16"]:
															raise ValueError(f"`router_dtype` must be one of 'float32', 'float16' or 'bfloat16', got {router_dtype}"			)
									A__					:       Optional[Any]     =router_dtype

									A__					:       Dict     =router_ignore_padding_tokens
									A__					:       Union[str, Any]     =batch_prioritized_routing
									A__					:       int     =second_expert_policy
									A__					:       Any     =normalize_router_prob_before_dropping
									A__					:       Tuple     =moe_eval_capacity_token_fraction
									A__					:       Optional[Any]     =moe_token_dropout
									A__					:       Optional[Any]     =output_router_logits
									super().__init__(
									    pad_token_id=lowerCAmelCase_		,      bos_token_id=lowerCAmelCase_		,      eos_token_id=lowerCAmelCase_		,      is_encoder_decoder=lowerCAmelCase_		,      decoder_start_token_id=lowerCAmelCase_		,      **lowerCAmelCase_		,      )




'''simple docstring'''



import gc
import unittest

from parameterized import parameterized

from diffusers import FlaxUNetaDConditionModel
from diffusers.utils import is_flax_available
from diffusers.utils.testing_utils import load_hf_numpy, require_flax, slow


if is_flax_available():
		import jax
		import jax.numpy as jnp



@slow
@require_flax
class lowerCamelCase			(  unittest.TestCase						):



			'''simple docstring'''






			def lowercase__						(				self	:						Optional[Any]		,      lowerCAmelCase_	:						Union[str, Any]		,      lowerCAmelCase_	:						List[str]			)      ->		List[str]:





									'''simple docstring'''
									return f"gaussian_noise_s={seed}_shape={'_'.join([str(lowerCAmelCase_			) for s in shape]			)}.npy"


			def lowercase__						(				self	:						Optional[Any]			)      ->		Dict:





									'''simple docstring'''
									# clean up the VRAM after each test
									super().tearDown()
									gc.collect()


			def lowercase__						(				self	:						Tuple		,      lowerCAmelCase_	:						str=0		,      lowerCAmelCase_	:						List[str]=(4, 4, 64, 64)		,      lowerCAmelCase_	:						Optional[Any]=False			)      ->		str:





									'''simple docstring'''
									A__					:       Union[str, Any]     =jnp.bfloataa if fpaa else jnp.floataa
									A__					:       Any     =jnp.array(load_hf_numpy(self.get_file_format(lowerCAmelCase_		,      lowerCAmelCase_			)			)		,      dtype=lowerCAmelCase_			)
									return image


			def lowercase__						(				self	:						int		,      lowerCAmelCase_	:						Optional[int]=False		,      lowerCAmelCase_	:						Dict="CompVis/stable-diffusion-v1-4"			)      ->		List[Any]:





									'''simple docstring'''
									A__					:       Any     =jnp.bfloataa if fpaa else jnp.floataa
									A__					:       int     ="""bf16""" if fpaa else None

									A__   ,						A__					:       Any     =FlaxUNetaDConditionModel.from_pretrained(
									    lowerCAmelCase_		,      subfolder="""unet"""		,      dtype=lowerCAmelCase_		,      revision=lowerCAmelCase_			)
									return model, params


			def lowercase__						(				self	:						Optional[Any]		,      lowerCAmelCase_	:						int=0		,      lowerCAmelCase_	:						Optional[Any]=(4, 77, 7_68)		,      lowerCAmelCase_	:						List[str]=False			)      ->		Tuple:





									'''simple docstring'''
									A__					:       Union[str, Any]     =jnp.bfloataa if fpaa else jnp.floataa
									A__					:       Optional[int]     =jnp.array(load_hf_numpy(self.get_file_format(lowerCAmelCase_		,      lowerCAmelCase_			)			)		,      dtype=lowerCAmelCase_			)
									return hidden_states


			@parameterized.expand(
			    [
			        # fmt: off
			        [83, 4, [-0.2323, -0.1304, 0.0813, -0.3093, -0.0919, -0.1571, -0.1125, -0.5806]],
			        [17, 0.55, [-0.0831, -0.2443, 0.0901, -0.0919, 0.3396, 0.0103, -0.3743, 0.0701]],
			        [8, 0.89, [-0.4863, 0.0859, 0.0875, -0.1658, 0.9199, -0.0114, 0.4839, 0.4639]],
			        [3, 10_00, [-0.5649, 0.2402, -0.5518, 0.1248, 1.1328, -0.2443, -0.0325, -1.0078]],
			        # fmt: on
			    ]			)
			def lowercase__						(				self	:						Any		,      lowerCAmelCase_	:						Tuple		,      lowerCAmelCase_	:						Tuple		,      lowerCAmelCase_	:						str			)      ->		int:





									'''simple docstring'''
									A__   ,						A__					:       Tuple     =self.get_unet_model(model_id="""CompVis/stable-diffusion-v1-4"""		,      fpaa=lowerCAmelCase_			)
									A__					:       List[Any]     =self.get_latents(lowerCAmelCase_		,      fpaa=lowerCAmelCase_			)
									A__					:       Optional[int]     =self.get_encoder_hidden_states(lowerCAmelCase_		,      fpaa=lowerCAmelCase_			)

									A__					:       Union[str, Any]     =model.apply(
									    {"""params""": params}		,      lowerCAmelCase_		,      jnp.array(lowerCAmelCase_		,      dtype=jnp.intaa			)		,      encoder_hidden_states=lowerCAmelCase_		,      ).sample

									assert sample.shape == latents.shape

									A__					:       Optional[int]     =jnp.asarray(jax.device_get((sample[-1, -2:, -2:, :2].flatten())			)		,      dtype=jnp.floataa			)
									A__					:       Tuple     =jnp.array(lowerCAmelCase_		,      dtype=jnp.floataa			)

									# Found torch (float16) and flax (bfloat16) outputs to be within this tolerance, in the same hardware
									assert jnp.allclose(lowerCAmelCase_		,      lowerCAmelCase_		,      atol=1e-2			)



			@parameterized.expand(
			    [
			        # fmt: off
			        [83, 4, [0.1514, 0.0807, 0.1624, 0.1016, -0.1896, 0.0263, 0.0677, 0.2310]],
			        [17, 0.55, [0.1164, -0.0216, 0.0170, 0.1589, -0.3120, 0.1005, -0.0581, -0.1458]],
			        [8, 0.89, [-0.1758, -0.0169, 0.1004, -0.1411, 0.1312, 0.1103, -0.1996, 0.2139]],
			        [3, 10_00, [0.1214, 0.0352, -0.0731, -0.1562, -0.0994, -0.0906, -0.2340, -0.0539]],
			        # fmt: on
			    ]			)
			def lowercase__						(				self	:						Dict		,      lowerCAmelCase_	:						str		,      lowerCAmelCase_	:						int		,      lowerCAmelCase_	:						Optional[int]			)      ->		List[str]:





									'''simple docstring'''
									A__   ,						A__					:       List[str]     =self.get_unet_model(model_id="""stabilityai/stable-diffusion-2"""		,      fpaa=lowerCAmelCase_			)
									A__					:       Union[str, Any]     =self.get_latents(lowerCAmelCase_		,      shape=(4, 4, 96, 96)		,      fpaa=lowerCAmelCase_			)
									A__					:       Dict     =self.get_encoder_hidden_states(lowerCAmelCase_		,      shape=(4, 77, 10_24)		,      fpaa=lowerCAmelCase_			)

									A__					:       Optional[Any]     =model.apply(
									    {"""params""": params}		,      lowerCAmelCase_		,      jnp.array(lowerCAmelCase_		,      dtype=jnp.intaa			)		,      encoder_hidden_states=lowerCAmelCase_		,      ).sample

									assert sample.shape == latents.shape

									A__					:       Optional[Any]     =jnp.asarray(jax.device_get((sample[-1, -2:, -2:, :2].flatten())			)		,      dtype=jnp.floataa			)
									A__					:       List[Any]     =jnp.array(lowerCAmelCase_		,      dtype=jnp.floataa			)

									# Found torch (float16) and flax (bfloat16) outputs to be within this tolerance, on the same hardware
									assert jnp.allclose(lowerCAmelCase_		,      lowerCAmelCase_		,      atol=1e-2			)

