



from typing import Optional, Tuple, Union

import flax
import flax.linen as nn
import jax
import jax.numpy as jnp
from flax.core.frozen_dict import FrozenDict

from ..configuration_utils import ConfigMixin, flax_register_to_config
from ..utils import BaseOutput
from .embeddings_flax import FlaxTimestepEmbedding, FlaxTimesteps
from .modeling_flax_utils import FlaxModelMixin
from .unet_ad_blocks_flax import (
    FlaxCrossAttnDownBlockaD,
    FlaxCrossAttnUpBlockaD,
    FlaxDownBlockaD,
    FlaxUNetMidBlockaDCrossAttn,
    FlaxUpBlockaD,
)

@flax.struct.dataclass
class    _A	(     __UpperCAmelCase   ):
       UpperCamelCase__  :   jnp.ndarray


@flax_register_to_config
class    _A	(     nn.Module				,__UpperCAmelCase				,__UpperCAmelCase   ):
       UpperCamelCase__  :   int						 =						32
       UpperCamelCase__  :   int						 =						4
       UpperCamelCase__  :   int						 =						4
       UpperCamelCase__  :   Tuple[str]						 =						(
           "CrossAttnDownBlock2D",
           "CrossAttnDownBlock2D",
           "CrossAttnDownBlock2D",
           "DownBlock2D",
       )
       UpperCamelCase__  :   Tuple[str]						 =						("UpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D")
       UpperCamelCase__  :   Union[bool, Tuple[bool]]						 =						False
       UpperCamelCase__  :   Tuple[int]						 =						(320, 640, 1_280, 1_280)
       UpperCamelCase__  :   int						 =						2
       UpperCamelCase__  :   Union[int, Tuple[int]]						 =						8
       UpperCamelCase__  :   Optional[Union[int, Tuple[int]]]						 =						None
       UpperCamelCase__  :   int						 =						1_280
       UpperCamelCase__  :   float						 =						0.0
       UpperCamelCase__  :   bool						 =						False
       UpperCamelCase__  :   jnp.dtype						 =						jnp.floataa
       UpperCamelCase__  :   bool						 =						True
       UpperCamelCase__  :   int						 =						0
       UpperCamelCase__  :   bool						 =						False



       def 		_lowerCamelCase      (		self							:					str					,		__SCREAMING_SNAKE_CASE							:					jax.random.KeyArray):


           '''simple docstring'''

           __a							=    (1, self.in_channels, self.sample_size, self.sample_size)
           __a							=    jnp.zeros(__SCREAMING_SNAKE_CASE					,		dtype=jnp.floataa)
           __a							=    jnp.ones((1,)					,		dtype=jnp.intaa)
           __a							=    jnp.zeros((1, 1, self.cross_attention_dim)					,		dtype=jnp.floataa)

           __a	,       __a							=    jax.random.split(__SCREAMING_SNAKE_CASE)
           __a							=    {'''params''': params_rng, '''dropout''': dropout_rng}

           return self.init(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE)["params"]



       def 		_lowerCamelCase      (		self							:					Dict):


           '''simple docstring'''

           __a							=    self.block_out_channels
           __a							=    block_out_channels[0] * 4

           if self.num_attention_heads is not None:
               raise ValueError(
                   '''At the moment it is not possible to define the number of attention heads via `num_attention_heads` because of a naming issue as described in https://github.com/huggingface/diffusers/issues/2011#issuecomment-1547958131. Passing `num_attention_heads` will only be supported in diffusers v0.19.''')

           # If `num_attention_heads` is not defined (which is the case for most models)
           # it will default to `attention_head_dim`. This looks weird upon first reading it and it is.
           # The reason for this behavior is to correct for incorrectly named variables that were introduced
           # when this library was created. The incorrect naming was only discovered much later in https://github.com/huggingface/diffusers/issues/2011#issuecomment-1547958131
           # Changing `attention_head_dim` to `num_attention_heads` for 40,000+ configurations is too backwards breaking
           # which is why we correct for the naming here.
           __a							=    self.num_attention_heads or self.attention_head_dim

           # input
           __a							=    nn.Conv(
               block_out_channels[0]					,		kernel_size=(3, 3)					,		strides=(1, 1)					,		padding=((1, 1), (1, 1))					,		dtype=self.dtype					,		)

           # time
           __a							=    FlaxTimesteps(
               block_out_channels[0]					,		flip_sin_to_cos=self.flip_sin_to_cos					,		freq_shift=self.config.freq_shift)
           __a							=    FlaxTimestepEmbedding(__SCREAMING_SNAKE_CASE					,		dtype=self.dtype)

           __a							=    self.only_cross_attention
           if isinstance(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE):
               __a							=    (only_cross_attention,) * len(self.down_block_types)

           if isinstance(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE):
               __a							=    (num_attention_heads,) * len(self.down_block_types)

           # down
           __a							=    []
           __a							=    block_out_channels[0]
           for i, down_block_type in enumerate(self.down_block_types):
               __a							=    output_channel
               __a							=    block_out_channels[i]
               __a							=    i == len(__SCREAMING_SNAKE_CASE) - 1

               if down_block_type == "CrossAttnDownBlock2D":
                   __a							=    FlaxCrossAttnDownBlockaD(
                       in_channels=__SCREAMING_SNAKE_CASE					,		out_channels=__SCREAMING_SNAKE_CASE					,		dropout=self.dropout					,		num_layers=self.layers_per_block					,		num_attention_heads=num_attention_heads[i]					,		add_downsample=not is_final_block					,		use_linear_projection=self.use_linear_projection					,		only_cross_attention=only_cross_attention[i]					,		use_memory_efficient_attention=self.use_memory_efficient_attention					,		dtype=self.dtype					,		)
               else:
                   __a							=    FlaxDownBlockaD(
                       in_channels=__SCREAMING_SNAKE_CASE					,		out_channels=__SCREAMING_SNAKE_CASE					,		dropout=self.dropout					,		num_layers=self.layers_per_block					,		add_downsample=not is_final_block					,		dtype=self.dtype					,		)

               down_blocks.append(__SCREAMING_SNAKE_CASE)
           __a							=    down_blocks

           # mid
           __a							=    FlaxUNetMidBlockaDCrossAttn(
               in_channels=block_out_channels[-1]					,		dropout=self.dropout					,		num_attention_heads=num_attention_heads[-1]					,		use_linear_projection=self.use_linear_projection					,		use_memory_efficient_attention=self.use_memory_efficient_attention					,		dtype=self.dtype					,		)

           # up
           __a							=    []
           __a							=    list(reversed(__SCREAMING_SNAKE_CASE))
           __a							=    list(reversed(__SCREAMING_SNAKE_CASE))
           __a							=    list(reversed(__SCREAMING_SNAKE_CASE))
           __a							=    reversed_block_out_channels[0]
           for i, up_block_type in enumerate(self.up_block_types):
               __a							=    output_channel
               __a							=    reversed_block_out_channels[i]
               __a							=    reversed_block_out_channels[min(i + 1					,		len(__SCREAMING_SNAKE_CASE) - 1)]

               __a							=    i == len(__SCREAMING_SNAKE_CASE) - 1

               if up_block_type == "CrossAttnUpBlock2D":
                   __a							=    FlaxCrossAttnUpBlockaD(
                       in_channels=__SCREAMING_SNAKE_CASE					,		out_channels=__SCREAMING_SNAKE_CASE					,		prev_output_channel=__SCREAMING_SNAKE_CASE					,		num_layers=self.layers_per_block + 1					,		num_attention_heads=reversed_num_attention_heads[i]					,		add_upsample=not is_final_block					,		dropout=self.dropout					,		use_linear_projection=self.use_linear_projection					,		only_cross_attention=only_cross_attention[i]					,		use_memory_efficient_attention=self.use_memory_efficient_attention					,		dtype=self.dtype					,		)
               else:
                   __a							=    FlaxUpBlockaD(
                       in_channels=__SCREAMING_SNAKE_CASE					,		out_channels=__SCREAMING_SNAKE_CASE					,		prev_output_channel=__SCREAMING_SNAKE_CASE					,		num_layers=self.layers_per_block + 1					,		add_upsample=not is_final_block					,		dropout=self.dropout					,		dtype=self.dtype					,		)

               up_blocks.append(__SCREAMING_SNAKE_CASE)
               __a							=    output_channel
           __a							=    up_blocks

           # out
           __a							=    nn.GroupNorm(num_groups=32					,		epsilon=1E-5)
           __a							=    nn.Conv(
               self.out_channels					,		kernel_size=(3, 3)					,		strides=(1, 1)					,		padding=((1, 1), (1, 1))					,		dtype=self.dtype					,		)





       def __call__(		self							:					Optional[Any]					,		__SCREAMING_SNAKE_CASE							:					Optional[Any]					,		__SCREAMING_SNAKE_CASE							:					Optional[int]					,		__SCREAMING_SNAKE_CASE							:					str					,		__SCREAMING_SNAKE_CASE							:					Tuple=None					,		__SCREAMING_SNAKE_CASE							:					int=None					,		__SCREAMING_SNAKE_CASE							:					bool = True					,		__SCREAMING_SNAKE_CASE							:					bool = False					,		):


           '''simple docstring'''

           if not isinstance(__SCREAMING_SNAKE_CASE					,		jnp.ndarray):
               __a							=    jnp.array([timesteps]					,		dtype=jnp.intaa)
           elif isinstance(__SCREAMING_SNAKE_CASE					,		jnp.ndarray) and len(timesteps.shape) == 0:
               __a							=    timesteps.astype(dtype=jnp.floataa)
               __a							=    jnp.expand_dims(__SCREAMING_SNAKE_CASE					,		0)

           __a							=    self.time_proj(__SCREAMING_SNAKE_CASE)
           __a							=    self.time_embedding(__SCREAMING_SNAKE_CASE)

           # 2. pre-process
           __a							=    jnp.transpose(__SCREAMING_SNAKE_CASE					,		(0, 2, 3, 1))
           __a							=    self.conv_in(__SCREAMING_SNAKE_CASE)

           # 3. down
           __a							=    (sample,)
           for down_block in self.down_blocks:
               if isinstance(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE):
                   __a	,       __a							=    down_block(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE					,		deterministic=not train)
               else:
                   __a	,       __a							=    down_block(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE					,		deterministic=not train)
               down_block_res_samples += res_samples

           if down_block_additional_residuals is not None:
               __a							=    ()

               for down_block_res_sample, down_block_additional_residual in zip(
                   __SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE):
                   down_block_res_sample += down_block_additional_residual
                   new_down_block_res_samples += (down_block_res_sample,)

               __a							=    new_down_block_res_samples

           # 4. mid
           __a							=    self.mid_block(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE					,		deterministic=not train)

           if mid_block_additional_residual is not None:
               sample += mid_block_additional_residual

           # 5. up
           for up_block in self.up_blocks:
               __a							=    down_block_res_samples[-(self.layers_per_block + 1) :]
               __a							=    down_block_res_samples[: -(self.layers_per_block + 1)]
               if isinstance(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE):
                   __a							=    up_block(
                       __SCREAMING_SNAKE_CASE					,		temb=__SCREAMING_SNAKE_CASE					,		encoder_hidden_states=__SCREAMING_SNAKE_CASE					,		res_hidden_states_tuple=__SCREAMING_SNAKE_CASE					,		deterministic=not train					,		)
               else:
                   __a							=    up_block(__SCREAMING_SNAKE_CASE					,		temb=__SCREAMING_SNAKE_CASE					,		res_hidden_states_tuple=__SCREAMING_SNAKE_CASE					,		deterministic=not train)

        # 6. post-process
           __a							=    self.conv_norm_out(__SCREAMING_SNAKE_CASE)
           __a							=    nn.silu(__SCREAMING_SNAKE_CASE)
           __a							=    self.conv_out(__SCREAMING_SNAKE_CASE)
           __a							=    jnp.transpose(__SCREAMING_SNAKE_CASE					,		(0, 3, 1, 2))

           if not return_dict:
               return (sample,)

           return FlaxUNetaDConditionOutput(sample=__SCREAMING_SNAKE_CASE)










import collections
import inspect
import unittest

from transformers import SwinvaConfig
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch
 from torch import nn

 from transformers import SwinvaForImageClassification, SwinvaForMaskedImageModeling, SwinvaModel
 from transformers.models.swinva.modeling_swinva import SWINV2_PRETRAINED_MODEL_ARCHIVE_LIST

if is_vision_available():
 from PIL import Image

 from transformers import AutoImageProcessor

class    _A	:



       def __init__(		self							:					str					,		__SCREAMING_SNAKE_CASE							:					int					,		__SCREAMING_SNAKE_CASE							:					Tuple=13					,		__SCREAMING_SNAKE_CASE							:					Union[str, Any]=32					,		__SCREAMING_SNAKE_CASE							:					List[Any]=2					,		__SCREAMING_SNAKE_CASE							:					Dict=3					,		__SCREAMING_SNAKE_CASE							:					Any=16					,		__SCREAMING_SNAKE_CASE							:					Optional[int]=[1, 2, 1]					,		__SCREAMING_SNAKE_CASE							:					Optional[int]=[2, 2, 4]					,		__SCREAMING_SNAKE_CASE							:					Any=2					,		__SCREAMING_SNAKE_CASE							:					Tuple=2.0					,		__SCREAMING_SNAKE_CASE							:					List[str]=True					,		__SCREAMING_SNAKE_CASE							:					Dict=0.0					,		__SCREAMING_SNAKE_CASE							:					List[str]=0.0					,		__SCREAMING_SNAKE_CASE							:					str=0.1					,		__SCREAMING_SNAKE_CASE							:					int="gelu"					,		__SCREAMING_SNAKE_CASE							:					Tuple=False					,		__SCREAMING_SNAKE_CASE							:					List[Any]=True					,		__SCREAMING_SNAKE_CASE							:					Dict=0.02					,		__SCREAMING_SNAKE_CASE							:					Dict=1E-5					,		__SCREAMING_SNAKE_CASE							:					Optional[Any]=True					,		__SCREAMING_SNAKE_CASE							:					int=None					,		__SCREAMING_SNAKE_CASE							:					str=True					,		__SCREAMING_SNAKE_CASE							:					Optional[int]=10					,		__SCREAMING_SNAKE_CASE							:					int=8					,		):


           '''simple docstring'''

           __a							=    parent
           __a							=    batch_size
           __a							=    image_size
           __a							=    patch_size
           __a							=    num_channels
           __a							=    embed_dim
           __a							=    depths
           __a							=    num_heads
           __a							=    window_size
           __a							=    mlp_ratio
           __a							=    qkv_bias
           __a							=    hidden_dropout_prob
           __a							=    attention_probs_dropout_prob
           __a							=    drop_path_rate
           __a							=    hidden_act
           __a							=    use_absolute_embeddings
           __a							=    patch_norm
           __a							=    layer_norm_eps
           __a							=    initializer_range
           __a							=    is_training
           __a							=    scope
           __a							=    use_labels
           __a							=    type_sequence_label_size
           __a							=    encoder_stride



       def 		_lowerCamelCase      (		self							:					Dict):


           '''simple docstring'''

           __a							=    floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size])

           __a							=    None
           if self.use_labels:
               __a							=    ids_tensor([self.batch_size]					,		self.type_sequence_label_size)

           __a							=    self.get_config()

           return config, pixel_values, labels



       def 		_lowerCamelCase      (		self							:					List[Any]):


           '''simple docstring'''

           return SwinvaConfig(
               image_size=self.image_size					,		patch_size=self.patch_size					,		num_channels=self.num_channels					,		embed_dim=self.embed_dim					,		depths=self.depths					,		num_heads=self.num_heads					,		window_size=self.window_size					,		mlp_ratio=self.mlp_ratio					,		qkv_bias=self.qkv_bias					,		hidden_dropout_prob=self.hidden_dropout_prob					,		attention_probs_dropout_prob=self.attention_probs_dropout_prob					,		drop_path_rate=self.drop_path_rate					,		hidden_act=self.hidden_act					,		use_absolute_embeddings=self.use_absolute_embeddings					,		path_norm=self.patch_norm					,		layer_norm_eps=self.layer_norm_eps					,		initializer_range=self.initializer_range					,		encoder_stride=self.encoder_stride					,		)



       def 		_lowerCamelCase      (		self							:					str					,		__SCREAMING_SNAKE_CASE							:					str					,		__SCREAMING_SNAKE_CASE							:					List[str]					,		__SCREAMING_SNAKE_CASE							:					Optional[Any]):


           '''simple docstring'''

           __a							=    SwinvaModel(config=__SCREAMING_SNAKE_CASE)
           model.to(__SCREAMING_SNAKE_CASE)
           model.eval()
           __a							=    model(__SCREAMING_SNAKE_CASE)

           __a							=    ((config.image_size // config.patch_size) ** 2) // (4 ** (len(config.depths) - 1))
           __a							=    int(config.embed_dim * 2 ** (len(config.depths) - 1))

           self.parent.assertEqual(result.last_hidden_state.shape					,		(self.batch_size, expected_seq_len, expected_dim))



       def 		_lowerCamelCase      (		self							:					Dict					,		__SCREAMING_SNAKE_CASE							:					Optional[int]					,		__SCREAMING_SNAKE_CASE							:					Tuple					,		__SCREAMING_SNAKE_CASE							:					Any):


           '''simple docstring'''

           __a							=    SwinvaForMaskedImageModeling(config=__SCREAMING_SNAKE_CASE)
           model.to(__SCREAMING_SNAKE_CASE)
           model.eval()
           __a							=    model(__SCREAMING_SNAKE_CASE)
           self.parent.assertEqual(
               result.logits.shape					,		(self.batch_size, self.num_channels, self.image_size, self.image_size))

           # test greyscale images
           __a							=    1
           __a							=    SwinvaForMaskedImageModeling(__SCREAMING_SNAKE_CASE)
           model.to(__SCREAMING_SNAKE_CASE)
           model.eval()

           __a							=    floats_tensor([self.batch_size, 1, self.image_size, self.image_size])
           __a							=    model(__SCREAMING_SNAKE_CASE)
           self.parent.assertEqual(result.logits.shape					,		(self.batch_size, 1, self.image_size, self.image_size))



       def 		_lowerCamelCase      (		self							:					Optional[int]					,		__SCREAMING_SNAKE_CASE							:					Tuple					,		__SCREAMING_SNAKE_CASE							:					List[Any]					,		__SCREAMING_SNAKE_CASE							:					List[Any]):


           '''simple docstring'''

           __a							=    self.type_sequence_label_size
           __a							=    SwinvaForImageClassification(__SCREAMING_SNAKE_CASE)
           model.to(__SCREAMING_SNAKE_CASE)
           model.eval()
           __a							=    model(__SCREAMING_SNAKE_CASE					,		labels=__SCREAMING_SNAKE_CASE)
           self.parent.assertEqual(result.logits.shape					,		(self.batch_size, self.type_sequence_label_size))





       def 		_lowerCamelCase      (		self							:					Optional[Any]):


           '''simple docstring'''

           __a							=    self.prepare_config_and_inputs()
           __a	,       __a	,       __a							=    config_and_inputs
           __a							=    {'''pixel_values''': pixel_values}
           return config, inputs_dict


@require_torch
class    _A	(     __UpperCAmelCase				,__UpperCAmelCase				,unittest.TestCase   ):
       UpperCamelCase__  :   Dict						 =						(
           (SwinvaModel, SwinvaForImageClassification, SwinvaForMaskedImageModeling) if is_torch_available() else ()
       )
       UpperCamelCase__  :   Optional[int]						 =						(
           {'''feature-extraction''': SwinvaModel, '''image-classification''': SwinvaForImageClassification}
           if is_torch_available()
           else {}
       )

       UpperCamelCase__  :   int						 =						False
       UpperCamelCase__  :   Tuple						 =						False
       UpperCamelCase__  :   Optional[Any]						 =						False
       UpperCamelCase__  :   Optional[Any]						 =						False



       def 		_lowerCamelCase      (		self							:					List[Any]):


           '''simple docstring'''

           __a							=    SwinvaModelTester(self)
           __a							=    ConfigTester(self					,		config_class=__SCREAMING_SNAKE_CASE					,		embed_dim=37)



       def 		_lowerCamelCase      (		self							:					List[Any]):


           '''simple docstring'''

           self.config_tester.create_and_test_config_to_json_string()
           self.config_tester.create_and_test_config_to_json_file()
           self.config_tester.create_and_test_config_from_and_save_pretrained()
           self.config_tester.create_and_test_config_with_num_labels()
           self.config_tester.check_config_can_be_init_without_params()
           self.config_tester.check_config_arguments_init()



       def 		_lowerCamelCase      (		self							:					Optional[Any]):


           '''simple docstring'''

           __a							=    self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_model(*__SCREAMING_SNAKE_CASE)



       @unittest.skip(reason='''Got `CUDA error: misaligned address` with PyTorch 2.0.0.''')
       def 		_lowerCamelCase      (		self							:					List[Any]):


           '''simple docstring'''

           pass



       @unittest.skip(reason='''Swinv2 does not use inputs_embeds''')
       def 		_lowerCamelCase      (		self							:					List[str]):


           '''simple docstring'''

           pass



       def 		_lowerCamelCase      (		self							:					Any):


           '''simple docstring'''

           __a	,       __a							=    self.model_tester.prepare_config_and_inputs_for_common()

           for model_class in self.all_model_classes:
               __a							=    model_class(__SCREAMING_SNAKE_CASE)
               self.assertIsInstance(model.get_input_embeddings()					,		(nn.Module))
               __a							=    model.get_output_embeddings()
               self.assertTrue(x is None or isinstance(__SCREAMING_SNAKE_CASE					,		nn.Linear))



       def 		_lowerCamelCase      (		self							:					Optional[int]):


           '''simple docstring'''

           __a	,       __a							=    self.model_tester.prepare_config_and_inputs_for_common()

           for model_class in self.all_model_classes:
               __a							=    model_class(__SCREAMING_SNAKE_CASE)
               __a							=    inspect.signature(model.forward)
               # signature.parameters is an OrderedDict => so arg_names order is deterministic
               __a							=    [*signature.parameters.keys()]

               __a							=    ['''pixel_values''']
               self.assertListEqual(arg_names[:1]					,		__SCREAMING_SNAKE_CASE)



       def 		_lowerCamelCase      (		self							:					Union[str, Any]):


           '''simple docstring'''

           __a	,       __a							=    self.model_tester.prepare_config_and_inputs_for_common()
           __a							=    True

           for model_class in self.all_model_classes:
               __a							=    True
               __a							=    False
               __a							=    True
               __a							=    model_class(__SCREAMING_SNAKE_CASE)
               model.to(__SCREAMING_SNAKE_CASE)
               model.eval()
               with torch.no_grad():
                   __a							=    model(**self._prepare_for_class(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE))
               __a							=    outputs.attentions
               __a							=    len(self.model_tester.depths)
               self.assertEqual(len(__SCREAMING_SNAKE_CASE)					,		__SCREAMING_SNAKE_CASE)

               # check that output_attentions also work using config
               del inputs_dict["output_attentions"]
               __a							=    True
               __a							=    config.window_size**2
               __a							=    model_class(__SCREAMING_SNAKE_CASE)
               model.to(__SCREAMING_SNAKE_CASE)
               model.eval()
               with torch.no_grad():
                   __a							=    model(**self._prepare_for_class(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE))
               __a							=    outputs.attentions
               self.assertEqual(len(__SCREAMING_SNAKE_CASE)					,		__SCREAMING_SNAKE_CASE)

               self.assertListEqual(
                   list(attentions[0].shape[-3:])					,		[self.model_tester.num_heads[0], window_size_squared, window_size_squared]					,		)
               __a							=    len(__SCREAMING_SNAKE_CASE)

               # Check attention is always last and order is fine
               __a							=    True
               __a							=    True
               __a							=    model_class(__SCREAMING_SNAKE_CASE)
               model.to(__SCREAMING_SNAKE_CASE)
               model.eval()
               with torch.no_grad():
                   __a							=    model(**self._prepare_for_class(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE))

               if hasattr(self.model_tester					,		'''num_hidden_states_types'''):
                   __a							=    self.model_tester.num_hidden_states_types
               else:
                   # also another +1 for reshaped_hidden_states
                   __a							=    2
               self.assertEqual(out_len + added_hidden_states					,		len(__SCREAMING_SNAKE_CASE))

               __a							=    outputs.attentions

               self.assertEqual(len(__SCREAMING_SNAKE_CASE)					,		__SCREAMING_SNAKE_CASE)

               self.assertListEqual(
                   list(self_attentions[0].shape[-3:])					,		[self.model_tester.num_heads[0], window_size_squared, window_size_squared]					,		)



       def 		_lowerCamelCase      (		self							:					Optional[Any]					,		__SCREAMING_SNAKE_CASE							:					Any					,		__SCREAMING_SNAKE_CASE							:					Union[str, Any]					,		__SCREAMING_SNAKE_CASE							:					Dict					,		__SCREAMING_SNAKE_CASE							:					List[str]):


           '''simple docstring'''

           __a							=    model_class(__SCREAMING_SNAKE_CASE)
           model.to(__SCREAMING_SNAKE_CASE)
           model.eval()

           with torch.no_grad():
               __a							=    model(**self._prepare_for_class(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE))

           __a							=    outputs.hidden_states

           __a							=    getattr(
               self.model_tester					,		'''expected_num_hidden_layers'''					,		len(self.model_tester.depths) + 1)
           self.assertEqual(len(__SCREAMING_SNAKE_CASE)					,		__SCREAMING_SNAKE_CASE)

           # Swinv2 has a different seq_length
           __a							=    (
               config.patch_size
               if isinstance(config.patch_size					,		collections.abc.Iterable)
               else (config.patch_size, config.patch_size)
           )

           __a							=    (image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])

           self.assertListEqual(
               list(hidden_states[0].shape[-2:])					,		[num_patches, self.model_tester.embed_dim]					,		)

           __a							=    outputs.reshaped_hidden_states
           self.assertEqual(len(__SCREAMING_SNAKE_CASE)					,		__SCREAMING_SNAKE_CASE)

           __a	,       __a	,       __a	,       __a							=    reshaped_hidden_states[0].shape
           __a							=    (
               reshaped_hidden_states[0].view(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE					,		height * width).permute(0					,		2					,		1)
           )
           self.assertListEqual(
               list(reshaped_hidden_states.shape[-2:])					,		[num_patches, self.model_tester.embed_dim]					,		)



       def 		_lowerCamelCase      (		self							:					Optional[int]):


           '''simple docstring'''

           __a	,       __a							=    self.model_tester.prepare_config_and_inputs_for_common()

           __a							=    (
               self.model_tester.image_size
               if isinstance(self.model_tester.image_size					,		collections.abc.Iterable)
               else (self.model_tester.image_size, self.model_tester.image_size)
           )

           for model_class in self.all_model_classes:
               __a							=    True
               self.check_hidden_states_output(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE)

               # check that output_hidden_states also work using config
               del inputs_dict["output_hidden_states"]
               __a							=    True

               self.check_hidden_states_output(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE)



       def 		_lowerCamelCase      (		self							:					Optional[int]):


           '''simple docstring'''

           __a	,       __a							=    self.model_tester.prepare_config_and_inputs_for_common()
           __a							=    3

           __a							=    (
               self.model_tester.image_size
               if isinstance(self.model_tester.image_size					,		collections.abc.Iterable)
               else (self.model_tester.image_size, self.model_tester.image_size)
           )
           __a							=    (
               config.patch_size
               if isinstance(config.patch_size					,		collections.abc.Iterable)
               else (config.patch_size, config.patch_size)
           )

           __a							=    image_size[0] + patch_size[0] - (image_size[0] % patch_size[0])
           __a							=    image_size[1] + patch_size[1] - (image_size[1] % patch_size[1])

           for model_class in self.all_model_classes:
               __a							=    True
               self.check_hidden_states_output(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE					,		(padded_height, padded_width))

               # check that output_hidden_states also work using config
               del inputs_dict["output_hidden_states"]
               __a							=    True
               self.check_hidden_states_output(__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE					,		__SCREAMING_SNAKE_CASE					,		(padded_height, padded_width))



       def 		_lowerCamelCase      (		self							:					Union[str, Any]):


           '''simple docstring'''

           __a							=    self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_masked_image_modeling(*__SCREAMING_SNAKE_CASE)



       def 		_lowerCamelCase      (		self							:					List[Any]):


           '''simple docstring'''

           __a							=    self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_image_classification(*__SCREAMING_SNAKE_CASE)



       @slow
       def 		_lowerCamelCase      (		self							:					Union[str, Any]):


           '''simple docstring'''

           for model_name in SWINV2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
               __a							=    SwinvaModel.from_pretrained(__SCREAMING_SNAKE_CASE)
               self.assertIsNotNone(__SCREAMING_SNAKE_CASE)





       def 		_lowerCamelCase      (		self							:					Any):


           '''simple docstring'''

           __a	,       __a							=    self.model_tester.prepare_config_and_inputs_for_common()

           __a							=    _config_zero_init(__SCREAMING_SNAKE_CASE)
           for model_class in self.all_model_classes:
               __a							=    model_class(config=__SCREAMING_SNAKE_CASE)
               for name, param in model.named_parameters():
                   if "embeddings" not in name and "logit_scale" not in name and param.requires_grad:
                       self.assertIn(
                           ((param.data.mean() * 1E9).round() / 1E9).item()					,		[0.0, 1.0]					,		msg=F'Parameter {name} of model {model_class} seems not properly initialized'					,		)


@require_vision
@require_torch
class    _A	(     unittest.TestCase   ):



       @cached_property
       def 		_lowerCamelCase      (		self							:					Union[str, Any]):


           '''simple docstring'''

           return (
               AutoImageProcessor.from_pretrained('''microsoft/swinv2-tiny-patch4-window8-256''')
               if is_vision_available()
               else None
           )





       @slow
       def 		_lowerCamelCase      (		self							:					str):


           '''simple docstring'''

           __a							=    SwinvaForImageClassification.from_pretrained('''microsoft/swinv2-tiny-patch4-window8-256''').to(
               __SCREAMING_SNAKE_CASE)
           __a							=    self.default_image_processor

           __a							=    Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png''')
           __a							=    image_processor(images=__SCREAMING_SNAKE_CASE					,		return_tensors='''pt''').to(__SCREAMING_SNAKE_CASE)

           # forward pass
           with torch.no_grad():
               __a							=    model(**__SCREAMING_SNAKE_CASE)

           # verify the logits
           __a							=    torch.Size((1, 1_000))
           self.assertEqual(outputs.logits.shape					,		__SCREAMING_SNAKE_CASE)
           __a							=    torch.tensor([-0.39_47, -0.43_06, 0.00_26]).to(__SCREAMING_SNAKE_CASE)
           self.assertTrue(torch.allclose(outputs.logits[0, :3]					,		__SCREAMING_SNAKE_CASE					,		atol=1E-4))





