

import collections.abc
from typing import Optional, Tuple, Union

import torch
import torch.utils.checkpoint
from torch import nn
from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss

from ...activations import ACTaFN
from ...modeling_outputs import BaseModelOutputWithNoAttention, ImageClassifierOutputWithNoAttention
from ...modeling_utils import PreTrainedModel
from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging
from .configuration_poolformer import PoolFormerConfig


__UpperCAmelCase								=			logging.get_logger(__name__)

# General docstring
__UpperCAmelCase								=			"""PoolFormerConfig"""

# Base docstring
__UpperCAmelCase								=			"""sail/poolformer_s12"""
__UpperCAmelCase								=			[1, 512, 7, 7]

# Image classification docstring
__UpperCAmelCase								=			"""sail/poolformer_s12"""
__UpperCAmelCase								=			"""tabby, tabby cat"""

__UpperCAmelCase								=			[
    """sail/poolformer_s12""",
    # See all PoolFormer models at https://huggingface.co/models?filter=poolformer
]





def      snake_case_				(__A					:						Dict ,       __A					:						float = 0.0 ,       __A					:						bool = False					)      ->     List[str]:
   if drop_prob == 0.0 or not training:
      return input
   __lowerCAmelCase  :      str    =						1 - drop_prob
   __lowerCAmelCase  :      Tuple    =						(input.shape[0],) + (1,) * (input.ndim - 1)  # work with diff dim tensors, not just 2D ConvNets
   __lowerCAmelCase  :      Optional[int]    =						keep_prob + torch.rand(__A ,       dtype=input.dtype ,       device=input.device					)
   random_tensor.floor_()  # binarize
   __lowerCAmelCase  :      str    =						input.div(__A					) * random_tensor
   return output



class SCREAMING_SNAKE_CASE     (  nn.Module			):
  """simple docstring"""






  def __init__(      self			:   Optional[int]	,							lowerCAmelCase			:   Optional[float] = None   )							->     None:






     """simple docstring"""







     super().__init__()
     __lowerCAmelCase  :      int    =						drop_prob







  def 	SCREAMING_SNAKE_CASE   (      self			:   Tuple	,							lowerCAmelCase			:   torch.Tensor   )							->     torch.Tensor:






     """simple docstring"""







     return drop_path(lowerCAmelCase	,							self.drop_prob	,							self.training   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Any   )							->     str:






     """simple docstring"""







     return "p={}".format(self.drop_prob   )






class SCREAMING_SNAKE_CASE     (  nn.Module			):
  """simple docstring"""






  def __init__(      self			:   Dict	,							lowerCAmelCase			:   Optional[Any]	,							lowerCAmelCase			:   Dict	,							lowerCAmelCase			:   List[Any]	,							lowerCAmelCase			:   Optional[Any]	,							lowerCAmelCase			:   Tuple	,							lowerCAmelCase			:   int=None   )							->     Optional[Any]:






     """simple docstring"""







     super().__init__()
     __lowerCAmelCase  :      Any    =						patch_size if isinstance(lowerCAmelCase	,							collections.abc.Iterable   ) else (patch_size, patch_size)
     __lowerCAmelCase  :      Any    =						stride if isinstance(lowerCAmelCase	,							collections.abc.Iterable   ) else (stride, stride)
     __lowerCAmelCase  :      Dict    =						padding if isinstance(lowerCAmelCase	,							collections.abc.Iterable   ) else (padding, padding)

     __lowerCAmelCase  :      int    =						nn.Convad(lowerCAmelCase	,							lowerCAmelCase	,							kernel_size=lowerCAmelCase	,							stride=lowerCAmelCase	,							padding=lowerCAmelCase   )
     __lowerCAmelCase  :      Optional[Any]    =						norm_layer(lowerCAmelCase   ) if norm_layer else nn.Identity()







  def 	SCREAMING_SNAKE_CASE   (      self			:   Optional[int]	,							lowerCAmelCase			:   Any   )							->     Optional[int]:






     """simple docstring"""







     __lowerCAmelCase  :      Optional[int]    =						self.projection(lowerCAmelCase   )
     __lowerCAmelCase  :      List[Any]    =						self.norm(lowerCAmelCase   )
     return embeddings






class SCREAMING_SNAKE_CASE     (  nn.GroupNorm			):
  """simple docstring"""






  def __init__(      self			:   List[Any]	,							lowerCAmelCase			:   Optional[int]	,							**lowerCAmelCase			:   Any   )							->     Optional[Any]:






     """simple docstring"""







     super().__init__(1	,							lowerCAmelCase	,							**lowerCAmelCase   )






class SCREAMING_SNAKE_CASE     (  nn.Module			):
  """simple docstring"""






  def __init__(      self			:   Tuple	,							lowerCAmelCase			:   Optional[Any]   )							->     Optional[int]:






     """simple docstring"""







     super().__init__()
     __lowerCAmelCase  :      Any    =						nn.AvgPoolad(lowerCAmelCase	,							stride=1	,							padding=pool_size // 2	,							count_include_pad=lowerCAmelCase   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Union[str, Any]	,							lowerCAmelCase			:   Tuple   )							->     Optional[int]:






     """simple docstring"""







     return self.pool(lowerCAmelCase   ) - hidden_states






class SCREAMING_SNAKE_CASE     (  nn.Module			):
  """simple docstring"""






  def __init__(      self			:   Dict	,							lowerCAmelCase			:   Tuple	,							lowerCAmelCase			:   Union[str, Any]	,							lowerCAmelCase			:   Optional[int]	,							lowerCAmelCase			:   Optional[Any]   )							->     Any:






     """simple docstring"""







     super().__init__()
     __lowerCAmelCase  :      Optional[Any]    =						nn.Convad(lowerCAmelCase	,							lowerCAmelCase	,							1   )
     __lowerCAmelCase  :      Optional[int]    =						nn.Convad(lowerCAmelCase	,							lowerCAmelCase	,							1   )
     __lowerCAmelCase  :      Optional[int]    =						PoolFormerDropPath(lowerCAmelCase   )
     if isinstance(config.hidden_act	,							lowerCAmelCase   ):
        __lowerCAmelCase  :      Optional[int]    =						ACTaFN[config.hidden_act]
     else:
        __lowerCAmelCase  :      List[str]    =						config.hidden_act







  def 	SCREAMING_SNAKE_CASE   (      self			:   Optional[int]	,							lowerCAmelCase			:   str   )							->     Optional[int]:






     """simple docstring"""







     __lowerCAmelCase  :      Optional[int]    =						self.conva(lowerCAmelCase   )
     __lowerCAmelCase  :      Dict    =						self.act_fn(lowerCAmelCase   )
     __lowerCAmelCase  :      str    =						self.drop(lowerCAmelCase   )
     __lowerCAmelCase  :      Any    =						self.conva(lowerCAmelCase   )
     __lowerCAmelCase  :      Optional[int]    =						self.drop(lowerCAmelCase   )

     return hidden_states






class SCREAMING_SNAKE_CASE     (  nn.Module			):
  """simple docstring"""






  def __init__(      self			:   Any	,							lowerCAmelCase			:   Any	,							lowerCAmelCase			:   int	,							lowerCAmelCase			:   Union[str, Any]	,							lowerCAmelCase			:   Optional[Any]	,							lowerCAmelCase			:   List[str]	,							lowerCAmelCase			:   int   )							->     Union[str, Any]:






     """simple docstring"""







     super().__init__()
     __lowerCAmelCase  :      Union[str, Any]    =						PoolFormerPooling(lowerCAmelCase   )
     __lowerCAmelCase  :      Dict    =						PoolFormerOutput(lowerCAmelCase	,							lowerCAmelCase	,							lowerCAmelCase	,							lowerCAmelCase   )
     __lowerCAmelCase  :      Any    =						PoolFormerGroupNorm(lowerCAmelCase   )
     __lowerCAmelCase  :      Tuple    =						PoolFormerGroupNorm(lowerCAmelCase   )

     # Useful for training neural nets
     __lowerCAmelCase  :      List[str]    =						PoolFormerDropPath(lowerCAmelCase   ) if drop_path > 0.0 else nn.Identity()
     __lowerCAmelCase  :      Tuple    =						config.use_layer_scale
     if config.use_layer_scale:
        __lowerCAmelCase  :      Optional[int]    =						nn.Parameter(
            config.layer_scale_init_value * torch.ones((lowerCAmelCase)   )	,							requires_grad=lowerCAmelCase   )
        __lowerCAmelCase  :      str    =						nn.Parameter(
            config.layer_scale_init_value * torch.ones((lowerCAmelCase)   )	,							requires_grad=lowerCAmelCase   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   int	,							lowerCAmelCase			:   Optional[Any]   )							->     List[str]:






     """simple docstring"""







     if self.use_layer_scale:
        __lowerCAmelCase  :      Optional[int]    =						self.pooling(self.before_norm(lowerCAmelCase   )   )
        __lowerCAmelCase  :      int    =						self.layer_scale_a.unsqueeze(-1   ).unsqueeze(-1   ) * pooling_output
        # First residual connection
        __lowerCAmelCase  :      int    =						hidden_states + self.drop_path(lowerCAmelCase   )
        __lowerCAmelCase  :      Optional[int]    =						()

        __lowerCAmelCase  :      List[str]    =						self.output(self.after_norm(lowerCAmelCase   )   )
        __lowerCAmelCase  :      List[Any]    =						self.layer_scale_a.unsqueeze(-1   ).unsqueeze(-1   ) * layer_output
        # Second residual connection
        __lowerCAmelCase  :      Union[str, Any]    =						hidden_states + self.drop_path(lowerCAmelCase   )

        __lowerCAmelCase  :      List[Any]    =						(output,) + outputs
        return outputs

     else:
        __lowerCAmelCase  :      Dict    =						self.drop_path(self.pooling(self.before_norm(lowerCAmelCase   )   )   )
        # First residual connection
        __lowerCAmelCase  :      str    =						pooling_output + hidden_states
        __lowerCAmelCase  :      Optional[Any]    =						()

        # Second residual connection inside the PoolFormerOutput block
        __lowerCAmelCase  :      Tuple    =						self.drop_path(self.output(self.after_norm(lowerCAmelCase   )   )   )
        __lowerCAmelCase  :      Optional[Any]    =						hidden_states + layer_output

        __lowerCAmelCase  :      Any    =						(output,) + outputs
        return outputs






class SCREAMING_SNAKE_CASE     (  nn.Module			):
  """simple docstring"""






  def __init__(      self			:   Optional[int]	,							lowerCAmelCase			:   Optional[int]   )							->     Any:






     """simple docstring"""







     super().__init__()
     __lowerCAmelCase  :      Tuple    =						config
     # stochastic depth decay rule
     __lowerCAmelCase  :      Any    =						[x.item() for x in torch.linspace(0	,							config.drop_path_rate	,							sum(config.depths   )   )]

     # patch embeddings
     __lowerCAmelCase  :      int    =						[]
     for i in range(config.num_encoder_blocks   ):
        embeddings.append(
            PoolFormerEmbeddings(
                patch_size=config.patch_sizes[i]	,							stride=config.strides[i]	,							padding=config.padding[i]	,							num_channels=config.num_channels if i == 0 else config.hidden_sizes[i - 1]	,							hidden_size=config.hidden_sizes[i]	,							)   )
     __lowerCAmelCase  :      Optional[int]    =						nn.ModuleList(lowerCAmelCase   )

     # Transformer blocks
     __lowerCAmelCase  :      Optional[Any]    =						[]
     __lowerCAmelCase  :      Optional[int]    =						0
     for i in range(config.num_encoder_blocks   ):
        # each block consists of layers
        __lowerCAmelCase  :      Union[str, Any]    =						[]
        if i != 0:
           cur += config.depths[i - 1]
        for j in range(config.depths[i]   ):
           layers.append(
               PoolFormerLayer(
                   lowerCAmelCase	,							num_channels=config.hidden_sizes[i]	,							pool_size=config.pool_size	,							hidden_size=config.hidden_sizes[i]	,							intermediate_size=int(config.hidden_sizes[i] * config.mlp_ratio   )	,							drop_path=dpr[cur + j]	,							)   )
        blocks.append(nn.ModuleList(lowerCAmelCase   )   )

     __lowerCAmelCase  :      List[Any]    =						nn.ModuleList(lowerCAmelCase   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Any	,							lowerCAmelCase			:   Optional[int]	,							lowerCAmelCase			:   Tuple=False	,							lowerCAmelCase			:   str=True   )							->     Tuple:






     """simple docstring"""







     __lowerCAmelCase  :      List[str]    =						() if output_hidden_states else None

     __lowerCAmelCase  :      Optional[Any]    =						pixel_values
     for idx, layers in enumerate(zip(self.patch_embeddings	,							self.block   )   ):
        __lowerCAmelCase  :      str    =						layers
        # Get patch embeddings from hidden_states
        __lowerCAmelCase  :      Dict    =						embedding_layer(lowerCAmelCase   )
        # Send the embeddings through the blocks
        for _, blk in enumerate(lowerCAmelCase   ):
           __lowerCAmelCase  :      int    =						blk(lowerCAmelCase   )
           __lowerCAmelCase  :      Optional[int]    =						layer_outputs[0]

        if output_hidden_states:
           __lowerCAmelCase  :      Optional[int]    =						all_hidden_states + (hidden_states,)

     if not return_dict:
        return tuple(v for v in [hidden_states, all_hidden_states] if v is not None   )

     return BaseModelOutputWithNoAttention(last_hidden_state=lowerCAmelCase	,							hidden_states=lowerCAmelCase   )






class SCREAMING_SNAKE_CASE     (  a_			):
  """simple docstring"""






  lowerCamelCase   :      List[str]										=PoolFormerConfig
  lowerCamelCase   :      Tuple										="poolformer"
  lowerCamelCase   :      List[Any]										="pixel_values"
  lowerCamelCase   :      Optional[int]										=True







  def 	SCREAMING_SNAKE_CASE   (      self			:   Optional[int]	,							lowerCAmelCase			:   Union[str, Any]   )							->     Union[str, Any]:






     """simple docstring"""







     if isinstance(lowerCAmelCase	,							(nn.Linear, nn.Convad)   ):
        module.weight.data.normal_(mean=0.0	,							std=self.config.initializer_range   )
        if module.bias is not None:
           module.bias.data.zero_()
     elif isinstance(lowerCAmelCase	,							nn.LayerNorm   ):
        module.bias.data.zero_()
        module.weight.data.fill_(1.0   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Dict	,							lowerCAmelCase			:   Optional[int]	,							lowerCAmelCase			:   int=False   )							->     Tuple:






     """simple docstring"""







     if isinstance(lowerCAmelCase	,							lowerCAmelCase   ):
        __lowerCAmelCase  :      Dict    =						value






__UpperCAmelCase								=			R"""
    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use
    it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
    behavior.

    Parameters:
        config ([`PoolFormerConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""

__UpperCAmelCase								=			R"""
    Args:
        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See
            [`PoolFormerImageProcessor.__call__`] for details.
"""



@add_start_docstrings(
    "The bare PoolFormer Model transformer outputting raw hidden-states without any specific head on top."       ,							a_       ,							)
class SCREAMING_SNAKE_CASE     (  a_			):
  """simple docstring"""






  def __init__(      self			:   Union[str, Any]	,							lowerCAmelCase			:   str   )							->     Tuple:






     """simple docstring"""







     super().__init__(lowerCAmelCase   )
     __lowerCAmelCase  :      Dict    =						config

     __lowerCAmelCase  :      Dict    =						PoolFormerEncoder(lowerCAmelCase   )

     # Initialize weights and apply final processing
     self.post_init()







  def 	SCREAMING_SNAKE_CASE   (      self			:   Tuple   )							->     str:






     """simple docstring"""







     return self.embeddings.patch_embeddings







  @add_start_docstrings_to_model_forward(lowerCAmelCase   )
  @add_code_sample_docstrings(
      checkpoint=_CHECKPOINT_FOR_DOC	,							output_type=lowerCAmelCase	,							config_class=_CONFIG_FOR_DOC	,							modality="""vision"""	,							expected_output=_EXPECTED_OUTPUT_SHAPE	,							)
  def 	SCREAMING_SNAKE_CASE   (      self			:   int	,							lowerCAmelCase			:   Optional[torch.FloatTensor] = None	,							lowerCAmelCase			:   Optional[bool] = None	,							lowerCAmelCase			:   Optional[bool] = None	,							)							->     Union[Tuple, BaseModelOutputWithNoAttention]:






     """simple docstring"""







     __lowerCAmelCase  :      Union[str, Any]    =						(
         output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
     )
     __lowerCAmelCase  :      List[Any]    =						return_dict if return_dict is not None else self.config.use_return_dict

     if pixel_values is None:
        raise ValueError("""You have to specify pixel_values"""   )

     __lowerCAmelCase  :      Optional[int]    =						self.encoder(
         lowerCAmelCase	,							output_hidden_states=lowerCAmelCase	,							return_dict=lowerCAmelCase	,							)
     __lowerCAmelCase  :      Any    =						encoder_outputs[0]

     if not return_dict:
        return (sequence_output, None) + encoder_outputs[1:]

     return BaseModelOutputWithNoAttention(
         last_hidden_state=lowerCAmelCase	,							hidden_states=encoder_outputs.hidden_states	,							)






class SCREAMING_SNAKE_CASE     (  nn.Module			):
  """simple docstring"""






  def __init__(      self			:   str	,							lowerCAmelCase			:   Optional[int]   )							->     Union[str, Any]:






     """simple docstring"""







     super().__init__()
     __lowerCAmelCase  :      Optional[int]    =						nn.Linear(config.hidden_size	,							config.hidden_size   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Union[str, Any]	,							lowerCAmelCase			:   Optional[Any]   )							->     Optional[int]:






     """simple docstring"""







     __lowerCAmelCase  :      Tuple    =						self.dense(lowerCAmelCase   )
     return output






@add_start_docstrings(
    "\n    PoolFormer Model transformer with an image classification head on top\n    "       ,							a_       ,							)
class SCREAMING_SNAKE_CASE     (  a_			):
  """simple docstring"""






  def __init__(      self			:   Optional[int]	,							lowerCAmelCase			:   Tuple   )							->     Optional[Any]:






     """simple docstring"""







     super().__init__(lowerCAmelCase   )
     __lowerCAmelCase  :      str    =						config.num_labels
     __lowerCAmelCase  :      Dict    =						PoolFormerModel(lowerCAmelCase   )

     # Final norm
     __lowerCAmelCase  :      List[Any]    =						PoolFormerGroupNorm(config.hidden_sizes[-1]   )
     # Classifier head
     __lowerCAmelCase  :      int    =						(
         nn.Linear(config.hidden_sizes[-1]	,							config.num_labels   ) if config.num_labels > 0 else nn.Identity()
     )

     # Initialize weights and apply final processing
     self.post_init()







  @add_start_docstrings_to_model_forward(lowerCAmelCase   )
  @add_code_sample_docstrings(
      checkpoint=_IMAGE_CLASS_CHECKPOINT	,							output_type=lowerCAmelCase	,							config_class=_CONFIG_FOR_DOC	,							expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT	,							)
  def 	SCREAMING_SNAKE_CASE   (      self			:   List[Any]	,							lowerCAmelCase			:   Optional[torch.FloatTensor] = None	,							lowerCAmelCase			:   Optional[torch.LongTensor] = None	,							lowerCAmelCase			:   Optional[bool] = None	,							lowerCAmelCase			:   Optional[bool] = None	,							)							->     Union[Tuple, ImageClassifierOutputWithNoAttention]:






     """simple docstring"""







     __lowerCAmelCase  :      Union[str, Any]    =						return_dict if return_dict is not None else self.config.use_return_dict

     __lowerCAmelCase  :      List[Any]    =						self.poolformer(
         lowerCAmelCase	,							output_hidden_states=lowerCAmelCase	,							return_dict=lowerCAmelCase	,							)

     __lowerCAmelCase  :      Any    =						outputs[0]

     __lowerCAmelCase  :      str    =						self.classifier(self.norm(lowerCAmelCase   ).mean([-2, -1]   )   )

     __lowerCAmelCase  :      Union[str, Any]    =						None
     if labels is not None:
        if self.config.problem_type is None:
           if self.num_labels == 1:
              __lowerCAmelCase  :      Any    =						"""regression"""
           elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
              __lowerCAmelCase  :      Tuple    =						"""single_label_classification"""
           else:
              __lowerCAmelCase  :      Optional[Any]    =						"""multi_label_classification"""

        if self.config.problem_type == "regression":
           __lowerCAmelCase  :      List[str]    =						MSELoss()
           if self.num_labels == 1:
              __lowerCAmelCase  :      Dict    =						loss_fct(logits.squeeze()	,							labels.squeeze()   )
           else:
              __lowerCAmelCase  :      int    =						loss_fct(lowerCAmelCase	,							lowerCAmelCase   )
        elif self.config.problem_type == "single_label_classification":
           __lowerCAmelCase  :      Tuple    =						CrossEntropyLoss()
           __lowerCAmelCase  :      Dict    =						loss_fct(logits.view(-1	,							self.num_labels   )	,							labels.view(-1   )   )
        elif self.config.problem_type == "multi_label_classification":
           __lowerCAmelCase  :      Optional[int]    =						BCEWithLogitsLoss()
           __lowerCAmelCase  :      Any    =						loss_fct(lowerCAmelCase	,							lowerCAmelCase   )

     if not return_dict:
        __lowerCAmelCase  :      Optional[int]    =						(logits,) + outputs[2:]
        return ((loss,) + output) if loss is not None else output

     return ImageClassifierOutputWithNoAttention(loss=lowerCAmelCase	,							logits=lowerCAmelCase	,							hidden_states=outputs.hidden_states   )





import unittest

from transformers import MobileBertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
   import torch

   from transformers import (
       MODEL_FOR_PRETRAINING_MAPPING,
       MobileBertForMaskedLM,
       MobileBertForMultipleChoice,
       MobileBertForNextSentencePrediction,
       MobileBertForPreTraining,
       MobileBertForQuestionAnswering,
       MobileBertForSequenceClassification,
       MobileBertForTokenClassification,
       MobileBertModel,
   )



class SCREAMING_SNAKE_CASE     :
  """simple docstring"""






  def __init__(      self			:   Union[str, Any]	,							lowerCAmelCase			:   Dict	,							lowerCAmelCase			:   Dict=13	,							lowerCAmelCase			:   int=7	,							lowerCAmelCase			:   Any=True	,							lowerCAmelCase			:   str=True	,							lowerCAmelCase			:   Optional[Any]=True	,							lowerCAmelCase			:   str=True	,							lowerCAmelCase			:   Tuple=99	,							lowerCAmelCase			:   int=64	,							lowerCAmelCase			:   Any=32	,							lowerCAmelCase			:   str=5	,							lowerCAmelCase			:   List[str]=4	,							lowerCAmelCase			:   str=37	,							lowerCAmelCase			:   Union[str, Any]="gelu"	,							lowerCAmelCase			:   Dict=0.1	,							lowerCAmelCase			:   Dict=0.1	,							lowerCAmelCase			:   Optional[int]=5_12	,							lowerCAmelCase			:   List[str]=16	,							lowerCAmelCase			:   str=2	,							lowerCAmelCase			:   Union[str, Any]=0.02	,							lowerCAmelCase			:   Dict=3	,							lowerCAmelCase			:   int=4	,							lowerCAmelCase			:   Union[str, Any]=None	,							)							->     Optional[int]:






     """simple docstring"""







     __lowerCAmelCase  :      List[Any]    =						parent
     __lowerCAmelCase  :      Tuple    =						batch_size
     __lowerCAmelCase  :      Dict    =						seq_length
     __lowerCAmelCase  :      List[str]    =						is_training
     __lowerCAmelCase  :      Dict    =						use_input_mask
     __lowerCAmelCase  :      Optional[int]    =						use_token_type_ids
     __lowerCAmelCase  :      List[str]    =						use_labels
     __lowerCAmelCase  :      Dict    =						vocab_size
     __lowerCAmelCase  :      List[str]    =						hidden_size
     __lowerCAmelCase  :      Optional[int]    =						embedding_size
     __lowerCAmelCase  :      Optional[int]    =						num_hidden_layers
     __lowerCAmelCase  :      Optional[Any]    =						num_attention_heads
     __lowerCAmelCase  :      Optional[Any]    =						intermediate_size
     __lowerCAmelCase  :      Optional[int]    =						hidden_act
     __lowerCAmelCase  :      Any    =						hidden_dropout_prob
     __lowerCAmelCase  :      Optional[int]    =						attention_probs_dropout_prob
     __lowerCAmelCase  :      List[str]    =						max_position_embeddings
     __lowerCAmelCase  :      Optional[Any]    =						type_vocab_size
     __lowerCAmelCase  :      Optional[Any]    =						type_sequence_label_size
     __lowerCAmelCase  :      Optional[Any]    =						initializer_range
     __lowerCAmelCase  :      Optional[Any]    =						num_labels
     __lowerCAmelCase  :      Union[str, Any]    =						num_choices
     __lowerCAmelCase  :      Union[str, Any]    =						scope







  def 	SCREAMING_SNAKE_CASE   (      self			:   Dict   )							->     List[str]:






     """simple docstring"""







     __lowerCAmelCase  :      Union[str, Any]    =						ids_tensor([self.batch_size, self.seq_length]	,							self.vocab_size   )

     __lowerCAmelCase  :      Dict    =						None
     if self.use_input_mask:
        __lowerCAmelCase  :      Union[str, Any]    =						random_attention_mask([self.batch_size, self.seq_length]   )

     __lowerCAmelCase  :      List[str]    =						None
     if self.use_token_type_ids:
        __lowerCAmelCase  :      Tuple    =						ids_tensor([self.batch_size, self.seq_length]	,							self.type_vocab_size   )

     __lowerCAmelCase  :      List[Any]    =						None
     __lowerCAmelCase  :      Tuple    =						None
     __lowerCAmelCase  :      int    =						None
     if self.use_labels:
        __lowerCAmelCase  :      Tuple    =						ids_tensor([self.batch_size]	,							self.type_sequence_label_size   )
        __lowerCAmelCase  :      List[str]    =						ids_tensor([self.batch_size, self.seq_length]	,							self.num_labels   )
        __lowerCAmelCase  :      str    =						ids_tensor([self.batch_size]	,							self.num_choices   )

     __lowerCAmelCase  :      Optional[int]    =						self.get_config()

     return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels







  def 	SCREAMING_SNAKE_CASE   (      self			:   List[Any]   )							->     Union[str, Any]:






     """simple docstring"""







     return MobileBertConfig(
         vocab_size=self.vocab_size	,							hidden_size=self.hidden_size	,							num_hidden_layers=self.num_hidden_layers	,							num_attention_heads=self.num_attention_heads	,							intermediate_size=self.intermediate_size	,							embedding_size=self.embedding_size	,							hidden_act=self.hidden_act	,							hidden_dropout_prob=self.hidden_dropout_prob	,							attention_probs_dropout_prob=self.attention_probs_dropout_prob	,							max_position_embeddings=self.max_position_embeddings	,							type_vocab_size=self.type_vocab_size	,							is_decoder=lowerCAmelCase	,							initializer_range=self.initializer_range	,							)







  def 	SCREAMING_SNAKE_CASE   (      self			:   Optional[Any]	,							lowerCAmelCase			:   Dict	,							lowerCAmelCase			:   Optional[Any]	,							lowerCAmelCase			:   Optional[int]	,							lowerCAmelCase			:   int	,							lowerCAmelCase			:   Any	,							lowerCAmelCase			:   Tuple	,							lowerCAmelCase			:   List[Any]   )							->     Tuple:






     """simple docstring"""







     __lowerCAmelCase  :      Any    =						MobileBertModel(config=lowerCAmelCase   )
     model.to(lowerCAmelCase   )
     model.eval()
     __lowerCAmelCase  :      Union[str, Any]    =						model(lowerCAmelCase	,							attention_mask=lowerCAmelCase	,							token_type_ids=lowerCAmelCase   )
     __lowerCAmelCase  :      List[Any]    =						model(lowerCAmelCase	,							token_type_ids=lowerCAmelCase   )
     __lowerCAmelCase  :      Tuple    =						model(lowerCAmelCase   )

     self.parent.assertEqual(result.last_hidden_state.shape	,							(self.batch_size, self.seq_length, self.hidden_size)   )
     self.parent.assertEqual(result.pooler_output.shape	,							(self.batch_size, self.hidden_size)   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Optional[int]	,							lowerCAmelCase			:   Union[str, Any]	,							lowerCAmelCase			:   Any	,							lowerCAmelCase			:   str	,							lowerCAmelCase			:   List[Any]	,							lowerCAmelCase			:   Dict	,							lowerCAmelCase			:   Union[str, Any]	,							lowerCAmelCase			:   Optional[int]   )							->     Optional[Any]:






     """simple docstring"""







     __lowerCAmelCase  :      Any    =						MobileBertForMaskedLM(config=lowerCAmelCase   )
     model.to(lowerCAmelCase   )
     model.eval()
     __lowerCAmelCase  :      List[Any]    =						model(lowerCAmelCase	,							attention_mask=lowerCAmelCase	,							token_type_ids=lowerCAmelCase	,							labels=lowerCAmelCase   )
     self.parent.assertEqual(result.logits.shape	,							(self.batch_size, self.seq_length, self.vocab_size)   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Dict	,							lowerCAmelCase			:   List[str]	,							lowerCAmelCase			:   Union[str, Any]	,							lowerCAmelCase			:   Union[str, Any]	,							lowerCAmelCase			:   str	,							lowerCAmelCase			:   Optional[Any]	,							lowerCAmelCase			:   Dict	,							lowerCAmelCase			:   Optional[int]   )							->     List[str]:






     """simple docstring"""







     __lowerCAmelCase  :      Tuple    =						MobileBertForNextSentencePrediction(config=lowerCAmelCase   )
     model.to(lowerCAmelCase   )
     model.eval()
     __lowerCAmelCase  :      List[Any]    =						model(
         lowerCAmelCase	,							attention_mask=lowerCAmelCase	,							token_type_ids=lowerCAmelCase	,							labels=lowerCAmelCase	,							)
     self.parent.assertEqual(result.logits.shape	,							(self.batch_size, 2)   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Tuple	,							lowerCAmelCase			:   Any	,							lowerCAmelCase			:   Any	,							lowerCAmelCase			:   Optional[int]	,							lowerCAmelCase			:   Union[str, Any]	,							lowerCAmelCase			:   Tuple	,							lowerCAmelCase			:   List[Any]	,							lowerCAmelCase			:   List[Any]   )							->     Optional[Any]:






     """simple docstring"""







     __lowerCAmelCase  :      Any    =						MobileBertForPreTraining(config=lowerCAmelCase   )
     model.to(lowerCAmelCase   )
     model.eval()
     __lowerCAmelCase  :      Optional[int]    =						model(
         lowerCAmelCase	,							attention_mask=lowerCAmelCase	,							token_type_ids=lowerCAmelCase	,							labels=lowerCAmelCase	,							next_sentence_label=lowerCAmelCase	,							)
     self.parent.assertEqual(result.prediction_logits.shape	,							(self.batch_size, self.seq_length, self.vocab_size)   )
     self.parent.assertEqual(result.seq_relationship_logits.shape	,							(self.batch_size, 2)   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Dict	,							lowerCAmelCase			:   Tuple	,							lowerCAmelCase			:   int	,							lowerCAmelCase			:   Dict	,							lowerCAmelCase			:   Optional[int]	,							lowerCAmelCase			:   int	,							lowerCAmelCase			:   Union[str, Any]	,							lowerCAmelCase			:   Any   )							->     Any:






     """simple docstring"""







     __lowerCAmelCase  :      Optional[int]    =						MobileBertForQuestionAnswering(config=lowerCAmelCase   )
     model.to(lowerCAmelCase   )
     model.eval()
     __lowerCAmelCase  :      int    =						model(
         lowerCAmelCase	,							attention_mask=lowerCAmelCase	,							token_type_ids=lowerCAmelCase	,							start_positions=lowerCAmelCase	,							end_positions=lowerCAmelCase	,							)
     self.parent.assertEqual(result.start_logits.shape	,							(self.batch_size, self.seq_length)   )
     self.parent.assertEqual(result.end_logits.shape	,							(self.batch_size, self.seq_length)   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Dict	,							lowerCAmelCase			:   List[Any]	,							lowerCAmelCase			:   str	,							lowerCAmelCase			:   str	,							lowerCAmelCase			:   List[str]	,							lowerCAmelCase			:   int	,							lowerCAmelCase			:   Union[str, Any]	,							lowerCAmelCase			:   Union[str, Any]   )							->     Tuple:






     """simple docstring"""







     __lowerCAmelCase  :      List[str]    =						self.num_labels
     __lowerCAmelCase  :      int    =						MobileBertForSequenceClassification(lowerCAmelCase   )
     model.to(lowerCAmelCase   )
     model.eval()
     __lowerCAmelCase  :      Optional[int]    =						model(lowerCAmelCase	,							attention_mask=lowerCAmelCase	,							token_type_ids=lowerCAmelCase	,							labels=lowerCAmelCase   )
     self.parent.assertEqual(result.logits.shape	,							(self.batch_size, self.num_labels)   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Union[str, Any]	,							lowerCAmelCase			:   Optional[Any]	,							lowerCAmelCase			:   Union[str, Any]	,							lowerCAmelCase			:   Tuple	,							lowerCAmelCase			:   int	,							lowerCAmelCase			:   int	,							lowerCAmelCase			:   Any	,							lowerCAmelCase			:   Optional[int]   )							->     Optional[Any]:






     """simple docstring"""







     __lowerCAmelCase  :      List[Any]    =						self.num_labels
     __lowerCAmelCase  :      Dict    =						MobileBertForTokenClassification(config=lowerCAmelCase   )
     model.to(lowerCAmelCase   )
     model.eval()
     __lowerCAmelCase  :      Tuple    =						model(lowerCAmelCase	,							attention_mask=lowerCAmelCase	,							token_type_ids=lowerCAmelCase	,							labels=lowerCAmelCase   )
     self.parent.assertEqual(result.logits.shape	,							(self.batch_size, self.seq_length, self.num_labels)   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   List[str]	,							lowerCAmelCase			:   List[Any]	,							lowerCAmelCase			:   Dict	,							lowerCAmelCase			:   Union[str, Any]	,							lowerCAmelCase			:   Optional[Any]	,							lowerCAmelCase			:   Union[str, Any]	,							lowerCAmelCase			:   Any	,							lowerCAmelCase			:   List[Any]   )							->     Optional[Any]:






     """simple docstring"""







     __lowerCAmelCase  :      Any    =						self.num_choices
     __lowerCAmelCase  :      List[Any]    =						MobileBertForMultipleChoice(config=lowerCAmelCase   )
     model.to(lowerCAmelCase   )
     model.eval()
     __lowerCAmelCase  :      Dict    =						input_ids.unsqueeze(1   ).expand(-1	,							self.num_choices	,							-1   ).contiguous()
     __lowerCAmelCase  :      Union[str, Any]    =						token_type_ids.unsqueeze(1   ).expand(-1	,							self.num_choices	,							-1   ).contiguous()
     __lowerCAmelCase  :      Optional[int]    =						input_mask.unsqueeze(1   ).expand(-1	,							self.num_choices	,							-1   ).contiguous()
     __lowerCAmelCase  :      Optional[int]    =						model(
         lowerCAmelCase	,							attention_mask=lowerCAmelCase	,							token_type_ids=lowerCAmelCase	,							labels=lowerCAmelCase	,							)
     self.parent.assertEqual(result.logits.shape	,							(self.batch_size, self.num_choices)   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   str   )							->     List[str]:






     """simple docstring"""







     __lowerCAmelCase  :      Tuple    =						self.prepare_config_and_inputs()
     (
         (
         __lowerCAmelCase
     )   ,(
         __lowerCAmelCase
     )   ,(
         __lowerCAmelCase
     )   ,(
         __lowerCAmelCase
     )   ,(
         __lowerCAmelCase
     )   ,(
         __lowerCAmelCase
     )   ,(
         __lowerCAmelCase
     )   ,
     )  :      List[Any]    =						config_and_inputs
     __lowerCAmelCase  :      List[str]    =						{"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
     return config, inputs_dict






@require_torch
class SCREAMING_SNAKE_CASE     (  a_       ,							a_       ,							unittest.TestCase			):
  """simple docstring"""






  lowerCamelCase   :      str										=(
      (
          MobileBertModel,
          MobileBertForMaskedLM,
          MobileBertForMultipleChoice,
          MobileBertForNextSentencePrediction,
          MobileBertForPreTraining,
          MobileBertForQuestionAnswering,
          MobileBertForSequenceClassification,
          MobileBertForTokenClassification,
      )
      if is_torch_available()
      else ()
  )
  lowerCamelCase   :      Optional[int]										=(
      {
          "feature-extraction": MobileBertModel,
          "fill-mask": MobileBertForMaskedLM,
          "question-answering": MobileBertForQuestionAnswering,
          "text-classification": MobileBertForSequenceClassification,
          "token-classification": MobileBertForTokenClassification,
          "zero-shot": MobileBertForSequenceClassification,
      }
      if is_torch_available()
      else {}
  )
  lowerCamelCase   :      Union[str, Any]										=True







  def 	SCREAMING_SNAKE_CASE   (      self			:   List[Any]	,							lowerCAmelCase			:   str	,							lowerCAmelCase			:   List[str]	,							lowerCAmelCase			:   List[Any]=False   )							->     List[str]:






     """simple docstring"""







     __lowerCAmelCase  :      Union[str, Any]    =						super()._prepare_for_class(lowerCAmelCase	,							lowerCAmelCase	,							return_labels=lowerCAmelCase   )

     if return_labels:
        if model_class in get_values(lowerCAmelCase   ):
           __lowerCAmelCase  :      Tuple    =						torch.zeros(
               (self.model_tester.batch_size, self.model_tester.seq_length)	,							dtype=torch.long	,							device=lowerCAmelCase   )
           __lowerCAmelCase  :      Union[str, Any]    =						torch.zeros(
               self.model_tester.batch_size	,							dtype=torch.long	,							device=lowerCAmelCase   )
     return inputs_dict







  def 	SCREAMING_SNAKE_CASE   (      self			:   int   )							->     Optional[Any]:






     """simple docstring"""







     __lowerCAmelCase  :      int    =						MobileBertModelTester(self   )
     __lowerCAmelCase  :      Optional[Any]    =						ConfigTester(self	,							config_class=lowerCAmelCase	,							hidden_size=37   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   str   )							->     Optional[Any]:






     """simple docstring"""







     self.config_tester.run_common_tests()







  def 	SCREAMING_SNAKE_CASE   (      self			:   int   )							->     Optional[Any]:






     """simple docstring"""







     __lowerCAmelCase  :      List[str]    =						self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_mobilebert_model(*lowerCAmelCase   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Dict   )							->     Tuple:






     """simple docstring"""







     __lowerCAmelCase  :      str    =						self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_mobilebert_for_masked_lm(*lowerCAmelCase   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Optional[int]   )							->     Dict:






     """simple docstring"""







     __lowerCAmelCase  :      str    =						self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_mobilebert_for_multiple_choice(*lowerCAmelCase   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Union[str, Any]   )							->     Dict:






     """simple docstring"""







     __lowerCAmelCase  :      Dict    =						self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_mobilebert_for_next_sequence_prediction(*lowerCAmelCase   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Dict   )							->     Optional[int]:






     """simple docstring"""







     __lowerCAmelCase  :      Tuple    =						self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_mobilebert_for_pretraining(*lowerCAmelCase   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Union[str, Any]   )							->     str:






     """simple docstring"""







     __lowerCAmelCase  :      Dict    =						self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_mobilebert_for_question_answering(*lowerCAmelCase   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   List[str]   )							->     Tuple:






     """simple docstring"""







     __lowerCAmelCase  :      Dict    =						self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_mobilebert_for_sequence_classification(*lowerCAmelCase   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Optional[int]   )							->     Tuple:






     """simple docstring"""







     __lowerCAmelCase  :      List[str]    =						self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_mobilebert_for_token_classification(*lowerCAmelCase   )






def      snake_case_				(__A					:						Any					)      ->     Optional[Any]:
   return torch.tensor(
       __A ,       dtype=torch.long ,       device=__A ,       )


__UpperCAmelCase								=			1e-3



@require_torch
@require_sentencepiece
@require_tokenizers
class SCREAMING_SNAKE_CASE     (  unittest.TestCase			):
  """simple docstring"""






  @slow
  def 	SCREAMING_SNAKE_CASE   (      self			:   Dict   )							->     Union[str, Any]:






     """simple docstring"""







     __lowerCAmelCase  :      Optional[Any]    =						MobileBertModel.from_pretrained("""google/mobilebert-uncased"""   ).to(lowerCAmelCase   )
     __lowerCAmelCase  :      int    =						_long_tensor([[1_01, 71_10, 10_05, 10_56, 20_23, 1_13_33, 1_74_13, 10_29, 1_02]]   )
     with torch.no_grad():
        __lowerCAmelCase  :      List[str]    =						model(lowerCAmelCase   )[0]
     __lowerCAmelCase  :      List[Any]    =						torch.Size((1, 9, 5_12)   )
     self.assertEqual(output.shape	,							lowerCAmelCase   )
     __lowerCAmelCase  :      int    =						torch.tensor(
         [
             [
                 [-2.473_6526e07, 8.269_1656e04, 1.652_1838e05],
                 [-5.754_1704e-01, 3.905_6022e00, 4.401_1507e00],
                 [2.604_7359e00, 1.567_7652e00, -1.732_4188e-01],
             ]
         ]	,							device=lowerCAmelCase	,							)

     # MobileBERT results range from 10e0 to 10e8. Even a 0.0000001% difference with a value of 10e8 results in a
     # ~1 difference, it's therefore not a good idea to measure using addition.
     # Here, we instead divide the expected result with the result in order to obtain ~1. We then check that the
     # result is held between bounds: 1 - TOLERANCE < expected_result / result < 1 + TOLERANCE
     __lowerCAmelCase  :      Union[str, Any]    =						torch.all((expected_slice / output[..., :3, :3]) >= 1 - TOLERANCE   )
     __lowerCAmelCase  :      Union[str, Any]    =						torch.all((expected_slice / output[..., :3, :3]) <= 1 + TOLERANCE   )

     self.assertTrue(lower_bound and upper_bound   )


