

import argparse
from pathlib import Path

import requests
import torch
from PIL import Image

from transformers import (
    RobertaTokenizer,
    TrOCRConfig,
    TrOCRForCausalLM,
    TrOCRProcessor,
    VisionEncoderDecoderModel,
    ViTConfig,
    ViTImageProcessor,
    ViTModel,
)
from transformers.utils import logging


logging.set_verbosity_info()
__UpperCAmelCase								=			logging.get_logger(__name__)





def      snake_case_				(__A					:						Optional[int] ,       __A					:						Any					)      ->     Any:
   __lowerCAmelCase  :      Union[str, Any]    =						[]
   for i in range(encoder_config.num_hidden_layers					):
      # encoder layers: output projection, 2 feedforward neural networks and 2 layernorms
      rename_keys.append(
          (f'''encoder.deit.blocks.{i}.norm1.weight''', f'''encoder.encoder.layer.{i}.layernorm_before.weight''')					)
      rename_keys.append((f'''encoder.deit.blocks.{i}.norm1.bias''', f'''encoder.encoder.layer.{i}.layernorm_before.bias''')					)
      rename_keys.append(
          (f'''encoder.deit.blocks.{i}.attn.proj.weight''', f'''encoder.encoder.layer.{i}.attention.output.dense.weight''')					)
      rename_keys.append(
          (f'''encoder.deit.blocks.{i}.attn.proj.bias''', f'''encoder.encoder.layer.{i}.attention.output.dense.bias''')					)
      rename_keys.append(
          (f'''encoder.deit.blocks.{i}.norm2.weight''', f'''encoder.encoder.layer.{i}.layernorm_after.weight''')					)
      rename_keys.append((f'''encoder.deit.blocks.{i}.norm2.bias''', f'''encoder.encoder.layer.{i}.layernorm_after.bias''')					)
      rename_keys.append(
          (f'''encoder.deit.blocks.{i}.mlp.fc1.weight''', f'''encoder.encoder.layer.{i}.intermediate.dense.weight''')					)
      rename_keys.append(
          (f'''encoder.deit.blocks.{i}.mlp.fc1.bias''', f'''encoder.encoder.layer.{i}.intermediate.dense.bias''')					)
      rename_keys.append(
          (f'''encoder.deit.blocks.{i}.mlp.fc2.weight''', f'''encoder.encoder.layer.{i}.output.dense.weight''')					)
      rename_keys.append((f'''encoder.deit.blocks.{i}.mlp.fc2.bias''', f'''encoder.encoder.layer.{i}.output.dense.bias''')					)

   # cls token, position embeddings and patch embeddings of encoder
   rename_keys.extend(
       [
           ("""encoder.deit.cls_token""", """encoder.embeddings.cls_token"""),
           ("""encoder.deit.pos_embed""", """encoder.embeddings.position_embeddings"""),
           ("""encoder.deit.patch_embed.proj.weight""", """encoder.embeddings.patch_embeddings.projection.weight"""),
           ("""encoder.deit.patch_embed.proj.bias""", """encoder.embeddings.patch_embeddings.projection.bias"""),
           ("""encoder.deit.norm.weight""", """encoder.layernorm.weight"""),
           ("""encoder.deit.norm.bias""", """encoder.layernorm.bias"""),
       ]					)

   return rename_keys





def      snake_case_				(__A					:						List[str] ,       __A					:						str					)      ->     Optional[Any]:
   for i in range(encoder_config.num_hidden_layers					):
      # queries, keys and values (only weights, no biases)
      __lowerCAmelCase  :      Optional[Any]    =						state_dict.pop(f'''encoder.deit.blocks.{i}.attn.qkv.weight'''					)

      __lowerCAmelCase  :      Tuple    =						in_proj_weight[
          : encoder_config.hidden_size, :
      ]
      __lowerCAmelCase  :      str    =						in_proj_weight[
          encoder_config.hidden_size : encoder_config.hidden_size * 2, :
      ]
      __lowerCAmelCase  :      str    =						in_proj_weight[
          -encoder_config.hidden_size :, :
      ]





def      snake_case_				(__A					:						Union[str, Any] ,       __A					:						str ,       __A					:						Optional[Any]					)      ->     Optional[Any]:
   __lowerCAmelCase  :      Any    =						dct.pop(__A					)
   __lowerCAmelCase  :      str    =						val





def      snake_case_				(__A					:						int					)      ->     Tuple:
   if "handwritten" in checkpoint_url:
      __lowerCAmelCase  :      Tuple    =						"""https://fki.tic.heia-fr.ch/static/img/a01-122-02-00.jpg"""  # industry
      # url = "https://fki.tic.heia-fr.ch/static/img/a01-122-02-12.jpg" # have
      # url = "https://fki.tic.heia-fr.ch/static/img/a01-122-02-10.jpg" # let
      # url = "https://fki.tic.heia-fr.ch/static/img/a01-122-02.jpg"  #
      # url = "https://fki.tic.heia-fr.ch/static/img/a01-122.jpg"
   elif "printed" in checkpoint_url or "stage1" in checkpoint_url:
      __lowerCAmelCase  :      Optional[Any]    =						"""https://www.researchgate.net/profile/Dinh-Sang/publication/338099565/figure/fig8/AS:840413229350922@1577381536857/An-receipt-example-in-the-SROIE-2019-dataset_Q640.jpg"""
   __lowerCAmelCase  :      Dict    =						Image.open(requests.get(__A ,       stream=__A					).raw					).convert("""RGB"""					)
   return im





@torch.no_grad()
def      snake_case_				(__A					:						Any ,       __A					:						Union[str, Any]					)      ->     Optional[int]:
   __lowerCAmelCase  :      List[Any]    =						ViTConfig(image_size=3_8_4 ,       qkv_bias=__A					)
   __lowerCAmelCase  :      List[Any]    =						TrOCRConfig()

   # size of the architecture
   if "base" in checkpoint_url:
      __lowerCAmelCase  :      Union[str, Any]    =						7_6_8
   elif "large" in checkpoint_url:
      # use ViT-large encoder
      __lowerCAmelCase  :      Any    =						1_0_2_4
      __lowerCAmelCase  :      Any    =						4_0_9_6
      __lowerCAmelCase  :      Optional[int]    =						2_4
      __lowerCAmelCase  :      str    =						1_6
      __lowerCAmelCase  :      List[Any]    =						1_0_2_4
   else:
      raise ValueError("""Should either find 'base' or 'large' in checkpoint URL"""					)

   # the large-printed + stage1 checkpoints uses sinusoidal position embeddings, no layernorm afterwards
   if "large-printed" in checkpoint_url or "stage1" in checkpoint_url:
      __lowerCAmelCase  :      Tuple    =						False
      __lowerCAmelCase  :      Union[str, Any]    =						"""relu"""
      __lowerCAmelCase  :      List[Any]    =						1_0_2_4
      __lowerCAmelCase  :      Any    =						True
      __lowerCAmelCase  :      List[Any]    =						False
      __lowerCAmelCase  :      Dict    =						False

   # load HuggingFace model
   __lowerCAmelCase  :      Dict    =						ViTModel(__A ,       add_pooling_layer=__A					)
   __lowerCAmelCase  :      Union[str, Any]    =						TrOCRForCausalLM(__A					)
   __lowerCAmelCase  :      Any    =						VisionEncoderDecoderModel(encoder=__A ,       decoder=__A					)
   model.eval()

   # load state_dict of original model, rename some keys
   __lowerCAmelCase  :      Union[str, Any]    =						torch.hub.load_state_dict_from_url(__A ,       map_location="""cpu""" ,       check_hash=__A					)["""model"""]

   __lowerCAmelCase  :      Any    =						create_rename_keys(__A ,       __A					)
   for src, dest in rename_keys:
      rename_key(__A ,       __A ,       __A					)
   read_in_q_k_v(__A ,       __A					)

   # remove parameters we don't need
   del state_dict["encoder.deit.head.weight"]
   del state_dict["encoder.deit.head.bias"]
   del state_dict["decoder.version"]

   # add prefix to decoder keys
   for key, val in state_dict.copy().items():
      __lowerCAmelCase  :      Tuple    =						state_dict.pop(__A					)
      if key.startswith("""decoder"""					) and "output_projection" not in key:
         __lowerCAmelCase  :      str    =						val
      else:
         __lowerCAmelCase  :      Tuple    =						val

    # load state dict
   model.load_state_dict(__A					)

   # Check outputs on an image
   __lowerCAmelCase  :      List[Any]    =						ViTImageProcessor(size=encoder_config.image_size					)
   __lowerCAmelCase  :      List[str]    =						RobertaTokenizer.from_pretrained("""roberta-large"""					)
   __lowerCAmelCase  :      List[Any]    =						TrOCRProcessor(__A ,       __A					)

   __lowerCAmelCase  :      List[str]    =						processor(images=prepare_img(__A					) ,       return_tensors="""pt"""					).pixel_values

   # verify logits
   __lowerCAmelCase  :      List[str]    =						torch.tensor([[model.config.decoder.decoder_start_token_id]]					)
   __lowerCAmelCase  :      List[str]    =						model(pixel_values=__A ,       decoder_input_ids=__A					)
   __lowerCAmelCase  :      Optional[Any]    =						outputs.logits

   __lowerCAmelCase  :      Union[str, Any]    =						torch.Size([1, 1, 5_0_2_6_5]					)
   if "trocr-base-handwritten" in checkpoint_url:
      __lowerCAmelCase  :      Dict    =						torch.tensor(
          [-1.4502, -4.6683, -0.5347, -2.9291, 9.1435, -3.0571, 8.9764, 1.7560, 8.7358, -1.5311]					)
   elif "trocr-large-handwritten" in checkpoint_url:
      __lowerCAmelCase  :      List[Any]    =						torch.tensor(
          [-2.6437, -1.3129, -2.2596, -5.3455, 6.3539, 1.7604, 5.4991, 1.4702, 5.6113, 2.0170]					)
   elif "trocr-base-printed" in checkpoint_url:
      __lowerCAmelCase  :      Tuple    =						torch.tensor(
          [-5.6816, -5.8388, 1.1398, -6.9034, 6.8505, -2.4393, 1.2284, -1.0232, -1.9661, -3.9210]					)
   elif "trocr-large-printed" in checkpoint_url:
      __lowerCAmelCase  :      List[Any]    =						torch.tensor(
          [-6.0162, -7.0959, 4.4155, -5.1063, 7.0468, -3.1631, 2.6466, -0.3081, -0.8106, -1.7535]					)

   if "stage1" not in checkpoint_url:
      assert logits.shape == expected_shape, "Shape of logits not as expected"
      assert torch.allclose(logits[0, 0, :1_0] ,       __A ,       atol=1e-3					), "First elements of logits not as expected"

   Path(__A					).mkdir(exist_ok=__A					)
   print(f'''Saving model to {pytorch_dump_folder_path}'''					)
   model.save_pretrained(__A					)
   print(f'''Saving processor to {pytorch_dump_folder_path}'''					)
   processor.save_pretrained(__A					)


if __name__ == "__main__":
   __UpperCAmelCase								=			argparse.ArgumentParser()

   parser.add_argument(
       """--checkpoint_url""",
       default="""https://layoutlm.blob.core.windows.net/trocr/model_zoo/fairseq/trocr-base-handwritten.pt""",
       type=str,
       help="""URL to the original PyTorch checkpoint (.pth file).""",
   )
   parser.add_argument(
       """--pytorch_dump_folder_path""", default=None, type=str, help="""Path to the folder to output PyTorch model."""
   )
   __UpperCAmelCase								=			parser.parse_args()
   convert_tr_ocr_checkpoint(args.checkpoint_url, args.pytorch_dump_folder_path)





import unittest

from transformers import MobileBertConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
   import torch

   from transformers import (
       MODEL_FOR_PRETRAINING_MAPPING,
       MobileBertForMaskedLM,
       MobileBertForMultipleChoice,
       MobileBertForNextSentencePrediction,
       MobileBertForPreTraining,
       MobileBertForQuestionAnswering,
       MobileBertForSequenceClassification,
       MobileBertForTokenClassification,
       MobileBertModel,
   )



class SCREAMING_SNAKE_CASE     :
  """simple docstring"""






  def __init__(      self			:   Union[str, Any]	,							lowerCAmelCase			:   Dict	,							lowerCAmelCase			:   Dict=13	,							lowerCAmelCase			:   int=7	,							lowerCAmelCase			:   Any=True	,							lowerCAmelCase			:   str=True	,							lowerCAmelCase			:   Optional[Any]=True	,							lowerCAmelCase			:   str=True	,							lowerCAmelCase			:   Tuple=99	,							lowerCAmelCase			:   int=64	,							lowerCAmelCase			:   Any=32	,							lowerCAmelCase			:   str=5	,							lowerCAmelCase			:   List[str]=4	,							lowerCAmelCase			:   str=37	,							lowerCAmelCase			:   Union[str, Any]="gelu"	,							lowerCAmelCase			:   Dict=0.1	,							lowerCAmelCase			:   Dict=0.1	,							lowerCAmelCase			:   Optional[int]=5_12	,							lowerCAmelCase			:   List[str]=16	,							lowerCAmelCase			:   str=2	,							lowerCAmelCase			:   Union[str, Any]=0.02	,							lowerCAmelCase			:   Dict=3	,							lowerCAmelCase			:   int=4	,							lowerCAmelCase			:   Union[str, Any]=None	,							)							->     Optional[int]:






     """simple docstring"""







     __lowerCAmelCase  :      List[Any]    =						parent
     __lowerCAmelCase  :      Tuple    =						batch_size
     __lowerCAmelCase  :      Dict    =						seq_length
     __lowerCAmelCase  :      List[str]    =						is_training
     __lowerCAmelCase  :      Dict    =						use_input_mask
     __lowerCAmelCase  :      Optional[int]    =						use_token_type_ids
     __lowerCAmelCase  :      List[str]    =						use_labels
     __lowerCAmelCase  :      Dict    =						vocab_size
     __lowerCAmelCase  :      List[str]    =						hidden_size
     __lowerCAmelCase  :      Optional[int]    =						embedding_size
     __lowerCAmelCase  :      Optional[int]    =						num_hidden_layers
     __lowerCAmelCase  :      Optional[Any]    =						num_attention_heads
     __lowerCAmelCase  :      Optional[Any]    =						intermediate_size
     __lowerCAmelCase  :      Optional[int]    =						hidden_act
     __lowerCAmelCase  :      Any    =						hidden_dropout_prob
     __lowerCAmelCase  :      Optional[int]    =						attention_probs_dropout_prob
     __lowerCAmelCase  :      List[str]    =						max_position_embeddings
     __lowerCAmelCase  :      Optional[Any]    =						type_vocab_size
     __lowerCAmelCase  :      Optional[Any]    =						type_sequence_label_size
     __lowerCAmelCase  :      Optional[Any]    =						initializer_range
     __lowerCAmelCase  :      Optional[Any]    =						num_labels
     __lowerCAmelCase  :      Union[str, Any]    =						num_choices
     __lowerCAmelCase  :      Union[str, Any]    =						scope







  def 	SCREAMING_SNAKE_CASE   (      self			:   Dict   )							->     List[str]:






     """simple docstring"""







     __lowerCAmelCase  :      Union[str, Any]    =						ids_tensor([self.batch_size, self.seq_length]	,							self.vocab_size   )

     __lowerCAmelCase  :      Dict    =						None
     if self.use_input_mask:
        __lowerCAmelCase  :      Union[str, Any]    =						random_attention_mask([self.batch_size, self.seq_length]   )

     __lowerCAmelCase  :      List[str]    =						None
     if self.use_token_type_ids:
        __lowerCAmelCase  :      Tuple    =						ids_tensor([self.batch_size, self.seq_length]	,							self.type_vocab_size   )

     __lowerCAmelCase  :      List[Any]    =						None
     __lowerCAmelCase  :      Tuple    =						None
     __lowerCAmelCase  :      int    =						None
     if self.use_labels:
        __lowerCAmelCase  :      Tuple    =						ids_tensor([self.batch_size]	,							self.type_sequence_label_size   )
        __lowerCAmelCase  :      List[str]    =						ids_tensor([self.batch_size, self.seq_length]	,							self.num_labels   )
        __lowerCAmelCase  :      str    =						ids_tensor([self.batch_size]	,							self.num_choices   )

     __lowerCAmelCase  :      Optional[int]    =						self.get_config()

     return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels







  def 	SCREAMING_SNAKE_CASE   (      self			:   List[Any]   )							->     Union[str, Any]:






     """simple docstring"""







     return MobileBertConfig(
         vocab_size=self.vocab_size	,							hidden_size=self.hidden_size	,							num_hidden_layers=self.num_hidden_layers	,							num_attention_heads=self.num_attention_heads	,							intermediate_size=self.intermediate_size	,							embedding_size=self.embedding_size	,							hidden_act=self.hidden_act	,							hidden_dropout_prob=self.hidden_dropout_prob	,							attention_probs_dropout_prob=self.attention_probs_dropout_prob	,							max_position_embeddings=self.max_position_embeddings	,							type_vocab_size=self.type_vocab_size	,							is_decoder=lowerCAmelCase	,							initializer_range=self.initializer_range	,							)







  def 	SCREAMING_SNAKE_CASE   (      self			:   Optional[Any]	,							lowerCAmelCase			:   Dict	,							lowerCAmelCase			:   Optional[Any]	,							lowerCAmelCase			:   Optional[int]	,							lowerCAmelCase			:   int	,							lowerCAmelCase			:   Any	,							lowerCAmelCase			:   Tuple	,							lowerCAmelCase			:   List[Any]   )							->     Tuple:






     """simple docstring"""







     __lowerCAmelCase  :      Any    =						MobileBertModel(config=lowerCAmelCase   )
     model.to(lowerCAmelCase   )
     model.eval()
     __lowerCAmelCase  :      Union[str, Any]    =						model(lowerCAmelCase	,							attention_mask=lowerCAmelCase	,							token_type_ids=lowerCAmelCase   )
     __lowerCAmelCase  :      List[Any]    =						model(lowerCAmelCase	,							token_type_ids=lowerCAmelCase   )
     __lowerCAmelCase  :      Tuple    =						model(lowerCAmelCase   )

     self.parent.assertEqual(result.last_hidden_state.shape	,							(self.batch_size, self.seq_length, self.hidden_size)   )
     self.parent.assertEqual(result.pooler_output.shape	,							(self.batch_size, self.hidden_size)   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Optional[int]	,							lowerCAmelCase			:   Union[str, Any]	,							lowerCAmelCase			:   Any	,							lowerCAmelCase			:   str	,							lowerCAmelCase			:   List[Any]	,							lowerCAmelCase			:   Dict	,							lowerCAmelCase			:   Union[str, Any]	,							lowerCAmelCase			:   Optional[int]   )							->     Optional[Any]:






     """simple docstring"""







     __lowerCAmelCase  :      Any    =						MobileBertForMaskedLM(config=lowerCAmelCase   )
     model.to(lowerCAmelCase   )
     model.eval()
     __lowerCAmelCase  :      List[Any]    =						model(lowerCAmelCase	,							attention_mask=lowerCAmelCase	,							token_type_ids=lowerCAmelCase	,							labels=lowerCAmelCase   )
     self.parent.assertEqual(result.logits.shape	,							(self.batch_size, self.seq_length, self.vocab_size)   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Dict	,							lowerCAmelCase			:   List[str]	,							lowerCAmelCase			:   Union[str, Any]	,							lowerCAmelCase			:   Union[str, Any]	,							lowerCAmelCase			:   str	,							lowerCAmelCase			:   Optional[Any]	,							lowerCAmelCase			:   Dict	,							lowerCAmelCase			:   Optional[int]   )							->     List[str]:






     """simple docstring"""







     __lowerCAmelCase  :      Tuple    =						MobileBertForNextSentencePrediction(config=lowerCAmelCase   )
     model.to(lowerCAmelCase   )
     model.eval()
     __lowerCAmelCase  :      List[Any]    =						model(
         lowerCAmelCase	,							attention_mask=lowerCAmelCase	,							token_type_ids=lowerCAmelCase	,							labels=lowerCAmelCase	,							)
     self.parent.assertEqual(result.logits.shape	,							(self.batch_size, 2)   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Tuple	,							lowerCAmelCase			:   Any	,							lowerCAmelCase			:   Any	,							lowerCAmelCase			:   Optional[int]	,							lowerCAmelCase			:   Union[str, Any]	,							lowerCAmelCase			:   Tuple	,							lowerCAmelCase			:   List[Any]	,							lowerCAmelCase			:   List[Any]   )							->     Optional[Any]:






     """simple docstring"""







     __lowerCAmelCase  :      Any    =						MobileBertForPreTraining(config=lowerCAmelCase   )
     model.to(lowerCAmelCase   )
     model.eval()
     __lowerCAmelCase  :      Optional[int]    =						model(
         lowerCAmelCase	,							attention_mask=lowerCAmelCase	,							token_type_ids=lowerCAmelCase	,							labels=lowerCAmelCase	,							next_sentence_label=lowerCAmelCase	,							)
     self.parent.assertEqual(result.prediction_logits.shape	,							(self.batch_size, self.seq_length, self.vocab_size)   )
     self.parent.assertEqual(result.seq_relationship_logits.shape	,							(self.batch_size, 2)   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Dict	,							lowerCAmelCase			:   Tuple	,							lowerCAmelCase			:   int	,							lowerCAmelCase			:   Dict	,							lowerCAmelCase			:   Optional[int]	,							lowerCAmelCase			:   int	,							lowerCAmelCase			:   Union[str, Any]	,							lowerCAmelCase			:   Any   )							->     Any:






     """simple docstring"""







     __lowerCAmelCase  :      Optional[int]    =						MobileBertForQuestionAnswering(config=lowerCAmelCase   )
     model.to(lowerCAmelCase   )
     model.eval()
     __lowerCAmelCase  :      int    =						model(
         lowerCAmelCase	,							attention_mask=lowerCAmelCase	,							token_type_ids=lowerCAmelCase	,							start_positions=lowerCAmelCase	,							end_positions=lowerCAmelCase	,							)
     self.parent.assertEqual(result.start_logits.shape	,							(self.batch_size, self.seq_length)   )
     self.parent.assertEqual(result.end_logits.shape	,							(self.batch_size, self.seq_length)   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Dict	,							lowerCAmelCase			:   List[Any]	,							lowerCAmelCase			:   str	,							lowerCAmelCase			:   str	,							lowerCAmelCase			:   List[str]	,							lowerCAmelCase			:   int	,							lowerCAmelCase			:   Union[str, Any]	,							lowerCAmelCase			:   Union[str, Any]   )							->     Tuple:






     """simple docstring"""







     __lowerCAmelCase  :      List[str]    =						self.num_labels
     __lowerCAmelCase  :      int    =						MobileBertForSequenceClassification(lowerCAmelCase   )
     model.to(lowerCAmelCase   )
     model.eval()
     __lowerCAmelCase  :      Optional[int]    =						model(lowerCAmelCase	,							attention_mask=lowerCAmelCase	,							token_type_ids=lowerCAmelCase	,							labels=lowerCAmelCase   )
     self.parent.assertEqual(result.logits.shape	,							(self.batch_size, self.num_labels)   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Union[str, Any]	,							lowerCAmelCase			:   Optional[Any]	,							lowerCAmelCase			:   Union[str, Any]	,							lowerCAmelCase			:   Tuple	,							lowerCAmelCase			:   int	,							lowerCAmelCase			:   int	,							lowerCAmelCase			:   Any	,							lowerCAmelCase			:   Optional[int]   )							->     Optional[Any]:






     """simple docstring"""







     __lowerCAmelCase  :      List[Any]    =						self.num_labels
     __lowerCAmelCase  :      Dict    =						MobileBertForTokenClassification(config=lowerCAmelCase   )
     model.to(lowerCAmelCase   )
     model.eval()
     __lowerCAmelCase  :      Tuple    =						model(lowerCAmelCase	,							attention_mask=lowerCAmelCase	,							token_type_ids=lowerCAmelCase	,							labels=lowerCAmelCase   )
     self.parent.assertEqual(result.logits.shape	,							(self.batch_size, self.seq_length, self.num_labels)   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   List[str]	,							lowerCAmelCase			:   List[Any]	,							lowerCAmelCase			:   Dict	,							lowerCAmelCase			:   Union[str, Any]	,							lowerCAmelCase			:   Optional[Any]	,							lowerCAmelCase			:   Union[str, Any]	,							lowerCAmelCase			:   Any	,							lowerCAmelCase			:   List[Any]   )							->     Optional[Any]:






     """simple docstring"""







     __lowerCAmelCase  :      Any    =						self.num_choices
     __lowerCAmelCase  :      List[Any]    =						MobileBertForMultipleChoice(config=lowerCAmelCase   )
     model.to(lowerCAmelCase   )
     model.eval()
     __lowerCAmelCase  :      Dict    =						input_ids.unsqueeze(1   ).expand(-1	,							self.num_choices	,							-1   ).contiguous()
     __lowerCAmelCase  :      Union[str, Any]    =						token_type_ids.unsqueeze(1   ).expand(-1	,							self.num_choices	,							-1   ).contiguous()
     __lowerCAmelCase  :      Optional[int]    =						input_mask.unsqueeze(1   ).expand(-1	,							self.num_choices	,							-1   ).contiguous()
     __lowerCAmelCase  :      Optional[int]    =						model(
         lowerCAmelCase	,							attention_mask=lowerCAmelCase	,							token_type_ids=lowerCAmelCase	,							labels=lowerCAmelCase	,							)
     self.parent.assertEqual(result.logits.shape	,							(self.batch_size, self.num_choices)   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   str   )							->     List[str]:






     """simple docstring"""







     __lowerCAmelCase  :      Tuple    =						self.prepare_config_and_inputs()
     (
         (
         __lowerCAmelCase
     )   ,(
         __lowerCAmelCase
     )   ,(
         __lowerCAmelCase
     )   ,(
         __lowerCAmelCase
     )   ,(
         __lowerCAmelCase
     )   ,(
         __lowerCAmelCase
     )   ,(
         __lowerCAmelCase
     )   ,
     )  :      List[Any]    =						config_and_inputs
     __lowerCAmelCase  :      List[str]    =						{"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
     return config, inputs_dict






@require_torch
class SCREAMING_SNAKE_CASE     (  a_       ,							a_       ,							unittest.TestCase			):
  """simple docstring"""






  lowerCamelCase   :      str										=(
      (
          MobileBertModel,
          MobileBertForMaskedLM,
          MobileBertForMultipleChoice,
          MobileBertForNextSentencePrediction,
          MobileBertForPreTraining,
          MobileBertForQuestionAnswering,
          MobileBertForSequenceClassification,
          MobileBertForTokenClassification,
      )
      if is_torch_available()
      else ()
  )
  lowerCamelCase   :      Optional[int]										=(
      {
          "feature-extraction": MobileBertModel,
          "fill-mask": MobileBertForMaskedLM,
          "question-answering": MobileBertForQuestionAnswering,
          "text-classification": MobileBertForSequenceClassification,
          "token-classification": MobileBertForTokenClassification,
          "zero-shot": MobileBertForSequenceClassification,
      }
      if is_torch_available()
      else {}
  )
  lowerCamelCase   :      Union[str, Any]										=True







  def 	SCREAMING_SNAKE_CASE   (      self			:   List[Any]	,							lowerCAmelCase			:   str	,							lowerCAmelCase			:   List[str]	,							lowerCAmelCase			:   List[Any]=False   )							->     List[str]:






     """simple docstring"""







     __lowerCAmelCase  :      Union[str, Any]    =						super()._prepare_for_class(lowerCAmelCase	,							lowerCAmelCase	,							return_labels=lowerCAmelCase   )

     if return_labels:
        if model_class in get_values(lowerCAmelCase   ):
           __lowerCAmelCase  :      Tuple    =						torch.zeros(
               (self.model_tester.batch_size, self.model_tester.seq_length)	,							dtype=torch.long	,							device=lowerCAmelCase   )
           __lowerCAmelCase  :      Union[str, Any]    =						torch.zeros(
               self.model_tester.batch_size	,							dtype=torch.long	,							device=lowerCAmelCase   )
     return inputs_dict







  def 	SCREAMING_SNAKE_CASE   (      self			:   int   )							->     Optional[Any]:






     """simple docstring"""







     __lowerCAmelCase  :      int    =						MobileBertModelTester(self   )
     __lowerCAmelCase  :      Optional[Any]    =						ConfigTester(self	,							config_class=lowerCAmelCase	,							hidden_size=37   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   str   )							->     Optional[Any]:






     """simple docstring"""







     self.config_tester.run_common_tests()







  def 	SCREAMING_SNAKE_CASE   (      self			:   int   )							->     Optional[Any]:






     """simple docstring"""







     __lowerCAmelCase  :      List[str]    =						self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_mobilebert_model(*lowerCAmelCase   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Dict   )							->     Tuple:






     """simple docstring"""







     __lowerCAmelCase  :      str    =						self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_mobilebert_for_masked_lm(*lowerCAmelCase   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Optional[int]   )							->     Dict:






     """simple docstring"""







     __lowerCAmelCase  :      str    =						self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_mobilebert_for_multiple_choice(*lowerCAmelCase   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Union[str, Any]   )							->     Dict:






     """simple docstring"""







     __lowerCAmelCase  :      Dict    =						self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_mobilebert_for_next_sequence_prediction(*lowerCAmelCase   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Dict   )							->     Optional[int]:






     """simple docstring"""







     __lowerCAmelCase  :      Tuple    =						self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_mobilebert_for_pretraining(*lowerCAmelCase   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Union[str, Any]   )							->     str:






     """simple docstring"""







     __lowerCAmelCase  :      Dict    =						self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_mobilebert_for_question_answering(*lowerCAmelCase   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   List[str]   )							->     Tuple:






     """simple docstring"""







     __lowerCAmelCase  :      Dict    =						self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_mobilebert_for_sequence_classification(*lowerCAmelCase   )







  def 	SCREAMING_SNAKE_CASE   (      self			:   Optional[int]   )							->     Tuple:






     """simple docstring"""







     __lowerCAmelCase  :      List[str]    =						self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_mobilebert_for_token_classification(*lowerCAmelCase   )






def      snake_case_				(__A					:						Any					)      ->     Optional[Any]:
   return torch.tensor(
       __A ,       dtype=torch.long ,       device=__A ,       )


__UpperCAmelCase								=			1e-3



@require_torch
@require_sentencepiece
@require_tokenizers
class SCREAMING_SNAKE_CASE     (  unittest.TestCase			):
  """simple docstring"""






  @slow
  def 	SCREAMING_SNAKE_CASE   (      self			:   Dict   )							->     Union[str, Any]:






     """simple docstring"""







     __lowerCAmelCase  :      Optional[Any]    =						MobileBertModel.from_pretrained("""google/mobilebert-uncased"""   ).to(lowerCAmelCase   )
     __lowerCAmelCase  :      int    =						_long_tensor([[1_01, 71_10, 10_05, 10_56, 20_23, 1_13_33, 1_74_13, 10_29, 1_02]]   )
     with torch.no_grad():
        __lowerCAmelCase  :      List[str]    =						model(lowerCAmelCase   )[0]
     __lowerCAmelCase  :      List[Any]    =						torch.Size((1, 9, 5_12)   )
     self.assertEqual(output.shape	,							lowerCAmelCase   )
     __lowerCAmelCase  :      int    =						torch.tensor(
         [
             [
                 [-2.473_6526e07, 8.269_1656e04, 1.652_1838e05],
                 [-5.754_1704e-01, 3.905_6022e00, 4.401_1507e00],
                 [2.604_7359e00, 1.567_7652e00, -1.732_4188e-01],
             ]
         ]	,							device=lowerCAmelCase	,							)

     # MobileBERT results range from 10e0 to 10e8. Even a 0.0000001% difference with a value of 10e8 results in a
     # ~1 difference, it's therefore not a good idea to measure using addition.
     # Here, we instead divide the expected result with the result in order to obtain ~1. We then check that the
     # result is held between bounds: 1 - TOLERANCE < expected_result / result < 1 + TOLERANCE
     __lowerCAmelCase  :      Union[str, Any]    =						torch.all((expected_slice / output[..., :3, :3]) >= 1 - TOLERANCE   )
     __lowerCAmelCase  :      Union[str, Any]    =						torch.all((expected_slice / output[..., :3, :3]) <= 1 + TOLERANCE   )

     self.assertTrue(lower_bound and upper_bound   )


