"""simple docstring"""




import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
     from transformers import LlamaTokenizerFast
except ImportError as e:
     warnings.warn(e)
     warnings.warn(
         'The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion'
     )
     _a				:		int	   =       None
_a				:		Optional[int]	   =       {
    '7B': 11_008,
    '13B': 13_824,
    '30B': 17_920,
    '65B': 22_016,
    '70B': 28_672,
}
_a				:		Tuple	   =       {
    '7B': 1,
    '7Bf': 1,
    '13B': 2,
    '13Bf': 2,
    '30B': 4,
    '65B': 8,
    '70B': 8,
    '70Bf': 8,
}





def 							SCREAMING_SNAKE_CASE					(      _lowerCamelCase						:	List[Any]      ,_lowerCamelCase						:	str=1      ,_lowerCamelCase						:	Optional[Any]=256     )	->				List[str]:
  return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3     )     ) + multiple_of - 1) // multiple_of)





def 							SCREAMING_SNAKE_CASE					(      _lowerCamelCase						:	Union[str, Any]     )	->				Tuple:
  with open(lowerCAmelCase_      ,"""r"""     ) as f:
    return json.load(lowerCAmelCase_     )





def 							SCREAMING_SNAKE_CASE					(      _lowerCamelCase						:	Tuple      ,_lowerCamelCase						:	Tuple     )	->				List[str]:
  with open(lowerCAmelCase_      ,"""w"""     ) as f:
    json.dump(lowerCAmelCase_      ,lowerCAmelCase_     )





def 							SCREAMING_SNAKE_CASE					(      _lowerCamelCase						:	str      ,_lowerCamelCase						:	Optional[int]      ,_lowerCamelCase						:	Tuple      ,_lowerCamelCase						:	Optional[Any]=True     )	->				str:
  os.makedirs(lowerCAmelCase_      ,exist_ok=lowerCAmelCase_     )
  _lowerCAmelCase						:	Any        =					os.path.join(lowerCAmelCase_      ,"""tmp"""     )
  os.makedirs(lowerCAmelCase_      ,exist_ok=lowerCAmelCase_     )

  _lowerCAmelCase						:	List[str]        =					read_json(os.path.join(lowerCAmelCase_      ,"""params.json"""     )     )
  _lowerCAmelCase						:	str        =					NUM_SHARDS[model_size]
  _lowerCAmelCase						:	int        =					params['n_layers']
  _lowerCAmelCase						:	Union[str, Any]        =					params['n_heads']
  _lowerCAmelCase						:	Any        =					n_heads // num_shards
  _lowerCAmelCase						:	Dict        =					params['dim']
  _lowerCAmelCase						:	List[str]        =					dim // n_heads
  _lowerCAmelCase						:	Union[str, Any]        =					10000.0
  _lowerCAmelCase						:	str        =					1.0 / (base ** (torch.arange(0      ,lowerCAmelCase_      ,2     ).float() / dims_per_head))

  if "n_kv_heads" in params:
    _lowerCAmelCase						:	int        =					params['n_kv_heads']  # for GQA / MQA
    _lowerCAmelCase						:	List[str]        =					n_heads_per_shard // num_key_value_heads
    _lowerCAmelCase						:	List[Any]        =					dim // num_key_value_heads
  else:  # compatibility with other checkpoints
    _lowerCAmelCase						:	Optional[Any]        =					n_heads
    _lowerCAmelCase						:	Union[str, Any]        =					n_heads_per_shard
    _lowerCAmelCase						:	str        =					dim

  # permute for sliced rotary
  def permute(_lowerCamelCase						:	Any      ,_lowerCamelCase						:	List[Any]=n_heads      ,_lowerCamelCase						:	List[str]=dim      ,_lowerCamelCase						:	Dict=dim     ):
    return w.view(lowerCAmelCase_      ,dima // n_heads // 2      ,2      ,lowerCAmelCase_     ).transpose(1      ,2     ).reshape(lowerCAmelCase_      ,lowerCAmelCase_     )

  print(f"Fetching all parameters from the checkpoint at {input_base_path}."     )
  # Load weights
  if model_size == "7B":
    # Not sharded
    # (The sharded implementation would also work, but this is simpler.)
    _lowerCAmelCase						:	Dict        =					torch.load(os.path.join(lowerCAmelCase_      ,"""consolidated.00.pth"""     )      ,map_location="""cpu"""     )
  else:
    # Sharded
    _lowerCAmelCase						:	Optional[Any]        =					[
        torch.load(os.path.join(lowerCAmelCase_      ,f"consolidated.{i:02d}.pth"     )      ,map_location="""cpu"""     )
        for i in range(lowerCAmelCase_     )
    ]
  _lowerCAmelCase						:	Optional[int]        =					0
  _lowerCAmelCase						:	Any        =					{'weight_map': {}}
  for layer_i in range(lowerCAmelCase_     ):
    _lowerCAmelCase						:	Optional[Any]        =					f"pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin"
    if model_size == "7B":
      # Unsharded
      _lowerCAmelCase						:	List[str]        =					{
          f"model.layers.{layer_i}.self_attn.q_proj.weight": permute(
              loaded[f"layers.{layer_i}.attention.wq.weight"]     ),
          f"model.layers.{layer_i}.self_attn.k_proj.weight": permute(
              loaded[f"layers.{layer_i}.attention.wk.weight"]     ),
          f"model.layers.{layer_i}.self_attn.v_proj.weight": loaded[f"layers.{layer_i}.attention.wv.weight"],
          f"model.layers.{layer_i}.self_attn.o_proj.weight": loaded[f"layers.{layer_i}.attention.wo.weight"],
          f"model.layers.{layer_i}.mlp.gate_proj.weight": loaded[f"layers.{layer_i}.feed_forward.w1.weight"],
          f"model.layers.{layer_i}.mlp.down_proj.weight": loaded[f"layers.{layer_i}.feed_forward.w2.weight"],
          f"model.layers.{layer_i}.mlp.up_proj.weight": loaded[f"layers.{layer_i}.feed_forward.w3.weight"],
          f"model.layers.{layer_i}.input_layernorm.weight": loaded[f"layers.{layer_i}.attention_norm.weight"],
          f"model.layers.{layer_i}.post_attention_layernorm.weight": loaded[f"layers.{layer_i}.ffn_norm.weight"],
      }
    else:
      # Sharded
      # Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
      # the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
      # redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

      _lowerCAmelCase						:	Union[str, Any]        =					{
          f"model.layers.{layer_i}.input_layernorm.weight": loaded[0][
              f"layers.{layer_i}.attention_norm.weight"
          ].clone(),
          f"model.layers.{layer_i}.post_attention_layernorm.weight": loaded[0][
              f"layers.{layer_i}.ffn_norm.weight"
          ].clone(),
      }
      _lowerCAmelCase						:	Dict        =					permute(
          torch.cat(
              [
                  loaded[i][f"layers.{layer_i}.attention.wq.weight"].view(lowerCAmelCase_      ,lowerCAmelCase_      ,lowerCAmelCase_     )
                  for i in range(lowerCAmelCase_     )
              ]      ,dim=0      ,).reshape(lowerCAmelCase_      ,lowerCAmelCase_     )     )
      _lowerCAmelCase						:	Tuple        =					permute(
          torch.cat(
              [
                  loaded[i][f"layers.{layer_i}.attention.wk.weight"].view(
                      lowerCAmelCase_      ,lowerCAmelCase_      ,lowerCAmelCase_     )
                  for i in range(lowerCAmelCase_     )
              ]      ,dim=0      ,).reshape(lowerCAmelCase_      ,lowerCAmelCase_     )      ,lowerCAmelCase_      ,lowerCAmelCase_      ,lowerCAmelCase_      ,)
      _lowerCAmelCase						:	Tuple        =					torch.cat(
          [
              loaded[i][f"layers.{layer_i}.attention.wv.weight"].view(
                  lowerCAmelCase_      ,lowerCAmelCase_      ,lowerCAmelCase_     )
              for i in range(lowerCAmelCase_     )
          ]      ,dim=0      ,).reshape(lowerCAmelCase_      ,lowerCAmelCase_     )

      _lowerCAmelCase						:	Union[str, Any]        =					torch.cat(
          [loaded[i][f"layers.{layer_i}.attention.wo.weight"] for i in range(lowerCAmelCase_     )]      ,dim=1     )
      _lowerCAmelCase						:	Tuple        =					torch.cat(
          [loaded[i][f"layers.{layer_i}.feed_forward.w1.weight"] for i in range(lowerCAmelCase_     )]      ,dim=0     )
      _lowerCAmelCase						:	List[Any]        =					torch.cat(
          [loaded[i][f"layers.{layer_i}.feed_forward.w2.weight"] for i in range(lowerCAmelCase_     )]      ,dim=1     )
      _lowerCAmelCase						:	Tuple        =					torch.cat(
          [loaded[i][f"layers.{layer_i}.feed_forward.w3.weight"] for i in range(lowerCAmelCase_     )]      ,dim=0     )

    _lowerCAmelCase						:	str        =					inv_freq
    for k, v in state_dict.items():
      _lowerCAmelCase						:	Tuple        =					filename
      param_count += v.numel()
    torch.save(lowerCAmelCase_      ,os.path.join(lowerCAmelCase_      ,lowerCAmelCase_     )     )

  _lowerCAmelCase						:	Optional[Any]        =					f"pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin"
  if model_size == "7B":
    # Unsharded
    _lowerCAmelCase						:	List[Any]        =					{
        'model.embed_tokens.weight': loaded['tok_embeddings.weight'],
        'model.norm.weight': loaded['norm.weight'],
        'lm_head.weight': loaded['output.weight'],
    }
  else:
    _lowerCAmelCase						:	str        =					{
        'model.norm.weight': loaded[0]['norm.weight'],
        'model.embed_tokens.weight': torch.cat(
            [loaded[i]["""tok_embeddings.weight"""] for i in range(lowerCAmelCase_     )]      ,dim=1     ),
        'lm_head.weight': torch.cat([loaded[i]["""output.weight"""] for i in range(lowerCAmelCase_     )]      ,dim=0     ),
    }

  for k, v in state_dict.items():
    _lowerCAmelCase						:	Union[str, Any]        =					filename
    param_count += v.numel()
  torch.save(lowerCAmelCase_      ,os.path.join(lowerCAmelCase_      ,lowerCAmelCase_     )     )

  # Write configs
  _lowerCAmelCase						:	Dict        =					{'total_size': param_count * 2}
  write_json(lowerCAmelCase_      ,os.path.join(lowerCAmelCase_      ,"""pytorch_model.bin.index.json"""     )     )
  _lowerCAmelCase						:	List[Any]        =					params['ffn_dim_multiplier'] if 'ffn_dim_multiplier' in params else 1
  _lowerCAmelCase						:	Optional[Any]        =					params['multiple_of'] if 'multiple_of' in params else 256
  _lowerCAmelCase						:	Optional[int]        =					LlamaConfig(
      hidden_size=lowerCAmelCase_      ,intermediate_size=compute_intermediate_size(lowerCAmelCase_      ,lowerCAmelCase_      ,lowerCAmelCase_     )      ,num_attention_heads=params["""n_heads"""]      ,num_hidden_layers=params["""n_layers"""]      ,rms_norm_eps=params["""norm_eps"""]      ,num_key_value_heads=lowerCAmelCase_      ,)
  config.save_pretrained(lowerCAmelCase_     )

  # Make space so we can load the model properly now.
  del state_dict
  del loaded
  gc.collect()

  print("""Loading the checkpoint in a Llama model."""     )
  _lowerCAmelCase						:	List[str]        =					LlamaForCausalLM.from_pretrained(lowerCAmelCase_      ,torch_dtype=torch.floataa      ,low_cpu_mem_usage=lowerCAmelCase_     )
  # Avoid saving this as part of the config.
  del model.config._name_or_path

  print("""Saving in the Transformers format."""     )
  model.save_pretrained(lowerCAmelCase_      ,safe_serialization=lowerCAmelCase_     )
  shutil.rmtree(lowerCAmelCase_     )





def 							SCREAMING_SNAKE_CASE					(      _lowerCamelCase						:	Optional[int]      ,_lowerCamelCase						:	List[Any]     )	->				str:
  # Initialize the tokenizer based on the `spm` model
  _lowerCAmelCase						:	Dict        =					LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
  print(f"Saving a {tokenizer_class.__name__} to {tokenizer_path}."     )
  _lowerCAmelCase						:	Dict        =					tokenizer_class(lowerCAmelCase_     )
  tokenizer.save_pretrained(lowerCAmelCase_     )





def 							SCREAMING_SNAKE_CASE					(      )	->				Optional[int]:
  _lowerCAmelCase						:	str        =					argparse.ArgumentParser()
  parser.add_argument(
      """--input_dir"""      ,help="""Location of LLaMA weights, which contains tokenizer.model and model folders"""      ,)
  parser.add_argument(
      """--model_size"""      ,choices=["""7B""", """7Bf""", """13B""", """13Bf""", """30B""", """65B""", """70B""", """70Bf""", """tokenizer_only"""]      ,)
  parser.add_argument(
      """--output_dir"""      ,help="""Location to write HF model and tokenizer"""      ,)
  parser.add_argument("""--safe_serialization"""      ,type=lowerCAmelCase_      ,help="""Whether or not to save using `safetensors`."""     )
  _lowerCAmelCase						:	Tuple        =					parser.parse_args()
  if args.model_size != "tokenizer_only":
    write_model(
        model_path=args.output_dir      ,input_base_path=os.path.join(args.input_dir      ,args.model_size     )      ,model_size=args.model_size      ,safe_serialization=args.safe_serialization      ,)
  _lowerCAmelCase						:	List[Any]        =					os.path.join(args.input_dir      ,"""tokenizer.model"""     )
  write_tokenizer(args.output_dir      ,lowerCAmelCase_     )


if __name__ == "__main__":
     main()





"""simple docstring"""




import copy
import unittest

from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
     import torch

     from transformers import (
         MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
         MODEL_FOR_QUESTION_ANSWERING_MAPPING,
         MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
         MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
         LayoutLMvaConfig,
         LayoutLMvaForQuestionAnswering,
         LayoutLMvaForSequenceClassification,
         LayoutLMvaForTokenClassification,
         LayoutLMvaModel,
     )
     from transformers.models.layoutlmva.modeling_layoutlmva import LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST

if is_vision_available():
     from PIL import Image

     from transformers import LayoutLMvaImageProcessor
class 						__A							:






     def __init__(		self    ,					a__    ,					a__=2    ,					a__=3    ,					a__=4    ,					a__=2    ,					a__=7    ,					a__=True    ,					a__=True    ,					a__=True    ,					a__=True    ,					a__=99    ,					a__=36    ,					a__=3    ,					a__=4    ,					a__=37    ,					a__="gelu"    ,					a__=0.1    ,					a__=0.1    ,					a__=512    ,					a__=16    ,					a__=2    ,					a__=0.0_2    ,					a__=6    ,					a__=6    ,					a__=3    ,					a__=4    ,					a__=None    ,					a__=1000    ,					):
       _lowerCAmelCase						:	Union[str, Any]        =					parent
       _lowerCAmelCase						:	Optional[Any]        =					batch_size
       _lowerCAmelCase						:	List[Any]        =					num_channels
       _lowerCAmelCase						:	str        =					image_size
       _lowerCAmelCase						:	str        =					patch_size
       _lowerCAmelCase						:	str        =					text_seq_length
       _lowerCAmelCase						:	List[Any]        =					is_training
       _lowerCAmelCase						:	List[str]        =					use_input_mask
       _lowerCAmelCase						:	List[Any]        =					use_token_type_ids
       _lowerCAmelCase						:	Optional[Any]        =					use_labels
       _lowerCAmelCase						:	Any        =					vocab_size
       _lowerCAmelCase						:	str        =					hidden_size
       _lowerCAmelCase						:	Dict        =					num_hidden_layers
       _lowerCAmelCase						:	List[Any]        =					num_attention_heads
       _lowerCAmelCase						:	str        =					intermediate_size
       _lowerCAmelCase						:	str        =					hidden_act
       _lowerCAmelCase						:	Optional[Any]        =					hidden_dropout_prob
       _lowerCAmelCase						:	Any        =					attention_probs_dropout_prob
       _lowerCAmelCase						:	Tuple        =					max_position_embeddings
       _lowerCAmelCase						:	Dict        =					type_vocab_size
       _lowerCAmelCase						:	Tuple        =					type_sequence_label_size
       _lowerCAmelCase						:	Union[str, Any]        =					initializer_range
       _lowerCAmelCase						:	Dict        =					coordinate_size
       _lowerCAmelCase						:	Optional[int]        =					shape_size
       _lowerCAmelCase						:	str        =					num_labels
       _lowerCAmelCase						:	Optional[Any]        =					num_choices
       _lowerCAmelCase						:	str        =					scope
       _lowerCAmelCase						:	Dict        =					range_bbox

       # LayoutLMv3's sequence length equals the number of text tokens + number of patches + 1 (we add 1 for the CLS token)
       _lowerCAmelCase						:	Optional[int]        =					text_seq_length
       _lowerCAmelCase						:	Any        =					(image_size // patch_size) ** 2 + 1
       _lowerCAmelCase						:	Any        =					self.text_seq_length + self.image_seq_length






     def    __A  (		self  ):
       _lowerCAmelCase						:	Tuple        =					ids_tensor([self.batch_size, self.text_seq_length]    ,					self.vocab_size  )

       _lowerCAmelCase						:	Tuple        =					ids_tensor([self.batch_size, self.text_seq_length, 4]    ,					self.range_bbox  )
       # Ensure that bbox is legal
       for i in range(bbox.shape[0]  ):
         for j in range(bbox.shape[1]  ):
           if bbox[i, j, 3] < bbox[i, j, 1]:
             _lowerCAmelCase						:	Optional[Any]        =					bbox[i, j, 3]
             _lowerCAmelCase						:	List[str]        =					bbox[i, j, 1]
             _lowerCAmelCase						:	List[Any]        =					t
           if bbox[i, j, 2] < bbox[i, j, 0]:
             _lowerCAmelCase						:	int        =					bbox[i, j, 2]
             _lowerCAmelCase						:	Optional[int]        =					bbox[i, j, 0]
             _lowerCAmelCase						:	Optional[int]        =					t

       _lowerCAmelCase						:	Dict        =					floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]  )

       _lowerCAmelCase						:	List[str]        =					None
       if self.use_input_mask:
         _lowerCAmelCase						:	int        =					random_attention_mask([self.batch_size, self.text_seq_length]  )

       _lowerCAmelCase						:	str        =					None
       if self.use_token_type_ids:
         _lowerCAmelCase						:	Tuple        =					ids_tensor([self.batch_size, self.text_seq_length]    ,					self.type_vocab_size  )

       _lowerCAmelCase						:	int        =					None
       _lowerCAmelCase						:	int        =					None
       if self.use_labels:
         _lowerCAmelCase						:	List[str]        =					ids_tensor([self.batch_size]    ,					self.type_sequence_label_size  )
         _lowerCAmelCase						:	str        =					ids_tensor([self.batch_size, self.text_seq_length]    ,					self.num_labels  )

       _lowerCAmelCase						:	str        =					LayoutLMvaConfig(
           vocab_size=self.vocab_size    ,					hidden_size=self.hidden_size    ,					num_hidden_layers=self.num_hidden_layers    ,					num_attention_heads=self.num_attention_heads    ,					intermediate_size=self.intermediate_size    ,					hidden_act=self.hidden_act    ,					hidden_dropout_prob=self.hidden_dropout_prob    ,					attention_probs_dropout_prob=self.attention_probs_dropout_prob    ,					max_position_embeddings=self.max_position_embeddings    ,					type_vocab_size=self.type_vocab_size    ,					initializer_range=self.initializer_range    ,					coordinate_size=self.coordinate_size    ,					shape_size=self.shape_size    ,					input_size=self.image_size    ,					patch_size=self.patch_size    ,					)

       return config, input_ids, bbox, pixel_values, token_type_ids, input_mask, sequence_labels, token_labels






     def    __A  (		self    ,					a__    ,					a__    ,					a__    ,					a__    ,					a__    ,					a__    ,					a__    ,					a__  ):
       _lowerCAmelCase						:	int        =					LayoutLMvaModel(config=a__  )
       model.to(a__  )
       model.eval()

       # text + image
       _lowerCAmelCase						:	Optional[Any]        =					model(a__    ,					pixel_values=a__  )
       _lowerCAmelCase						:	Any        =					model(
           a__    ,					bbox=a__    ,					pixel_values=a__    ,					attention_mask=a__    ,					token_type_ids=a__  )
       _lowerCAmelCase						:	List[Any]        =					model(a__    ,					bbox=a__    ,					pixel_values=a__    ,					token_type_ids=a__  )
       _lowerCAmelCase						:	Tuple        =					model(a__    ,					bbox=a__    ,					pixel_values=a__  )

       self.parent.assertEqual(result.last_hidden_state.shape    ,					(self.batch_size, self.seq_length, self.hidden_size)  )

       # text only
       _lowerCAmelCase						:	Dict        =					model(a__  )
       self.parent.assertEqual(
           result.last_hidden_state.shape    ,					(self.batch_size, self.text_seq_length, self.hidden_size)  )

       # image only
       _lowerCAmelCase						:	Optional[Any]        =					model(pixel_values=a__  )
       self.parent.assertEqual(
           result.last_hidden_state.shape    ,					(self.batch_size, self.image_seq_length, self.hidden_size)  )






     def    __A  (		self    ,					a__    ,					a__    ,					a__    ,					a__    ,					a__    ,					a__    ,					a__    ,					a__  ):
       _lowerCAmelCase						:	int        =					self.num_labels
       _lowerCAmelCase						:	int        =					LayoutLMvaForSequenceClassification(a__  )
       model.to(a__  )
       model.eval()
       _lowerCAmelCase						:	Tuple        =					model(
           a__    ,					bbox=a__    ,					pixel_values=a__    ,					attention_mask=a__    ,					token_type_ids=a__    ,					labels=a__    ,					)
       self.parent.assertEqual(result.logits.shape    ,					(self.batch_size, self.num_labels)  )






     def    __A  (		self    ,					a__    ,					a__    ,					a__    ,					a__    ,					a__    ,					a__    ,					a__    ,					a__  ):
       _lowerCAmelCase						:	Optional[Any]        =					self.num_labels
       _lowerCAmelCase						:	str        =					LayoutLMvaForTokenClassification(config=a__  )
       model.to(a__  )
       model.eval()
       _lowerCAmelCase						:	List[str]        =					model(
           a__    ,					bbox=a__    ,					pixel_values=a__    ,					attention_mask=a__    ,					token_type_ids=a__    ,					labels=a__    ,					)
       self.parent.assertEqual(result.logits.shape    ,					(self.batch_size, self.text_seq_length, self.num_labels)  )






     def    __A  (		self    ,					a__    ,					a__    ,					a__    ,					a__    ,					a__    ,					a__    ,					a__    ,					a__  ):
       _lowerCAmelCase						:	Dict        =					LayoutLMvaForQuestionAnswering(config=a__  )
       model.to(a__  )
       model.eval()
       _lowerCAmelCase						:	str        =					model(
           a__    ,					bbox=a__    ,					pixel_values=a__    ,					attention_mask=a__    ,					token_type_ids=a__    ,					start_positions=a__    ,					end_positions=a__    ,					)
       self.parent.assertEqual(result.start_logits.shape    ,					(self.batch_size, self.seq_length)  )
       self.parent.assertEqual(result.end_logits.shape    ,					(self.batch_size, self.seq_length)  )






     def    __A  (		self  ):
       _lowerCAmelCase						:	List[str]        =					self.prepare_config_and_inputs()
       (
           (
           _lowerCAmelCase
       )      ,     (
           _lowerCAmelCase
       )      ,     (
           _lowerCAmelCase
       )      ,     (
           _lowerCAmelCase
       )      ,     (
           _lowerCAmelCase
       )      ,     (
           _lowerCAmelCase
       )      ,     (
           _lowerCAmelCase
       )      ,     (
           _lowerCAmelCase
       )      ,     
       )						:	Any        =					config_and_inputs
       _lowerCAmelCase						:	Union[str, Any]        =					{
           """input_ids""": input_ids,
           """bbox""": bbox,
           """pixel_values""": pixel_values,
           """token_type_ids""": token_type_ids,
           """attention_mask""": input_mask,
       }
       return config, inputs_dict







@require_torch
class 						__A							(			SCREAMING_SNAKE_CASE_   ,     SCREAMING_SNAKE_CASE_   ,     unittest.TestCase		):
     _UpperCamelCase							:      Tuple		 =		False
     _UpperCamelCase							:      Any		 =		False
     _UpperCamelCase							:      Any		 =		False

     _UpperCamelCase							:      int		 =		(
         (
             LayoutLMvaModel,
             LayoutLMvaForSequenceClassification,
             LayoutLMvaForTokenClassification,
             LayoutLMvaForQuestionAnswering,
         )
         if is_torch_available()
         else ()
     )
     _UpperCamelCase							:      Union[str, Any]		 =		(
         {"document-question-answering": LayoutLMvaForQuestionAnswering, "feature-extraction": LayoutLMvaModel}
         if is_torch_available()
         else {}
     )






     def    __A  (		self    ,					a__    ,					a__    ,					a__    ,					a__    ,					a__  ):
       # `DocumentQuestionAnsweringPipeline` is expected to work with this model, but it combines the text and visual
       # embedding along the sequence dimension (dim 1), which causes an error during post-processing as `p_mask` has
       # the sequence dimension of the text embedding only.
       # (see the line `embedding_output = torch.cat([embedding_output, visual_embeddings], dim=1)`)
       return True






     def    __A  (		self  ):
       _lowerCAmelCase						:	Any        =					LayoutLMvaModelTester(self  )
       _lowerCAmelCase						:	int        =					ConfigTester(self    ,					config_class=a__    ,					hidden_size=37  )






     def    __A  (		self    ,					a__    ,					a__    ,					a__=False  ):
       _lowerCAmelCase						:	List[str]        =					copy.deepcopy(a__  )
       if model_class in get_values(a__  ):
         _lowerCAmelCase						:	Optional[int]        =					{
             k: v.unsqueeze(1  ).expand(-1    ,					self.model_tester.num_choices    ,					-1  ).contiguous()
             if isinstance(a__    ,					torch.Tensor  ) and v.ndim > 1
             else v
             for k, v in inputs_dict.items()
         }
       if return_labels:
         if model_class in get_values(a__  ):
           _lowerCAmelCase						:	List[Any]        =					torch.ones(self.model_tester.batch_size    ,					dtype=torch.long    ,					device=a__  )
         elif model_class in get_values(a__  ):
           _lowerCAmelCase						:	Optional[int]        =					torch.zeros(
               self.model_tester.batch_size    ,					dtype=torch.long    ,					device=a__  )
           _lowerCAmelCase						:	Any        =					torch.zeros(
               self.model_tester.batch_size    ,					dtype=torch.long    ,					device=a__  )
         elif model_class in [
             *get_values(a__  ),
         ]:
           _lowerCAmelCase						:	Optional[int]        =					torch.zeros(
               self.model_tester.batch_size    ,					dtype=torch.long    ,					device=a__  )
         elif model_class in [
             *get_values(a__  ),
         ]:
           _lowerCAmelCase						:	int        =					torch.zeros(
               (self.model_tester.batch_size, self.model_tester.text_seq_length)    ,					dtype=torch.long    ,					device=a__    ,					)

       return inputs_dict






     def    __A  (		self  ):
       self.config_tester.run_common_tests()






     def    __A  (		self  ):
       _lowerCAmelCase						:	Optional[int]        =					self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_model(*a__  )






     def    __A  (		self  ):
       _lowerCAmelCase						:	Optional[Any]        =					self.model_tester.prepare_config_and_inputs()
       for type in ["absolute", "relative_key", "relative_key_query"]:
         _lowerCAmelCase						:	int        =					type
         self.model_tester.create_and_check_model(*a__  )






     def    __A  (		self  ):
       _lowerCAmelCase						:	Optional[Any]        =					self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_sequence_classification(*a__  )






     def    __A  (		self  ):
       _lowerCAmelCase						:	Tuple        =					self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_token_classification(*a__  )






     def    __A  (		self  ):
       _lowerCAmelCase						:	Optional[int]        =					self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_question_answering(*a__  )






     @slow
     def    __A  (		self  ):
       for model_name in LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
         _lowerCAmelCase						:	str        =					LayoutLMvaModel.from_pretrained(a__  )
         self.assertIsNotNone(a__  )







def 							SCREAMING_SNAKE_CASE					(      )	->				str:
  _lowerCAmelCase						:	Any        =					Image.open("""./tests/fixtures/tests_samples/COCO/000000039769.png"""     )
  return image






@require_torch
class 						__A							(			unittest.TestCase		):






     @cached_property
     def    __A  (		self  ):
       return LayoutLMvaImageProcessor(apply_ocr=a__  ) if is_vision_available() else None






     @slow
     def    __A  (		self  ):
       _lowerCAmelCase						:	Optional[int]        =					LayoutLMvaModel.from_pretrained("""microsoft/layoutlmv3-base"""  ).to(a__  )

       _lowerCAmelCase						:	Dict        =					self.default_image_processor
       _lowerCAmelCase						:	Optional[int]        =					prepare_img()
       _lowerCAmelCase						:	Dict        =					image_processor(images=a__    ,					return_tensors="""pt"""  ).pixel_values.to(a__  )

       _lowerCAmelCase						:	Optional[Any]        =					torch.tensor([[1, 2]]  )
       _lowerCAmelCase						:	Dict        =					torch.tensor([[1, 2, 3, 4], [5, 6, 7, 8]]  ).unsqueeze(0  )

       # forward pass
       _lowerCAmelCase						:	str        =					model(
           input_ids=input_ids.to(a__  )    ,					bbox=bbox.to(a__  )    ,					pixel_values=pixel_values.to(a__  )    ,					)

       # verify the logits
       _lowerCAmelCase						:	Optional[int]        =					torch.Size((1, 199, 768)  )
       self.assertEqual(outputs.last_hidden_state.shape    ,					a__  )

       _lowerCAmelCase						:	Union[str, Any]        =					torch.tensor(
           [[-0.0_5_2_9, 0.3_6_1_8, 0.1_6_3_2], [-0.1_5_8_7, -0.1_6_6_7, -0.0_4_0_0], [-0.1_5_5_7, -0.1_6_7_1, -0.0_5_0_5]]  ).to(a__  )

       self.assertTrue(torch.allclose(outputs.last_hidden_state[0, :3, :3]    ,					a__    ,					atol=1e-4  )  )




