

"""simple docstring"""





import argparse

import requests
import torch

# pip3 install salesforce-lavis
# I'm actually installing a slightly modified version: pip3 install git+https://github.com/nielsrogge/LAVIS.git@fix_lavis_float32 (there's also the fix_lavis branch)
# also note: to convert Vicuna checkpoints, we had to include /home/niels/python_projects/checkpoints/FastChat/vicuna-7b in lavis/configs/models/blip2/blip2_instruct_vicuna7b.yaml
# same for Vicuna-13b
from lavis.models import load_model_and_preprocess
from PIL import Image

from transformers import (
    AutoTokenizer,
    BlipImageProcessor,
    InstructBlipConfig,
    InstructBlipForConditionalGeneration,
    InstructBlipProcessor,
    InstructBlipQFormerConfig,
    InstructBlipVisionConfig,
    LlamaConfig,
    LlamaTokenizerFast,
    TaConfig,
    TaTokenizerFast,
)
from transformers.utils.constants import OPENAI_CLIP_MEAN, OPENAI_CLIP_STD




def   _A  (  )     ->						Optional[Any]:
   '''simple docstring'''


   __lowercase								=						"https://raw.githubusercontent.com/salesforce/LAVIS/main/docs/_static/Confusing-Pictures.jpg"
   __lowercase								=						Image.open(requests.get(UpperCamelCase_,      stream=UpperCamelCase_).raw).convert("RGB")

   return image




def   _A  (  UpperCamelCase_					:	List[Any])     ->						Optional[Any]:
   '''simple docstring'''


   __lowercase								=						[]
   # fmt: off

   # vision encoder
   rename_keys.append(("visual_encoder.cls_token", "vision_model.embeddings.class_embedding"))
   rename_keys.append(("visual_encoder.pos_embed", "vision_model.embeddings.position_embedding"))
   rename_keys.append(("visual_encoder.patch_embed.proj.weight", "vision_model.embeddings.patch_embedding.weight"))
   rename_keys.append(("visual_encoder.patch_embed.proj.bias", "vision_model.embeddings.patch_embedding.bias"))
   rename_keys.append(("ln_vision.weight", "vision_model.post_layernorm.weight"))
   rename_keys.append(("ln_vision.bias", "vision_model.post_layernorm.bias"))

   for i in range(config.vision_config.num_hidden_layers):
      rename_keys.append((F"""visual_encoder.blocks.{i}.norm1.weight""", F"""vision_model.encoder.layers.{i}.layer_norm1.weight"""))
      rename_keys.append((F"""visual_encoder.blocks.{i}.norm1.bias""", F"""vision_model.encoder.layers.{i}.layer_norm1.bias"""))
      rename_keys.append((F"""visual_encoder.blocks.{i}.norm2.weight""", F"""vision_model.encoder.layers.{i}.layer_norm2.weight"""))
      rename_keys.append((F"""visual_encoder.blocks.{i}.norm2.bias""", F"""vision_model.encoder.layers.{i}.layer_norm2.bias"""))
      rename_keys.append((F"""visual_encoder.blocks.{i}.attn.qkv.weight""", F"""vision_model.encoder.layers.{i}.self_attn.qkv.weight"""))
      rename_keys.append((F"""visual_encoder.blocks.{i}.attn.proj.weight""", F"""vision_model.encoder.layers.{i}.self_attn.projection.weight""",))
      rename_keys.append((F"""visual_encoder.blocks.{i}.attn.proj.bias""", F"""vision_model.encoder.layers.{i}.self_attn.projection.bias"""))
      rename_keys.append((F"""visual_encoder.blocks.{i}.mlp.fc1.weight""", F"""vision_model.encoder.layers.{i}.mlp.fc1.weight"""))
      rename_keys.append((F"""visual_encoder.blocks.{i}.mlp.fc1.bias""", F"""vision_model.encoder.layers.{i}.mlp.fc1.bias"""))
      rename_keys.append((F"""visual_encoder.blocks.{i}.mlp.fc2.weight""", F"""vision_model.encoder.layers.{i}.mlp.fc2.weight"""))
      rename_keys.append((F"""visual_encoder.blocks.{i}.mlp.fc2.bias""", F"""vision_model.encoder.layers.{i}.mlp.fc2.bias"""))

   # QFormer
   rename_keys.append(("Qformer.bert.embeddings.LayerNorm.weight", "qformer.embeddings.layernorm.weight"))
   rename_keys.append(("Qformer.bert.embeddings.LayerNorm.bias", "qformer.embeddings.layernorm.bias"))

   # fmt: on
   return rename_keys




def   _A  (  UpperCamelCase_					:	Optional[Any],      UpperCamelCase_					:	Any,      UpperCamelCase_					:	List[Any])     ->						Any:
   '''simple docstring'''


   __lowercase								=						dct.pop(UpperCamelCase_)
   __lowercase								=						val




def   _A  (  UpperCamelCase_					:	Union[str, Any],      UpperCamelCase_					:	int)     ->						int:
   '''simple docstring'''


   for i in range(config.vision_config.num_hidden_layers):
      # read in original q and v biases
      __lowercase								=						state_dict.pop(F"""visual_encoder.blocks.{i}.attn.q_bias""")
      __lowercase								=						state_dict.pop(F"""visual_encoder.blocks.{i}.attn.v_bias""")

      # next, set bias in the state dict
      __lowercase								=						torch.cat((q_bias, torch.zeros_like(UpperCamelCase_,      requires_grad=UpperCamelCase_), v_bias))
      __lowercase								=						qkv_bias




def   _A  (  UpperCamelCase_					:	List[Any])     ->						List[str]:
   '''simple docstring'''


   __lowercase								=						364 if "coco" in model_name else 224
   __lowercase								=						InstructBlipVisionConfig(image_size=UpperCamelCase_).to_dict()

   # make sure the models have proper bos_token_id and eos_token_id set (important for generation)
   # seems like flan-T5 models don't have bos_token_id properly set?
   if "t5-xl" in model_name:
      __lowercase								=						TaConfig.from_pretrained("google/flan-t5-xl",      dense_act_fn="gelu",      bos_token_id=1).to_dict()
   elif "t5-xxl" in model_name:
      __lowercase								=						TaConfig.from_pretrained("google/flan-t5-xxl",      dense_act_fn="gelu",      bos_token_id=1).to_dict()
   elif "vicuna-7b" in model_name:
      __lowercase								=						LlamaConfig.from_pretrained("decapoda-research/llama-7b-hf",      vocab_size=32001).to_dict()
   elif "vicuna-13b" in model_name:
      __lowercase								=						LlamaConfig.from_pretrained("decapoda-research/llama-13b-hf",      vocab_size=32001).to_dict()
   else:
      raise ValueError("Model name not supported")

   # the authors add one special "[DEC]" token to the vocab of Q-Former, hence vocab size = 30522 + 1
   __lowercase								=						InstructBlipQFormerConfig(vocab_size=30523).to_dict()
   __lowercase								=						InstructBlipConfig(vision_config=UpperCamelCase_,      text_config=UpperCamelCase_,      qformer_config=UpperCamelCase_)

   return config, image_size




@torch.no_grad()
def   _A  (  UpperCamelCase_					:	List[str],      UpperCamelCase_					:	int=None,      UpperCamelCase_					:	Any=False)     ->						Tuple:
   '''simple docstring'''


   __lowercase								=						AutoTokenizer.from_pretrained("bert-base-uncased",      truncation_side="left")
   qformer_tokenizer.add_special_tokens({"bos_token": "[DEC]"})

   if "t5" in model_name:
      __lowercase								=						TaTokenizerFast.from_pretrained("google/flan-t5-xl",      truncation_side="left")
   elif "vicuna" in model_name:
      # the following was used in the original implementation:
      # tokenizer = LlamaTokenizer.from_pretrained("huggyllama/llama-7b", use_fast=False, truncation_side="left")
      # tokenizer.add_special_tokens({"pad_token": "[PAD]"})
      # tokenizer.add_special_tokens({"bos_token": "</s>"})
      # tokenizer.add_special_tokens({"eos_token": "</s>"})
      # tokenizer.add_special_tokens({"unk_token": "</s>"})
      __lowercase								=						LlamaTokenizerFast.from_pretrained(
          "huggyllama/llama-7b",      truncation_side="left",      bos_token="</s>",      unk_token="</s>")
      tokenizer.add_special_tokens({"pad_token": "[PAD]"})

   __lowercase		,__lowercase								=						get_blipa_config(UpperCamelCase_)
   __lowercase								=						InstructBlipForConditionalGeneration(UpperCamelCase_).eval()

   __lowercase								=						{
       "instructblip-vicuna-7b": ("blip2_vicuna_instruct", "vicuna7b"),
       "instructblip-vicuna-13b": ("blip2_vicuna_instruct", "vicuna13b"),
       "instructblip-flan-t5-xl": ("blip2_t5_instruct", "flant5xl"),
       "instructblip-flan-t5-xxl": ("blip2_t5_instruct", "flant5xxl"),
   }

   __lowercase		,__lowercase								=						model_name_to_original[model_name]

   # load original model
   print("Loading original model...")
   __lowercase								=						"cuda:1" if torch.cuda.is_available() else "cpu"
   __lowercase								=						"cuda:2" if torch.cuda.is_available() else "cpu"
   __lowercase		,__lowercase		,__lowercase								=						load_model_and_preprocess(
       name=UpperCamelCase_,      model_type=UpperCamelCase_,      is_eval=UpperCamelCase_,      device=UpperCamelCase_)
   original_model.eval()
   print("Done!")

   # update state dict keys
   __lowercase								=						original_model.state_dict()
   __lowercase								=						create_rename_keys(UpperCamelCase_)
   for src, dest in rename_keys:
      rename_key(UpperCamelCase_,      UpperCamelCase_,      UpperCamelCase_)

   # some keys can be renamed efficiently
   for key, val in state_dict.copy().items():
      __lowercase								=						state_dict.pop(UpperCamelCase_)
      if key.startswith("Qformer.bert"):
         __lowercase								=						key.replace("Qformer.bert",      "qformer")
      if "attention.self" in key:
         __lowercase								=						key.replace("self",      "attention")
      if "llm_proj" in key:
         __lowercase								=						key.replace("llm_proj",      "language_projection")
      if "t5_proj" in key:
         __lowercase								=						key.replace("t5_proj",      "language_projection")
      if key.startswith("llm_model"):
         __lowercase								=						key.replace("llm_model",      "language_model")
      if key.startswith("t5"):
         __lowercase								=						key.replace("t5",      "language")
      __lowercase								=						val

   # read in qv biases
   read_in_q_v_bias(UpperCamelCase_,      UpperCamelCase_)

   # note: weights get loaded in torch.float32 by default
   hf_model.load_state_dict(UpperCamelCase_,      strict=UpperCamelCase_)

   __lowercase								=						load_demo_image()
   __lowercase								=						"What is unusual about this image?"

   # create processor
   __lowercase								=						BlipImageProcessor(
       size={"height": image_size, "width": image_size},      image_mean=UpperCamelCase_,      image_std=UpperCamelCase_)
   __lowercase								=						InstructBlipProcessor(
       image_processor=UpperCamelCase_,      tokenizer=UpperCamelCase_,      qformer_tokenizer=UpperCamelCase_,      )
   __lowercase								=						processor(images=UpperCamelCase_,      text=UpperCamelCase_,      return_tensors="pt").to(UpperCamelCase_)

   # make sure processor creates exact same pixel values
   __lowercase								=						vis_processors["eval"](UpperCamelCase_).unsqueeze(0).to(UpperCamelCase_)
   __lowercase								=						inputs.pixel_values
   assert torch.allclose(original_pixel_values.to(pixel_values.device),      UpperCamelCase_)

   original_model.to(UpperCamelCase_)
   hf_model.to(UpperCamelCase_)
   with torch.no_grad():
      if "vicuna" in model_name:
         __lowercase								=						original_model({"image": original_pixel_values, "text_input": [prompt]}).logits
         __lowercase								=						hf_model(**UpperCamelCase_).logits
      else:
         __lowercase								=						original_model(
             {"image": original_pixel_values, "text_input": [prompt], "text_output": ["\n"]}).logits
         __lowercase								=						tokenizer("\n",      return_tensors="pt").input_ids.to(UpperCamelCase_)
         __lowercase								=						label_input_ids.masked_fill(label_input_ids == tokenizer.pad_token_id,      -100)
         __lowercase								=						hf_model(**UpperCamelCase_,      labels=UpperCamelCase_).logits

   print("First values of original logits:",      original_logits[0, :3, :3])
   print("First values of HF logits:",      logits[0, :3, :3])

   # assert values
   assert original_logits.shape == logits.shape
   __lowercase								=						1E-4 if "vicuna" in model_name else 1E-5
   assert torch.allclose(original_logits.to(logits.device),      UpperCamelCase_,      atol=UpperCamelCase_)
   print("Looks ok!")

   print("Generating with original model...")
   __lowercase								=						original_model.generate({"image": original_pixel_values, "prompt": prompt},      num_beams=5)

   # important: we need to cast the weights of the HF model to the appropriate type
   print("Generating with HF model...")
   __lowercase								=						hf_model.generate(
       **UpperCamelCase_,      do_sample=UpperCamelCase_,      num_beams=5,      max_length=256,      min_length=1,      top_p=0.9,      repetition_penalty=1.5,      length_penalty=1.0,      temperature=1,      )
   if "vicuna" in model_name:
      # convert output id 0 to 2 (eos_token_id)
      # TODO add this in the generate method?
      __lowercase								=						2
   print("Original generation:",      UpperCamelCase_)
   __lowercase								=						processor.batch_decode(UpperCamelCase_,      skip_special_tokens=UpperCamelCase_)
   __lowercase								=						[text.strip() for text in output_text]
   print("HF generation:",      UpperCamelCase_)

   if pytorch_dump_folder_path is not None:
      processor.save_pretrained(UpperCamelCase_)
      hf_model.save_pretrained(UpperCamelCase_)

   if push_to_hub:
      processor.push_to_hub(F"""Salesforce/{model_name}""")
      hf_model.push_to_hub(F"""Salesforce/{model_name}""")


if __name__ == "__main__":
   _a							=							argparse.ArgumentParser()
   _a							=							[
       'instructblip-vicuna-7b',
       'instructblip-vicuna-13b',
       'instructblip-flan-t5-xl',
       'instructblip-flan-t5-xxl',
   ]
   parser.add_argument(
       '--model_name',
       default='instructblip-flan-t5-xl',
       choices=choices,
       type=str,
       help='Path to hf config.json of model to convert',
   )
   parser.add_argument('--pytorch_dump_folder_path', default=None, type=str, help='Path to the output PyTorch model.')
   parser.add_argument(
       '--push_to_hub',
       action='store_true',
       help='Whether to push the model and processor to the hub after converting',
   )

   _a							=							parser.parse_args()

   convert_blipa_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)







"""simple docstring"""





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


_a							=							logging.get_logger(__name__)

_a							=							{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

_a							=							[
    'small',
    'small-base',
    'medium',
    'medium-base',
    'intermediate',
    'intermediate-base',
    'large',
    'large-base',
    'xlarge',
    'xlarge-base',
]

_a							=							{
    'vocab_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt',
        'funnel-transformer/small-base': 'https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt',
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt',
        'funnel-transformer/large-base': 'https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt',
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'funnel-transformer/small': 'https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json',
        'funnel-transformer/small-base': (
            'https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/medium': 'https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json',
        'funnel-transformer/medium-base': (
            'https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate': (
            'https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/intermediate-base': (
            'https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/large': 'https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json',
        'funnel-transformer/large-base': (
            'https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'
        ),
        'funnel-transformer/xlarge': 'https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json',
        'funnel-transformer/xlarge-base': (
            'https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'
        ),
    },
}
_a							=							{F"funnel-transformer/{name}": 5_12 for name in _model_names}
_a							=							{F"funnel-transformer/{name}": {'do_lower_case': True} for name in _model_names}




class  _lowerCAmelCase  (     lowercase       ):







 """simple docstring"""







 __UpperCAmelCase     : Optional[Any]       		=     VOCAB_FILES_NAMES
 __UpperCAmelCase     : Tuple       		=     PRETRAINED_VOCAB_FILES_MAP
 __UpperCAmelCase     : Union[str, Any]       		=     PRETRAINED_INIT_CONFIGURATION
 __UpperCAmelCase     : Optional[Any]       		=     FunnelTokenizer
 __UpperCAmelCase     : List[str]       		=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __UpperCAmelCase     : int       		=     2


 def __init__(      self							:      Dict, UpperCAmelCase__							:      Dict=None, UpperCAmelCase__							:      Optional[int]=None, UpperCAmelCase__							:      Dict=True, UpperCAmelCase__							:      List[str]="<unk>", UpperCAmelCase__							:      Optional[Any]="<sep>", UpperCAmelCase__							:      Optional[Any]="<pad>", UpperCAmelCase__							:      Union[str, Any]="<cls>", UpperCAmelCase__							:      str="<mask>", UpperCAmelCase__							:      Optional[Any]="<s>", UpperCAmelCase__							:      Tuple="</s>", UpperCAmelCase__							:      Union[str, Any]=True, UpperCAmelCase__							:      Optional[int]=True, UpperCAmelCase__							:      Optional[Any]=None, UpperCAmelCase__							:      Dict="##", **UpperCAmelCase__							:      List[str], ):
    super().__init__(
        UpperCAmelCase__, tokenizer_file=UpperCAmelCase__, do_lower_case=UpperCAmelCase__, unk_token=UpperCAmelCase__, sep_token=UpperCAmelCase__, pad_token=UpperCAmelCase__, cls_token=UpperCAmelCase__, mask_token=UpperCAmelCase__, bos_token=UpperCAmelCase__, eos_token=UpperCAmelCase__, clean_text=UpperCAmelCase__, tokenize_chinese_chars=UpperCAmelCase__, strip_accents=UpperCAmelCase__, wordpieces_prefix=UpperCAmelCase__, **UpperCAmelCase__, )

    __lowercase								=						json.loads(self.backend_tokenizer.normalizer.__getstate__()     )
    if (
        normalizer_state.get("lowercase", UpperCAmelCase__     ) != do_lower_case
        or normalizer_state.get("strip_accents", UpperCAmelCase__     ) != strip_accents
        or normalizer_state.get("handle_chinese_chars", UpperCAmelCase__     ) != tokenize_chinese_chars
    ):
       __lowercase								=						getattr(UpperCAmelCase__, normalizer_state.pop("type"     )     )
       __lowercase								=						do_lower_case
       __lowercase								=						strip_accents
       __lowercase								=						tokenize_chinese_chars
       __lowercase								=						normalizer_class(**UpperCAmelCase__     )

    __lowercase								=						do_lower_case


 def  _lowercase			(      self							:      Tuple, UpperCAmelCase__							:      Any, UpperCAmelCase__							:      Any=None     ):
    __lowercase								=						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

    if token_ids_a:
       output += token_ids_a + [self.sep_token_id]

    return output


 def  _lowercase			(      self							:      str, UpperCAmelCase__							:      List[int], UpperCAmelCase__							:      Optional[List[int]] = None     ):
    __lowercase								=						[self.sep_token_id]
    __lowercase								=						[self.cls_token_id]
    if token_ids_a is None:
       return len(cls     ) * [self.cls_token_type_id] + len(token_ids_a + sep     ) * [0]
    return len(cls     ) * [self.cls_token_type_id] + len(token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1]



 def  _lowercase			(      self							:      Optional[Any], UpperCAmelCase__							:      str, UpperCAmelCase__							:      Optional[str] = None     ):
    __lowercase								=						self._tokenizer.model.save(UpperCAmelCase__, name=UpperCAmelCase__     )
    return tuple(UpperCAmelCase__     )




