Update README.md

README.md

@@ -1,3 +1,30 @@
 ---
 license: gpl-3.0
 ---
+
+# Easy CLIP Embeddings alignment with Guannaco Models
+
+```python
+pipe = StableDiffusionPipeline(...)
+# llm_model = load_quant('/GuanacoOnConsumerHardware', 'guanaco7b-4bit-128g.pt', 4, 128, 0)
+llm_tokenizer = LlamaTokenizer.from_pretrained("JosephusCheung/Guanaco",use_fast=False,torch_dtype=torch.float16)
+llm_model = LlamaForCausalLM.from_pretrained("JosephusCheung/Guanaco",device_map="auto",torch_dtype=torch.float16)
+
+class LLMToCLIP(nn.Module):
+    def __init__(self):
+        super(LLMToCLIP, self).__init__()
+        self.proj = nn.Linear(4096, 4096, bias=False)
+        self.deproj = nn.Linear(4096, 768, bias=False)
+        
+    def forward(self, x):
+        a = self.proj(x)
+        b = self.deproj(a)
+        return b
+
+llm_to_clip=LLMToCLIP()
+llm_to_clip.load_state_dict(torch.load("toclip.pth"))
+
+llm_embeddings = llm_model(input_ids=input_ids, output_hidden_states=True).hidden_states[-1]
+
+image = pipe(prompt_embeds=llm_to_clip(llm_embeddings)).images[0]
+```