app.py

# ================================
#  app.py - Multimodal RAG Chatbot (Hugging Face Spaces Compatible)
# ================================

import streamlit as st
import torch
from transformers import AutoModelForSeq2SeqLM, AutoTokenizer, pipeline
from pinecone import Pinecone
from sentence_transformers import SentenceTransformer
from transformers import CLIPProcessor, CLIPModel
from PIL import Image

# ======================================
# 1. Setup Pinecone Connection
# ======================================
pinecone_api_key = "pcsk_3vjZtA_STcjYL9Ec6mXyHVT9jKUBafanqEt6KyWnwAGv535utBtXfuEdaKkS2UitgsM6un"  # 🔥 Replace
pc = Pinecone(api_key=pinecone_api_key)

text_index = pc.Index("rag-text-index")
image_index = pc.Index("rag-image-index")

# ======================================
# 2. Setup Local LLM (Flan-T5-Large)
# ======================================
model_name = "google/flan-t5-large"
device = "cuda" if torch.cuda.is_available() else "cpu"
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForSeq2SeqLM.from_pretrained(model_name).to(device)
rag_pipeline = pipeline("text2text-generation", model=model, tokenizer=tokenizer, device=0 if torch.cuda.is_available() else -1)

# ======================================
# 3. Helper Functions
# ======================================
def search_text_index(query_text, top_k=5):
    text_encoder = SentenceTransformer('sentence-transformers/all-mpnet-base-v2', device=device)
    query_embedding = text_encoder.encode(query_text).tolist()
    result = text_index.query(vector=query_embedding, top_k=top_k, include_metadata=True)
    return result['matches']

def search_image_index(uploaded_image, top_k=3):
    clip_model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32").to(device)
    clip_processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")
    inputs = clip_processor(images=uploaded_image, return_tensors="pt").to(device)
    with torch.no_grad():
        query_embedding = clip_model.get_image_features(**inputs)
    query_embedding = query_embedding[0].cpu().numpy().tolist()
    result = image_index.query(vector=query_embedding, top_k=top_k, include_metadata=True)
    return result['matches']

def prepare_context_from_matches(text_matches, image_matches):
    context = ""
    if text_matches:
        context += "TEXTUAL INFORMATION:\n"
        for match in text_matches:
            content = match['metadata'].get('content', '')
            page = match['metadata'].get('page', 'N/A')
            context += f"[Page {page}] {content}\n"
    if image_matches:
        context += "IMAGE INFORMATION:\n"
        for match in image_matches:
            page = match['metadata'].get('page', 'N/A')
            context += f"[Image extracted from Page {page}]\n"
    return context.strip()

def generate_final_answer(context, question):
    if len(context.split()) < 20:
        return "Not enough detailed information retrieved to answer properly."
    prompt = f"""
    You are a financial expert assistant. Answer ONLY based on the context provided below.
    Expand financial abbreviations (e.g., EPS → Earnings Per Share) and explain in full sentences.
    Provide at least 3 complete sentences.

    CONTEXT:
    {context}

    QUESTION:
    {question}

    FINAL ANSWER:
    """
    output = rag_pipeline(prompt)[0]['generated_text']
    return output.strip()

# ======================================
# 4. Streamlit Web App
# ======================================
st.set_page_config(page_title="Multimodal RAG Assistant", page_icon="🤖", layout="centered")
st.title("📚 Multimodal RAG Assistant")

st.write("Ask a question based on uploaded PDFs, or upload a relevant image:")

# Input options
user_query = st.text_input("Enter your question:")
uploaded_image = st.file_uploader("Or upload an image:", type=["png", "jpg", "jpeg"])

if st.button("Submit"):
    if user_query:
        text_matches = search_text_index(user_query)
        image_matches = []
    elif uploaded_image:
        img = Image.open(uploaded_image)
        text_matches = []
        image_matches = search_image_index(img)
    else:
        st.warning("Please either enter a question or upload an image.")
        st.stop()

    # Build context
    context = prepare_context_from_matches(text_matches, image_matches)

    # Generate final answer
    answer = generate_final_answer(context, user_query if user_query else "Describe this image.")

    # Show result
    st.success("✅ Answer:")
    st.write(answer)

    # Show matched chunks
    with st.expander("🔎 View Retrieved Context"):
        st.text(context)

st.sidebar.info("Built with Pinecone + FLAN-T5-Large + Streamlit 🚀")