Update app.py

app.py

@@ -1,9 +1,9 @@
 # ================================
-# 🪞 MoodMirror+ — Dataset-only Emotion + Advice
-# - GoEmotions (scikit-learn TF-IDF + OneVsRest LR)
-# - Advice stored in-code (not from dataset)
-# - Always returns exactly one advice (no headers/quotes)
-# - "New advice" gives a different tip and cycles the pool
+# 🪞 MoodMirror+ — Text + Voice Emotion Detector • Advice-only
+# - Text: GoEmotions (TF-IDF + OneVsRest LR, dataset-only)
+# - Audio: HF pipeline "firdhokk/speech-emotion-recognition-with-openai-whisper-large-v3"
+# - Advice stored in-code (not from a database)
+# - Always returns exactly one advice; "New advice" avoids duplicates
 # ================================
 import os
 import re
@@ -21,6 +21,10 @@ from sklearn.linear_model import LogisticRegression
 from sklearn.multiclass import OneVsRestClassifier
 from sklearn.pipeline import Pipeline
 
+# Optional audio pipeline (lazy import)
+AUDIO_MODEL_ID = "firdhokk/speech-emotion-recognition-with-openai-whisper-large-v3"
+AUDIO_PIPE = None   # loaded on first use
+
 # ---------------- Storage paths ----------------
 def _pick_data_dir():
     if os.path.isdir("/data") and os.access("/data", os.W_OK):
@@ -31,7 +35,7 @@ DATA_DIR = _pick_data_dir()
 os.makedirs(DATA_DIR, exist_ok=True)
 DB_PATH = os.path.join(DATA_DIR, "moodmirror.db")
 MODEL_PATH = os.path.join(DATA_DIR, "goemo_sklearn.joblib")
-MODEL_VERSION = "v7-prompt-spec"
+MODEL_VERSION = "v9-text+audio"
 
 # ---------------- Crisis & closing ----------------
 CRISIS_RE = re.compile(
@@ -52,7 +56,7 @@ CRISIS_NUMBERS = {
     "Other / Not listed": "Call local emergency (**112/911**) or search “suicide hotline” for your country.",
 }
 
-# ---------------- Advice library (concise, actionable; 12+ per emotion) ----------------
+# ---------------- Advice library (concise, actionable) ----------------
 SUGGESTIONS = {
     "sadness": [
         "Go for a 5-minute outside walk and notice three colors.",
@@ -63,7 +67,7 @@ SUGGESTIONS = {
         "Listen to a song that matches your mood, not one that hides it.",
         "Wrap yourself in a blanket and slow your exhale for 60 seconds.",
         "List 3 small things that kept you going today.",
-        "Tidy one tiny area (desk corner, sink) to regain a sense of control.",
+        "Tidy one tiny area to regain a sense of control.",
         "Watch something gentle/nostalgic for 10 minutes.",
         "Place a hand on your chest and say: ‘This will pass.’",
         "Write a note to your future self: ‘You made it through this day.’",
@@ -113,7 +117,7 @@ SUGGESTIONS = {
     "boredom": [
         "Set a 2-minute timer and start anything small.",
         "Change your soundtrack; one new song shifts mood.",
-        "Move furniture or objects slightly — new perspective.",
+        "Move objects slightly — new perspective.",
         "Read one paragraph on a random topic.",
         "Sketch, doodle, or hum for 90 seconds.",
         "Step outside; look up and find three shapes.",
@@ -133,7 +137,7 @@ SUGGESTIONS = {
         "Let tears come when they need to.",
         "Light a candle and sit quietly for two minutes.",
         "Walk somewhere meaningful and notice what you feel.",
-        "Create a small ritual (song, place, phrase) to honor them.",
+        "Create a small ritual to honor them.",
         "Schedule one kind plan for yourself this week.",
         "Say: ‘Missing you means I loved you.’",
         "Place your feet firmly and breathe into your belly.",
@@ -144,7 +148,7 @@ SUGGESTIONS = {
         "Offer yourself one gentle act you needed today.",
         "Listen fully to someone for one minute.",
         "Give a sincere compliment to a stranger.",
-        "Cook or prepare something with care for someone.",
+        "Prepare something with care for someone.",
         "Say ‘thank you’ out loud for something small.",
         "Ask a caring question and wait for the answer.",
         "Write what love means to you in 3 lines.",
@@ -177,7 +181,7 @@ SUGGESTIONS = {
         "Try a new route or view in your space.",
         "Learn a word and use it once.",
         "List five topics you’d like to explore.",
-        "Join a community/forum and read one thread.",
+        "Join a community and read one thread.",
         "Sketch a simple diagram of an idea.",
     ],
     "gratitude": [
@@ -205,7 +209,7 @@ SUGGESTIONS = {
         "Organize three items in your space.",
         "Set a 10-minute timer to focus on one thing.",
         "Do a gentle neck stretch for 30 seconds.",
-        "Check in with your posture; support your back.",
+        "Check your posture; support your back.",
         "Open a window and take three breaths.",
     ],
 }
@@ -230,7 +234,17 @@ GOEMO_TO_APP = {
     "sadness": "sadness", "surprise": "neutral", "neutral": "neutral",
 }
 
-# ---------------- Preprocessing & thresholds ----------------
+# Audio model label -> buckets (best effort; covers typical SER labels)
+AUDIO_LABEL_TO_BUCKET = {
+    "anger": "anger", "angry": "anger", "disgust": "anger",
+    "fear": "fear", "scared": "fear", "afraid": "fear",
+    "happy": "joy", "happiness": "joy", "joy": "joy",
+    "sad": "sadness", "sadness": "sadness",
+    "neutral": "neutral", "surprise": "neutral", "calm": "neutral",
+    "boredom": "boredom", "love": "love"
+}
+
+# ---------------- Preprocessing & thresholds (text) ----------------
 THRESHOLD_BASE = 0.30
 MIN_THRESHOLD = 0.10
 
@@ -247,39 +261,27 @@ NEGATION_HINTS_EN = {
     "not happy": "sadness", "not ok": "sadness", "no energy": "boredom",
     "can't focus": "nervousness", "cannot focus": "nervousness"
 }
-HINTS_FR = {
+HINTS_FR = {  # lightweight FR cues (dataset stays EN)
     "pas bien": "sadness", "triste": "sadness", "j'ai peur": "fear",
     "angoisse": "nervousness", "anxieux": "nervousness",
     "fatigué": "sadness", "épuisé": "sadness",
 }
 
 def augment_text(text: str, history=None) -> str:
-    """Clean + add emoji/negation hints and short-context when very short."""
+    """Clean + emoji/negation hints + short-context when short."""
     t = clean_text(text)
     hints = []
-
-    # emoji hints
     for k, v in EMOJI_HINTS.items():
-        if k in text:
-            hints.append(v)
-
-    # english negations
+        if k in text: hints.append(v)
     for k, v in NEGATION_HINTS_EN.items():
-        if k in t:
-            hints.append(v)
-
-    # french cues
+        if k in t: hints.append(v)
     lt = text.lower()
     for k, v in HINTS_FR.items():
-        if k in lt:
-            hints.append(v)
-
-    # short-context boost: if message < 8 words, append previous user line
+        if k in lt: hints.append(v)
     if history and len(t.split()) < 8:
         prev_user = history[-1][0] if history and history[-1] else ""
         if isinstance(prev_user, str) and prev_user:
             t = t + " " + clean_text(prev_user)
-
     if hints:
         t = t + " " + " ".join([f"emo_{h}" for h in hints])
     return t
@@ -300,11 +302,11 @@ def init_db():
 def log_session(country, msg, emotion):
     conn = get_conn()
     conn.execute("INSERT INTO sessions(ts,country,user_text,main_emotion)VALUES(?,?,?,?)",
-                 (datetime.utcnow().isoformat(timespec='seconds'), country, msg[:500], emotion))
+                 (datetime.utcnow().isoformat(timespec='seconds'), country, (msg or "")[:500], emotion))
     conn.commit()
     conn.close()
 
-# ---------------- Model: GoEmotions dataset-only ----------------
+# ---------------- Text model: GoEmotions dataset-only ----------------
 def load_goemotions_dataset():
     ds = load_dataset("google-research-datasets/go_emotions", "simplified")
     return ds, ds["train"].features["labels"].feature.names
@@ -316,10 +318,8 @@ def train_or_load_model():
             return bundle["pipeline"], bundle["mlb"], bundle["label_names"]
     ds, names = load_goemotions_dataset()
     X_train, y_train = ds["train"]["text"], ds["train"]["labels"]
-
     mlb = MultiLabelBinarizer(classes=list(range(len(names))))
     Y_train = mlb.fit_transform(y_train)
-
     clf = Pipeline([
         ("tfidf", TfidfVectorizer(lowercase=True, ngram_range=(1,2), min_df=2, max_df=0.9, strip_accents="unicode")),
         ("ovr", OneVsRestClassifier(
@@ -328,7 +328,6 @@ def train_or_load_model():
         ))
     ])
     clf.fit(X_train, Y_train)
-
     joblib.dump({"version": MODEL_VERSION, "pipeline": clf, "mlb": mlb, "label_names": names}, MODEL_PATH)
     return clf, mlb, names
 
@@ -338,11 +337,10 @@ except Exception as e:
     print("[ERROR] Model load/train:", e)
     CLASSIFIER, MLB, LABEL_NAMES = None, None, None
 
-# ---------------- Inference (dataset-only) ----------------
+# ---------------- Inference: TEXT ----------------
 def classify_text(text_augmented: str):
-    """Return list[(label_name, prob)] using adaptive threshold; fallback top1."""
-    if not CLASSIFIER:
-        return []
+    """Return list[(label_name, prob)] with adaptive threshold; fallback top1."""
+    if not CLASSIFIER: return []
     proba = CLASSIFIER.predict_proba([text_augmented])[0]
     max_p = float(np.max(proba)) if len(proba) else 0.0
     thr = max(MIN_THRESHOLD, THRESHOLD_BASE * max_p + 0.15)
@@ -350,8 +348,8 @@ def classify_text(text_augmented: str):
     idxs.sort(key=lambda i: proba[i], reverse=True)
     return [(LABEL_NAMES[i], float(proba[i])) for i in idxs]
 
-def detect_emotion(message: str, history):
-    labels = classify_text(augment_text(message, history))
+def detect_emotion_text(message: str, history):
+    labels = classify_text(augment_text(message or "", history))
     if not labels:
         return "neutral"
     bucket = {}
@@ -360,18 +358,60 @@ def detect_emotion(message: str, history):
         bucket[app] = max(bucket.get(app, 0.0), p)
     return max(bucket, key=bucket.get) if bucket else "neutral"
 
+# ---------------- Inference: AUDIO ----------------
+def get_audio_pipe():
+    global AUDIO_PIPE
+    if AUDIO_PIPE is not None:
+        return AUDIO_PIPE
+    try:
+        from transformers import pipeline as hf_pipeline
+        AUDIO_PIPE = hf_pipeline("audio-classification", model=AUDIO_MODEL_ID)
+    except Exception as e:
+        print("[WARN] Audio pipeline not available:", e)
+        AUDIO_PIPE = None
+    return AUDIO_PIPE
+
+def detect_emotion_audio(audio_np_tuple):
+    """
+    Gradio Audio (type='numpy') returns (sample_rate:int, data:np.ndarray) or None.
+    We return a bucket string or 'neutral' if unavailable.
+    """
+    if not audio_np_tuple:
+        return None  # signal to fallback
+    sr, data = audio_np_tuple
+    if data is None or (isinstance(data, np.ndarray) and data.size == 0):
+        return None
+    pipe = get_audio_pipe()
+    if pipe is None:
+        return None
+    try:
+        # transformers pipeline accepts (sr, data) tuple directly for recent versions
+        # If needed, you can pass a dict: {"array": data, "sampling_rate": sr}
+        out = pipe({"array": data, "sampling_rate": int(sr)})
+        # out is list of dicts sorted by score desc: [{"label":"happy","score":0.8},...]
+        if not out:
+            return None
+        top = out[0]["label"].lower()
+        # map to bucket
+        for k, v in AUDIO_LABEL_TO_BUCKET.items():
+            if k in top:
+                return v
+        # fallback guess
+        return "neutral"
+    except Exception as e:
+        print("[WARN] Audio infer failed:", e)
+        return None
+
 # ---------------- Advice selection with pool (no immediate repeats) ----------------
 def pick_advice_from_pool(emotion: str, pool: dict, last_tip: str = ""):
     """Pool structure: {emotion: {'unused': [tips], 'last': str}}"""
     tips_all = SUGGESTIONS.get(emotion, SUGGESTIONS["neutral"])
     entry = pool.get(emotion, {"unused": [], "last": ""})
-
-    # Refill when empty, avoid immediate repeat of last
+    # Refill when empty, avoid immediate repeat
     if not entry["unused"]:
-        refill = [t for t in tips_all if t != entry.get("last", "")] or tips_all[:]
+        refill = [t for t in tips_all if t != entry.get("last","")] or tips_all[:]
         random.shuffle(refill)
         entry["unused"] = refill
-
     tip = entry["unused"].pop(0)
     entry["last"] = tip
     pool[emotion] = entry
@@ -382,31 +422,54 @@ def crisis_block(country):
     msg = CRISIS_NUMBERS.get(country, CRISIS_NUMBERS["Other / Not listed"])
     return f"💛 You matter. If you're in danger or thinking of harming yourself, please reach out now.\n\n{msg}"
 
-def chat_step(message, history, country, save_session, advice_pool):
-    if CRISIS_RE.search(message):
+def chat_step(user_text, user_audio, history, country, save_session, advice_pool):
+    # Crisis only on clear text cues
+    if user_text and CRISIS_RE.search(user_text):
         return crisis_block(country), "#FFD6E7", "neutral", "", advice_pool
 
-    if CLOSING_RE.search(message):
-        # Gentle closing with one final neutral tip
+    # Closing minimal
+    if user_text and CLOSING_RE.search(user_text):
         tip, advice_pool = pick_advice_from_pool("neutral", advice_pool)
         return f"• {tip}", "#FFFFFF", "neutral", tip, advice_pool
 
-    emotion = detect_emotion(message, history)
+    # Prefer audio if provided; else text; else neutral
+    emotion = None
+    audio_bucket = detect_emotion_audio(user_audio)
+    if audio_bucket:
+        emotion = audio_bucket
+    elif user_text and user_text.strip():
+        emotion = detect_emotion_text(user_text, history)
+    else:
+        emotion = "neutral"
+
     color = COLOR_MAP.get(emotion, "#F5F5F5")
     if save_session:
-        log_session(country, message, emotion)
+        # store text (if any) so DB stays light
+        log_session(country, user_text or "[audio only]", emotion)
 
     tip, advice_pool = pick_advice_from_pool(emotion, advice_pool)
     return f"• {tip}", color, emotion, tip, advice_pool
 
 # ---------------- UI ----------------
+def get_conn():
+    return sqlite3.connect(DB_PATH, check_same_thread=False, timeout=10)
+
+def init_db():
+    conn = get_conn()
+    conn.execute("""CREATE TABLE IF NOT EXISTS sessions(
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        ts TEXT, country TEXT, user_text TEXT, main_emotion TEXT
+    )""")
+    conn.commit()
+    conn.close()
+
 init_db()
 
-with gr.Blocks(title="🪞 MoodMirror+ — Emotion & Advice") as demo:
+with gr.Blocks(title="🪞 MoodMirror+ — Text & Voice Emotion • Advice-only") as demo:
     style = gr.HTML("")
     gr.Markdown(
-        "### 🪞 MoodMirror+ — Emotion-aware advice (dataset-only)\n"
-        "Offline classifier trained on GoEmotions. Always one concrete tip.\n\n"
+        "### 🪞 MoodMirror+ — Emotion-aware advice (text + voice)\n"
+        "Text model: GoEmotions (dataset-only). Voice model: speech emotion pipeline.\n\n"
         "_Not medical advice. If unsafe, please reach out for help._"
     )
 
@@ -414,26 +477,31 @@ with gr.Blocks(title="🪞 MoodMirror+ — Emotion & Advice") as demo:
         country = gr.Dropdown(list(CRISIS_NUMBERS.keys()), value="Other / Not listed", label="Country")
         save_ok = gr.Checkbox(False, label="Save anonymized session")
 
-    chat = gr.Chatbot(height=380)
-    msg = gr.Textbox(label="Your message", placeholder="Share how you feel...")
+    chat = gr.Chatbot(height=400)
+
+    with gr.Row():
+        msg = gr.Textbox(label="Your message (text)", placeholder="Share how you feel...")
+    with gr.Row():
+        audio = gr.Audio(sources=["microphone", "upload"], type="numpy", label="Or speak (optional)")
+
     with gr.Row():
         send = gr.Button("Send", variant="primary")
         regen = gr.Button("🔁 New advice", variant="secondary")
 
-    # State: last detected emotion, last tip, and per-emotion advice pool
     last_emotion = gr.State("neutral")
     last_tip = gr.State("")
     advice_pool = gr.State({})  # emotion -> {"unused":[...], "last":""}
 
-    def respond(user_msg, chat_hist, country_choice, save_flag, _emotion, _tip, _pool):
-        if not user_msg or not user_msg.strip():
-            return chat_hist + [[user_msg, "Please share how you feel 🙂"]], "", _emotion, _tip, _pool
+    def respond(user_msg, user_audio, chat_hist, country_choice, save_flag, _emotion, _tip, _pool):
+        # If both text and audio empty:
+        if (not user_msg or not user_msg.strip()) and (user_audio is None):
+            return chat_hist + [[user_msg, "Please share a short message or record audio 🙂"]], "", _emotion, _tip, _pool
 
         reply, color, emotion, tip, _pool = chat_step(
-            user_msg, chat_hist, country_choice, bool(save_flag), _pool
+            user_msg, user_audio, chat_hist, country_choice, bool(save_flag), _pool
         )
         style_tag = f"<style>:root,body,.gradio-container{{background:{color}!important;}}</style>"
-        return chat_hist + [[user_msg, reply]], style_tag, emotion, tip, _pool
+        return chat_hist + [[user_msg if user_msg else "[voice]", reply]], style_tag, emotion, tip, _pool
 
     def new_advice(chat_hist, _emotion, _tip, _pool):
         tip, _pool = pick_advice_from_pool(_emotion, _pool, last_tip=_tip)
@@ -442,13 +510,13 @@ with gr.Blocks(title="🪞 MoodMirror+ — Emotion & Advice") as demo:
 
     send.click(
         respond,
-        inputs=[msg, chat, country, save_ok, last_emotion, last_tip, advice_pool],
+        inputs=[msg, audio, chat, country, save_ok, last_emotion, last_tip, advice_pool],
         outputs=[chat, style, last_emotion, last_tip, advice_pool],
         queue=True
     )
     msg.submit(
         respond,
-        inputs=[msg, chat, country, save_ok, last_emotion, last_tip, advice_pool],
+        inputs=[msg, audio, chat, country, save_ok, last_emotion, last_tip, advice_pool],
         outputs=[chat, style, last_emotion, last_tip, advice_pool],
         queue=True
     )