Add Matrix NL-query Q&A surface (W2 step 5)

Read-only natural-language query over the curated nl_query endpoint, answered
in-thread. Two entry points (room-per-purpose model): a dedicated Q&A room
(MATRIX_QUERY_ROOM) where every top-level message is a question, plus the
?/@bot trigger in the intake room as a cross-room convenience. Both routes hit
the same handle_query -> crm_client.nl_query -> POST /api/query/nl; translation
runs on the box's local model, nothing leaves the box, and there is no write
path so no approval gate applies.

Pure logic (trigger parsing, answer rendering) in query.py with offline tests;
async room wiring in bot.py (live-smoke only, per the bot's convention).

Bot-side only, ships on the Spark via git pull + restart. Depends on the
box-side /api/query/nl endpoint, which lands with the v93 s9pk (reminders + W2):
until v93 is installed the Q&A surface 404s, so the bot deploy is staged to
follow that install.

backend/matrix_intake/bot.py

@@ -18,6 +18,7 @@ import email_proposals
 import matrix_io
 import parse
 import proposals
+import query
 import settings
 
 UNCLEAR_HELP = (
@@ -42,6 +43,7 @@ async def main():
     if roster:
         print(f"matrix-intake: team roster loaded ({len(roster)} names)", flush=True)
     review_room = settings.email_review_room()  # CRM-drafted email proposals (empty → feature off)
+    query_room = settings.query_room()  # dedicated read-only Q&A room (empty → use the intake trigger)
     email_threads = {}  # Matrix thread-root event_id -> {id, investor_name, note} for an email proposal
 
     async def handle_intake(room_id, root, text):
@@ -97,6 +99,19 @@ async def main():
         # easy to miss inside a thread (the full card + yes/edit/no stay in the thread).
         await nudge(room_id, proposals.summary_line(proposal), root)
 
+    async def handle_query(room_id, root, question):
+        """A read-only NL question ('@bot …' / '?…') — translate + run it on the BOX (local Qwen,
+        nothing leaves the box) and post the answer in a thread. No write path, no approval gate:
+        it only reads curated, parameterized queries. The endpoint returns its structured result
+        even on a soft no-match / model-down, so we render that; a transport/auth failure raises
+        and we show a brief error."""
+        try:
+            result = await asyncio.to_thread(crm_client.nl_query, question)
+        except Exception as exc:
+            await say(room_id, f"⚠️ couldn't run that query: {str(exc)[:200]}", root)
+            return
+        await say(room_id, query.render_answer(result), root)
+
     async def handle_reply(room_id, root, text):
         # Claim the proposal synchronously — BEFORE any await — so a second reply that
         # arrives while a commit is in flight can't double-process it. asyncio is
@@ -299,6 +314,12 @@ async def main():
             if root and root in email_threads:
                 await handle_email_reply(room.room_id, root, text)
             return
+        # Dedicated Q&A room: every top-level message IS a question — no trigger needed. Threaded
+        # messages (the answers we post, or follow-ups) aren't acted on in v1.
+        if query_room and room.room_id == query_room:
+            if not root:
+                await handle_query(room.room_id, event.event_id, text)
+            return
         if room.room_id != intake_room:
             return
         if root and store.has(root):
@@ -306,7 +327,15 @@ async def main():
         elif root:
             return  # threaded message not tied to a live proposal — ignore
         else:
-            await handle_intake(room.room_id, event.event_id, text)
+            # A top-level message is either an NL question (explicitly addressed with '?'/'@bot')
+            # or an intake note. The trigger is required, so plain notes still flow to intake.
+            q = query.parse_trigger(text)
+            if q is None:
+                await handle_intake(room.room_id, event.event_id, text)
+            elif not q:
+                await say(room.room_id, query.HELP, event.event_id)
+            else:
+                await handle_query(room.room_id, event.event_id, q)
 
     # Prime the sync token past history, THEN register the callback — only react to messages
     # arriving after startup (no backlog replay). (matrix-bridge pattern.)
@@ -325,6 +354,14 @@ async def main():
             print(f"matrix-intake: could not join review room {review_room}: {exc}", flush=True)
         tasks.append(asyncio.create_task(poll_email_proposals()))
         print(f"matrix-intake: reviewing email proposals in room {review_room} (every {EMAIL_POLL_SEC}s)", flush=True)
+    if query_room:
+        # Read-only Q&A room — just join and listen (no poll task; questions are interactive).
+        # "Invited" isn't "joined": the bot must join before it can post answers (idempotent).
+        try:
+            await client.join(query_room)
+        except Exception as exc:
+            print(f"matrix-intake: could not join Q&A room {query_room}: {exc}", flush=True)
+        print(f"matrix-intake: answering questions in room {query_room}", flush=True)
     try:
         await asyncio.gather(*tasks)
     finally:

backend/matrix_intake/crm_client.py

@@ -139,6 +139,19 @@ def decide_email_proposal(proposal_id, decision, note=None):
     return data.get("data") or {}
 
 
+def nl_query(question):
+    """Ask the read-only NL-query endpoint (POST /api/query/nl). Translation runs on the box's
+    LOCAL model — the question never leaves the box and no write is possible. Returns the
+    endpoint's structured result dict ({intent, slots, rows, summary, ...} or {error, detail});
+    the server returns that same body on a hit AND on the soft 503 (model down) / 500 (query
+    fault) status codes, so we hand it straight to the renderer. Any OTHER status — auth (403),
+    a malformed request (400), an unexpected shape — raises so the caller posts a brief error."""
+    status, data = _authed("POST", "/api/query/nl", {"question": question, "source": "matrix"})
+    if status not in (200, 500, 503):
+        raise RuntimeError(f"nl-query failed ({status}): {data.get('error') or data}")
+    return data.get("data") or {}
+
+
 def build_commit_payload(proposal):
     """Pure: map a proposal to the /api/fundraising/log-communication request body.
 

backend/matrix_intake/query.py

@@ -0,0 +1,189 @@
+"""NL-query Matrix surface (W2 step 5) — turn an '@bot <question>' message into a read-only
+answer from the CRM's curated NL-query endpoint, and render that answer for the chat room.
+
+This module is PURE (no network, no matrix-nio) so it's unit-testable offline; the async wiring
+(call the endpoint, post in a thread) lives in bot.py. The endpoint does the real work:
+translation runs on the box's LOCAL model (nothing leaves the box) and only the curated,
+parameterized queries can run — there is no write path here, so no approval gate applies.
+
+Trigger: a top-level message starting with '?' / '@bot' / '/ask' (see parse_trigger). We
+deliberately do NOT accept a bare leading 'ask', which would collide with intake notes like
+"Ask Jane to send the Q3 deck".
+"""
+
+# Markers a human wouldn't start an intake note with. '?' is handled separately (single char).
+QUERY_PREFIXES = ("@bot", "/ask", "/query", "/q")
+
+# Soft cap on rows rendered into a single chat answer. The endpoint already caps the SQL result
+# (server MAX_ROWS), but 500 rows is unreadable on mobile — show the first N and say how many
+# more there are (never a silent cut). Refine the question or use the web Ask box for the rest.
+MAX_DISPLAY_ROWS = 30
+
+# Column-name hints used only for nicer formatting (money / dates). Cosmetic — never affects
+# what's queried (that's fixed in intents.py).
+_MONEY_HINTS = ("amount", "invested", "total", "expected", "committed")
+# 0/1 flag columns: suppress when 0 (noise), show a label when 1.
+_FLAG_LABELS = {"graveyard": "retired", "overdue": "⚠️ overdue"}
+
+
+def parse_trigger(text):
+    """If `text` is addressed to the query bot, return the question (the remainder after the
+    trigger, possibly an empty string when the trigger is bare). Return None if it isn't a query,
+    so the caller routes it to intake instead."""
+    s = (text or "").strip()
+    if not s:
+        return None
+    if s[0] == "?":
+        return s[1:].strip()
+    low = s.lower()
+    for p in QUERY_PREFIXES:
+        if low.startswith(p):
+            rest = s[len(p):]
+            # Require a separator so '/asking …' isn't read as the '/ask' trigger.
+            if rest == "" or rest[0] in " \t\n:,":
+                return rest.lstrip(" \t\n:,").strip()
+    return None
+
+
+def _examples():
+    return ("Try things like:\n"
+            "• `?which investors haven't we contacted in 90 days?`\n"
+            "• `?top 10 investors by committed capital`\n"
+            "• `?when did we last reach out to Acme Capital?`\n"
+            "• `?how many emails has Grant sent this month?`")
+
+
+HELP = ("💬 Ask me about the fundraising database — start your message with `?` (or `@bot`).\n\n"
+        + _examples())
+
+
+def _is_money_col(name):
+    n = name.lower()
+    return any(h in n for h in _MONEY_HINTS)
+
+
+def _fmt_value(col, val):
+    """Format one scalar cell for chat: dates -> YYYY-MM-DD, money columns -> $1,234, else str."""
+    if val is None:
+        return ""
+    name = col.lower()
+    if name.endswith("_at") or name.endswith("date"):
+        return str(val)[:10]
+    if isinstance(val, (int, float)) and _is_money_col(col):
+        return f"${val:,.0f}"
+    return str(val)
+
+
+def _render_contacts(contacts):
+    """investor_lookup's nested contact dicts -> 'Name <email> (title · city, state)' lines."""
+    out = []
+    for c in contacts:
+        bits = c.get("full_name") or "?"
+        if c.get("email"):
+            bits += f" <{c['email']}>"
+        loc = ", ".join(x for x in (c.get("city"), c.get("state"), c.get("country")) if x)
+        extra = " · ".join(x for x in (c.get("title"), loc) if x)
+        if extra:
+            bits += f" ({extra})"
+        out.append(bits)
+    return out
+
+
+def _render_commitments(commitments):
+    """investor_lookup's nested per-fund commitments -> 'Fund: $amount' lines."""
+    out = []
+    for c in commitments:
+        fund = c.get("fund_name") or "?"
+        amt = c.get("amount")
+        out.append(f"{fund}: ${amt:,.0f}" if isinstance(amt, (int, float)) else f"{fund}: {amt}")
+    return out
+
+
+def _render_row(i, row, columns):
+    cols = columns or list(row.keys())
+    lead = None
+    scalars = []
+    sublines = []
+    for col in cols:
+        val = row.get(col)
+        if isinstance(val, list):
+            if not val:
+                continue
+            if col == "contacts":
+                sublines += [f"    – {x}" for x in _render_contacts(val)]
+            elif col == "commitments":
+                sublines += [f"    – {x}" for x in _render_commitments(val)]
+            else:  # generic list-of-dicts fallback (no intent uses this yet)
+                sublines += [f"    – {', '.join(f'{k}={v}' for k, v in d.items())}"
+                             for d in val if isinstance(d, dict)]
+            continue
+        if col in _FLAG_LABELS:
+            if val:
+                scalars.append(_FLAG_LABELS[col])
+            continue
+        s = _fmt_value(col, val)
+        if s == "":
+            continue
+        if lead is None:  # first non-empty column is the bold identifier for the row
+            lead = s
+        else:
+            scalars.append(f"{col}: {s}")
+    head = f"{i}. **{lead}**" if lead else f"{i}."
+    if scalars:
+        head += " — " + " · ".join(scalars)
+    return "\n".join([head] + sublines)
+
+
+def _render_interpretation(intent, slots):
+    if not intent:
+        return ""
+    if slots:
+        return f"read as: {intent} ({', '.join(f'{k}={v}' for k, v in slots.items())})"
+    return f"read as: {intent}"
+
+
+def _render_error(err, result):
+    detail = (result.get("detail") or "").strip()
+    if err == "no_match":
+        return "🤷 I couldn't map that to one of my saved queries.\n\n" + _examples()
+    if err == "model_unavailable":
+        return "⚠️ The local query model is unreachable right now — try again in a moment."
+    if err == "query_failed":
+        return f"⚠️ That query failed to run{(': ' + detail) if detail else ''}."
+    # unknown_intent / bad_slot / anything unexpected
+    return (f"⚠️ I couldn't run that ({err}){(': ' + detail) if detail else ''}.\n\n" + _examples())
+
+
+def render_answer(result):
+    """Render the NL-query endpoint's structured result into a Matrix markdown answer.
+
+    `result` is the endpoint body: a hit {intent, slots, columns, rows, summary, truncated} or
+    an error {error, detail}. Results never go back to any model — this is a deterministic format."""
+    result = result or {}
+    err = result.get("error")
+    if err:
+        return _render_error(err, result)
+
+    summary = (result.get("summary") or "").strip()
+    rows = result.get("rows") or []
+    columns = result.get("columns") or []
+    header = f"📊 {summary}" if summary else "📊 Done."
+    interp = _render_interpretation(result.get("intent"), result.get("slots") or {})
+    if interp:
+        header += f"\n_{interp}_"
+    if not rows:
+        return header + "\n\n(no matching records)"
+
+    shown = rows[:MAX_DISPLAY_ROWS]
+    blocks = [_render_row(i + 1, r, columns) for i, r in enumerate(shown)]
+    out = header + "\n\n" + "\n".join(blocks)
+
+    notes = []
+    extra = len(rows) - len(shown)
+    if extra > 0:
+        notes.append(f"+{extra} more not shown")
+    if result.get("truncated"):
+        notes.append("results hit the server cap")
+    if notes:
+        out += "\n\n_" + "; ".join(notes) + " — refine your question or use the web Ask box._"
+    return out

backend/matrix_intake/settings.py

@@ -68,3 +68,13 @@ def team_roster():
 # intake flow. Unset/empty disables the whole email-review poll loop (the bot just does intake).
 def email_review_room():
     return os.environ.get("MATRIX_EMAIL_REVIEW_ROOM", "").strip()
+
+
+# Dedicated Q&A room for read-only natural-language queries (W2). In this room EVERY top-level
+# message is treated as a question — no '?'/'@bot' trigger needed (the trigger only exists to
+# disambiguate question-vs-note when Q&A shares the intake room; here that's unnecessary). The
+# '?'/'@bot' trigger still works in the intake room too, as a cross-room convenience. Unset/empty
+# just means no dedicated room (questions then go through the intake-room trigger). The bot must be
+# a member of this room. Read-only — no approval gate, no redaction, no special power level needed.
+def query_room():
+    return os.environ.get("MATRIX_QUERY_ROOM", "").strip()

backend/matrix_intake/test_crm_client.py

@@ -113,6 +113,41 @@ def test_match_no_query_skips_network():
     assert res == {"match": None, "candidates": []}
 
 
+def test_nl_query_returns_endpoint_data():
+    cap = {}
+    orig = _with_stub_authed(
+        (200, {"data": {"intent": "top_investors_committed", "rows": [], "summary": "ok"}}), cap)
+    try:
+        res = crm_client.nl_query("top investors")
+    finally:
+        crm_client._authed = orig
+    assert res["intent"] == "top_investors_committed"
+    assert cap["path"] == "/api/query/nl"
+
+
+def test_nl_query_passes_through_soft_503():
+    # Model-down still carries a structured body (the endpoint 503s with the error in `data`) —
+    # return it for the renderer to surface, don't raise.
+    orig = _with_stub_authed((503, {"data": {"error": "model_unavailable"}}))
+    try:
+        res = crm_client.nl_query("anything")
+    finally:
+        crm_client._authed = orig
+    assert res["error"] == "model_unavailable"
+
+
+def test_nl_query_raises_on_auth_failure():
+    orig = _with_stub_authed((403, {"error": "Bot or admin required"}))
+    raised = False
+    try:
+        crm_client.nl_query("x")
+    except RuntimeError:
+        raised = True
+    finally:
+        crm_client._authed = orig
+    assert raised
+
+
 if __name__ == "__main__":
     fns = [v for k, v in sorted(globals().items()) if k.startswith("test_") and callable(v)]
     for fn in fns:

backend/matrix_intake/test_query.py

@@ -0,0 +1,112 @@
+"""Tests for the NL-query Matrix surface: trigger detection + answer rendering (pure, no network)."""
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+import query  # noqa: E402
+
+
+# ── parse_trigger ───────────────────────────────────────────────────────────────────────
+def test_trigger_question_mark():
+    assert query.parse_trigger("?who are our top investors") == "who are our top investors"
+    assert query.parse_trigger("  ? spaced out ") == "spaced out"
+
+
+def test_trigger_at_bot():
+    assert query.parse_trigger("@bot top 10 investors") == "top 10 investors"
+    assert query.parse_trigger("@bot: top 10 investors") == "top 10 investors"   # pill-style colon
+    assert query.parse_trigger("@BOT  spaced") == "spaced"                       # case-insensitive
+
+
+def test_trigger_slash_forms():
+    assert query.parse_trigger("/ask when did we last email Acme?") == "when did we last email Acme?"
+    assert query.parse_trigger("/query top investors") == "top investors"
+    assert query.parse_trigger("/q top investors") == "top investors"
+
+
+def test_trigger_bare_returns_empty_string():
+    # A bare trigger is matched (so we show help) but carries no question.
+    assert query.parse_trigger("@bot") == ""
+    assert query.parse_trigger("?") == ""
+
+
+def test_non_trigger_routes_to_intake():
+    assert query.parse_trigger("New investor: Acme — Jane <jane@acme.com>") is None
+    # 'ask' as a note verb must NOT trigger (would collide with real intake notes).
+    assert query.parse_trigger("Ask Jane to send the Q3 deck") is None
+    assert query.parse_trigger("/asking for a friend") is None  # needs a separator after /ask
+    assert query.parse_trigger("") is None
+    assert query.parse_trigger("   ") is None
+
+
+# ── render_answer ───────────────────────────────────────────────────────────────────────
+def test_render_scalar_rows():
+    out = query.render_answer({
+        "intent": "top_investors_committed", "slots": {"limit": 2},
+        "summary": "Top 2 investor(s) by committed capital.",
+        "columns": ["investor_name", "total_invested", "lead"],
+        "rows": [{"investor_name": "Acme Capital", "total_invested": 5000000, "lead": "Grant"},
+                 {"investor_name": "Beta Fund", "total_invested": 2500000, "lead": "Jonathan"}],
+        "truncated": False})
+    assert "Top 2 investor(s)" in out
+    assert "**Acme Capital**" in out
+    assert "$5,000,000" in out                       # money formatting
+    assert "read as: top_investors_committed" in out  # interpretation footer
+
+
+def test_render_nested_contacts_and_commitments():
+    out = query.render_answer({
+        "intent": "investor_lookup", "slots": {"name": "Acme"},
+        "summary": '1 investor(s) matching "Acme".',
+        "columns": ["investor_name", "lead", "total_invested", "graveyard", "contacts", "commitments"],
+        "rows": [{"investor_name": "Acme Capital", "lead": "Grant", "total_invested": 5000000,
+                  "graveyard": 0,
+                  "contacts": [{"full_name": "Jane Doe", "email": "jane@acme.com", "title": "GP",
+                                "city": "Austin", "state": "TX", "country": ""}],
+                  "commitments": [{"fund_name": "Fund I", "amount": 5000000}]}],
+        "truncated": False})
+    assert "Jane Doe <jane@acme.com>" in out
+    assert "Fund I: $5,000,000" in out
+    assert "graveyard" not in out  # 0-valued flag column suppressed
+
+
+def test_render_flag_when_set():
+    out = query.render_answer({
+        "intent": "investors_follow_up", "slots": {},
+        "summary": "1 investor(s) with an open follow-up reminder.",
+        "columns": ["investor_name", "title", "due_date", "status", "overdue"],
+        "rows": [{"investor_name": "Acme", "title": "Send deck", "due_date": "2026-01-01",
+                  "status": "open", "overdue": 1}]})
+    assert "⚠️ overdue" in out
+    assert "2026-01-01" in out  # date truncated to YYYY-MM-DD
+
+
+def test_render_no_rows():
+    out = query.render_answer({"intent": "investors_by_city", "slots": {"city": "Nowhere"},
+                               "summary": '0 investor contact(s) in "Nowhere".',
+                               "columns": [], "rows": []})
+    assert "no matching" in out.lower()
+
+
+def test_render_overflow_note():
+    rows = [{"investor_name": f"Inv {i}", "total_invested": i}
+            for i in range(query.MAX_DISPLAY_ROWS + 5)]
+    out = query.render_answer({"intent": "top_investors_committed", "slots": {}, "summary": "many",
+                               "columns": ["investor_name", "total_invested"], "rows": rows})
+    assert "+5 more not shown" in out
+
+
+def test_render_errors():
+    assert "couldn't map" in query.render_answer({"error": "no_match", "question": "huh"}).lower()
+    assert "unreachable" in query.render_answer({"error": "model_unavailable"}).lower()
+    assert "failed" in query.render_answer({"error": "query_failed", "detail": "boom"}).lower()
+    assert "bad_slot" in query.render_answer({"error": "bad_slot", "detail": "x"})
+
+
+if __name__ == "__main__":
+    fns = [v for k, v in sorted(globals().items()) if k.startswith("test_") and callable(v)]
+    for fn in fns:
+        fn()
+        print(f"ok  {fn.__name__}")
+    print(f"\n{len(fns)} passed")