grant/ten31-database
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Email search/query + windowed digest preview (v0.1.0:83)

Browse Source 
Communications tab (search/query roadmap items 1 & 2):
- Fix the investor dropdown: the facet only listed grid investors, so it
  came back empty whenever email matched a classic contact or org domain
  (no grid id — the common case). It now mirrors the email list, resolving
  each link to a typed identity (fund:/org:/contact:/addr:) with precedence
  grid -> org -> contact -> address; investor_id accepts the typed key
  (bare id = fund: for back-compat) and an unknown prefix matches nothing.
- Add a date-range filter and a click-to-expand full-body view
  (GET /api/email/detail, admin, soft-delete-gated; body_text only, never
  raw remote HTML).
- Add a "Search content" mode: GET /api/email/search wraps the ingest
  hybrid_search over the Qdrant email index (doc_type=email), hydrated and
  soft-delete-filtered against SQLite (canonical), 503 if Spark/Qdrant down.

Daily digest:
- Settings -> Admin builds a digest over a chosen window (last 24h or since
  a date) as an in-app preview before sending (POST /api/admin/digest/preview),
  so the local-Spark summarizer can be verified on demand even on a quiet day.
  Manual send uses the same window; neither advances the daily cursor, so a
  preview never suppresses the scheduled digest.

Code-only, migrations no-op. 22/22 backend tests, render-smoke pass.
This commit is contained in:
Keysat
2026-06-16 20:46:15 -05:00
parent c29ac2f2ee
commit c7b74a2704
14 changed files with 989 additions and 138 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/email_integration/test_email_activity_panel.py
+178 -57
View File
@@ -2,10 +2,13 @@
"""Test the admin-only email-activity panel (Communications tab, v0.1.0:80).
Covers the pure query (`db.query_email_activity`): matched-only scope (unmatched
cold/unknown-sender email is never surfaced), investor/mailbox/search/direction
filters, per-sighting soft-delete, direction at the email level, mailbox + investor
roll-ups, and the filter facets.
Also asserts the route handler enforces admin server-side. Synthetic data only.
cold/unknown-sender email is never surfaced), investor/mailbox/search/direction/
date-range filters, per-sighting soft-delete, direction at the email level, mailbox
roll-ups, and the *typed* investor facet (grid investor / org / contact), including
the v83 fix where an email matched only to a classic contact or org domain — not yet
wired to a grid investor — still resolves to a real name and appears in the dropdown
(previously the facet came back empty). Also asserts the route handler enforces admin
server-side. Synthetic data only.
Run: cd backend && python3 email_integration/test_email_activity_panel.py
"""
@@ -31,17 +34,28 @@ def make_db():
conn.executescript("""
CREATE TABLE email_accounts (id TEXT PRIMARY KEY, email_address TEXT);
CREATE TABLE emails (id TEXT PRIMARY KEY, subject TEXT, from_name TEXT, from_email TEXT,
sent_at TEXT, snippet TEXT, has_attachments INT DEFAULT 0, is_matched INT DEFAULT 0,
sent_at TEXT, snippet TEXT, body_text TEXT, body_html TEXT, gmail_thread_id TEXT,
has_attachments INT DEFAULT 0, is_matched INT DEFAULT 0,
match_status TEXT DEFAULT 'unmatched');
CREATE TABLE email_account_messages (id TEXT PRIMARY KEY, email_id TEXT, account_id TEXT,
is_sent INT DEFAULT 0, deleted_at TEXT);
CREATE TABLE email_recipients (id TEXT PRIMARY KEY, email_id TEXT, address TEXT,
display_name TEXT, kind TEXT);
CREATE TABLE email_attachments (id TEXT PRIMARY KEY, email_id TEXT, filename TEXT,
mime_type TEXT, size_bytes INTEGER, download_status TEXT);
CREATE TABLE email_investor_links (id TEXT PRIMARY KEY, email_id TEXT,
fundraising_investor_id TEXT, fundraising_contact_id TEXT, matched_address TEXT);
fundraising_investor_id TEXT, fundraising_contact_id TEXT,
organization_id TEXT, contact_id TEXT, matched_address TEXT);
CREATE TABLE fundraising_investors (id TEXT PRIMARY KEY, investor_name TEXT, graveyard INTEGER DEFAULT 0);
CREATE TABLE fundraising_contacts (id TEXT PRIMARY KEY, investor_id TEXT, full_name TEXT);
CREATE TABLE organizations (id TEXT PRIMARY KEY, name TEXT, deleted_at TEXT);
CREATE TABLE contacts (id TEXT PRIMARY KEY, first_name TEXT, last_name TEXT,
organization_id TEXT, deleted_at TEXT);
""")
# Two mailboxes (us), three investors (one reached only via a contact link;
# one graveyarded but still with captured email history).
# Two mailboxes (us); investors reached different ways: a grid investor directly,
# a grid investor only via a contact link, a graveyarded grid investor, an org-only
# (domain) match, and a classic-contact-only match (the case that left the dropdown
# empty before v83 — neither carries a grid id).
conn.executemany("INSERT INTO email_accounts VALUES (?,?)", [
("acc-grant", "grant@ten31.xyz"),
("acc-jon", "jonathan@ten31.xyz"),
@@ -52,12 +66,17 @@ def make_db():
("inv-dead", "Dead Deal LP", 1),
])
conn.execute("INSERT INTO fundraising_contacts VALUES ('fc-1','inv-pacific','Sarah Williams')")
conn.execute("INSERT INTO organizations VALUES ('org-bridge','Bridgewater',NULL)")
conn.execute("INSERT INTO contacts VALUES ('c-solo','Nina','Park',NULL,NULL)")
# Emails:
# e1 outbound (from us) -> Harbor, seen by grant
# e2 inbound (from LP) -> Harbor, seen by grant + jonathan
# e3 inbound (from LP) -> Pacific via contact link, seen by jonathan
# e4 inbound, UNMATCHED (no investor link), seen by grant -> must be excluded (matched-only)
# e5 inbound, only sighting is tombstoned -> must be excluded
# e1 outbound -> Harbor (grid), seen by grant
# e2 inbound -> Harbor (grid), seen by grant + jonathan
# e3 inbound -> Pacific via grid contact link, seen by jonathan
# e4 inbound, UNMATCHED -> excluded (matched-only)
# e5 inbound, only sighting tombstoned -> excluded
# e6 inbound -> Dead Deal LP (graveyard grid investor)
# e7 inbound -> Bridgewater via ORG-domain match (no grid id)
# e8 inbound -> Nina Park via CLASSIC-contact match (no grid id, no org)
conn.executemany(
"INSERT INTO emails (id,subject,from_name,from_email,sent_at,snippet,has_attachments,is_matched,match_status) VALUES (?,?,?,?,?,?,?,?,?)",
[
@@ -67,6 +86,8 @@ def make_db():
("e4", "Cold inbound", "Random", "noreply@spam.example", "2026-06-08T08:00:00", "buy now", 0, 0, "unmatched"),
("e5", "Deleted thread", "Ghost", "ghost@x.example", "2026-06-09T08:00:00", "gone", 0, 1, "matched"),
("e6", "Old dead-deal thread", "Dead LP", "lp@deaddeal.example", "2026-06-01T00:00:00", "we passed", 0, 1, "matched"),
("e7", "Macro view", "Ray", "ray@bridgewater.example", "2026-06-10T08:00:00", "rates outlook", 0, 1, "matched"),
("e8", "Coffee?", "Nina Park", "nina@solo.example", "2026-06-11T08:00:00", "in town next week", 0, 1, "matched"),
])
conn.executemany(
"INSERT INTO email_account_messages (id,email_id,account_id,is_sent,deleted_at) VALUES (?,?,?,?,?)",
@@ -78,16 +99,32 @@ def make_db():
("m5", "e4", "acc-grant", 0, None),
("m6", "e5", "acc-grant", 0, "2026-06-10T00:00:00"), # tombstoned
("m7", "e6", "acc-grant", 0, None),
("m8", "e7", "acc-grant", 0, None),
("m9", "e8", "acc-jon", 0, None),
])
conn.executemany(
"INSERT INTO email_investor_links (id,email_id,fundraising_investor_id,fundraising_contact_id,matched_address) VALUES (?,?,?,?,?)",
"INSERT INTO email_investor_links (id,email_id,fundraising_investor_id,fundraising_contact_id,organization_id,contact_id,matched_address) VALUES (?,?,?,?,?,?,?)",
[
("l1", "e1", "inv-harbor", None, "lp@harborvine.example"),
("l2", "e2", "inv-harbor", None, "lp@harborvine.example"),
("l3", "e3", None, "fc-1", "sarah@pacificcap.example"),
("l5", "e5", "inv-harbor", None, "lp@harborvine.example"),
("l6", "e6", "inv-dead", None, "lp@deaddeal.example"),
("l1", "e1", "inv-harbor", None, None, None, "lp@harborvine.example"),
("l2", "e2", "inv-harbor", None, None, None, "lp@harborvine.example"),
("l3", "e3", None, "fc-1", None, None, "sarah@pacificcap.example"),
("l5", "e5", "inv-harbor", None, None, None, "lp@harborvine.example"),
("l6", "e6", "inv-dead", None, None, None, "lp@deaddeal.example"),
("l7", "e7", None, None, "org-bridge", None, "ray@bridgewater.example"),
("l8", "e8", None, None, None, "c-solo", "nina@solo.example"),
])
# Full body + recipients + an attachment on e2, for the detail view.
conn.execute("UPDATE emails SET body_text = ?, gmail_thread_id = ?, has_attachments = 1 WHERE id = 'e2'",
("Thanks for the deck — one question on the carry.", "thr-harbor"))
conn.executemany(
"INSERT INTO email_recipients (id,email_id,address,display_name,kind) VALUES (?,?,?,?,?)",
[
("r1", "e2", "grant@ten31.xyz", "Grant", "to"),
("r2", "e2", "jonathan@ten31.xyz", "Jonathan", "cc"),
("r3", "e2", "lp@harborvine.example", "LP Harbor", "from"), # from -> not surfaced
])
conn.execute("INSERT INTO email_attachments (id,email_id,filename,mime_type,size_bytes,download_status) "
"VALUES ('a1','e2','term_sheet.pdf','application/pdf',20480,'downloaded')")
conn.commit()
return conn
@@ -101,9 +138,9 @@ def main():
# --- baseline: matched live emails only, newest first, tombstoned excluded ---
res = _db.query_email_activity(conn)
check(ids(res) == ["e3", "e2", "e1", "e6"],
check(ids(res) == ["e8", "e7", "e3", "e2", "e1", "e6"],
f"matched live emails newest-first; e5 (tombstoned) + e4 (unmatched) excluded; got {ids(res)}")
check(res["count"] == 4 and res["truncated"] is False, "count + not truncated")
check(res["count"] == 6 and res["truncated"] is False, "count + not truncated")
check("e4" not in ids(res), "unmatched email (no investor link) never surfaces in the panel")
# --- direction at the email level ---
@@ -114,25 +151,44 @@ def main():
check(_db.query_email_activity(conn, direction="outbound")["emails"][0]["id"] == "e1"
and len(_db.query_email_activity(conn, direction="outbound")["emails"]) == 1,
"direction=outbound returns only e1")
check(ids(_db.query_email_activity(conn, direction="inbound")) == ["e3", "e2", "e6"],
check(ids(_db.query_email_activity(conn, direction="inbound")) == ["e8", "e7", "e3", "e2", "e6"],
"direction=inbound excludes the outbound e1 (and unmatched e4)")
# --- mailbox roll-up + per-account filter ---
check(set(e2["mailboxes"]) == {"grant@ten31.xyz", "jonathan@ten31.xyz"}, "e2 seen by both mailboxes")
check(ids(_db.query_email_activity(conn, account_id="acc-jon")) == ["e3", "e2"],
check(ids(_db.query_email_activity(conn, account_id="acc-jon")) == ["e8", "e3", "e2"],
"account_id=acc-jon returns only emails that mailbox saw")
# --- investor filter: direct link and via-contact link ---
check(set(ids(_db.query_email_activity(conn, investor_id="inv-harbor"))) == {"e2", "e1"},
"investor_id=inv-harbor -> e1,e2")
check(ids(_db.query_email_activity(conn, investor_id="inv-pacific")) == ["e3"],
"investor_id=inv-pacific resolved through fundraising_contacts -> e3")
# --- date-range filter [since, until) over sent_at ---
check(ids(_db.query_email_activity(conn, since="2026-06-07T00:00:00", until="2026-06-11T00:00:00")) == ["e7", "e3"],
"date range [06-07, 06-11) -> e7,e3 (excludes 06-11 e8 and earlier e2/e1/e6)")
check(ids(_db.query_email_activity(conn, since="2026-06-10T00:00:00")) == ["e8", "e7"],
"since=06-10 -> e8,e7 only")
# --- investor name roll-up + unmatched fallback ---
check(e1["investors"] == [{"id": "inv-harbor", "name": "Harbor & Vine"}], "e1 investor resolved to name")
# --- investor filter: typed keys + legacy bare-id back-compat ---
check(set(ids(_db.query_email_activity(conn, investor_id="fund:inv-harbor"))) == {"e2", "e1"},
"investor_id=fund:inv-harbor -> e1,e2")
check(set(ids(_db.query_email_activity(conn, investor_id="inv-harbor"))) == {"e2", "e1"},
"legacy bare id treated as fund: -> e1,e2")
check(ids(_db.query_email_activity(conn, investor_id="fund:inv-pacific")) == ["e3"],
"fund:inv-pacific resolved through fundraising_contacts -> e3")
check(ids(_db.query_email_activity(conn, investor_id="org:org-bridge")) == ["e7"],
"org:org-bridge -> e7 (org-domain match)")
check(ids(_db.query_email_activity(conn, investor_id="contact:c-solo")) == ["e8"],
"contact:c-solo -> e8 (classic-contact match)")
check(_db.query_email_activity(conn, investor_id="bogus:x")["emails"] == [],
"unknown investor_id key prefix -> match nothing (never silently unfiltered)")
# --- investor identity roll-up, typed + resolved name ---
check(e1["investors"] == [{"id": "fund:inv-harbor", "name": "Harbor & Vine"}], "e1 grid investor resolved")
e3 = next(e for e in res["emails"] if e["id"] == "e3")
check(e3["investors"] == [{"id": "inv-pacific", "name": "Pacific Capital"}], "e3 investor resolved via contact")
check("e4" not in ids(res), "e4 unmatched -> excluded from the matched-only panel")
check(e3["investors"] == [{"id": "fund:inv-pacific", "name": "Pacific Capital"}], "e3 resolved via grid contact")
e7 = next(e for e in res["emails"] if e["id"] == "e7")
check(e7["investors"] == [{"id": "org:org-bridge", "name": "Bridgewater"}],
"e7 org-domain match resolves to the org name (not a raw address)")
e8 = next(e for e in res["emails"] if e["id"] == "e8")
check(e8["investors"] == [{"id": "contact:c-solo", "name": "Nina Park"}],
"e8 classic-contact match resolves to the contact name")
# --- free-text search over subject / snippet / sender ---
check(set(ids(_db.query_email_activity(conn, search="Fund III"))) == {"e1", "e2"}, "search subject")
@@ -141,18 +197,21 @@ def main():
check(ids(_db.query_email_activity(conn, search="buy now")) == [],
"unmatched email never surfaces, even by free-text search")
# --- facets ---
# --- facets: typed entries spanning grid / org / contact matches ---
check([a["email_address"] for a in res["accounts"]] == ["grant@ten31.xyz", "jonathan@ten31.xyz"],
"accounts facet sorted")
facet_inv = {i["id"] for i in res["investors"]}
check(facet_inv == {"inv-harbor", "inv-pacific"}, "investor facet covers direct + via-contact activity")
check(facet_inv == {"fund:inv-harbor", "fund:inv-pacific", "org:org-bridge", "contact:c-solo"},
f"investor facet now mirrors the list (grid + org + contact), not just grid; got {facet_inv}")
check([i["name"] for i in res["investors"]] == sorted(i["name"] for i in res["investors"]),
"facet sorted by display name")
# --- graveyard: hidden from the picker, but its email stays visible + findable ---
check("inv-dead" not in facet_inv, "graveyard investor excluded from the facet dropdown")
check("fund:inv-dead" not in facet_inv, "graveyard investor excluded from the facet dropdown")
check("e6" in ids(res), "graveyard investor's email still shows in the unfiltered list (audit completeness)")
e6 = next(e for e in res["emails"] if e["id"] == "e6")
check(e6["investors"] == [{"id": "inv-dead", "name": "Dead Deal LP"}], "graveyard email still shows its investor chip")
check(ids(_db.query_email_activity(conn, investor_id="inv-dead")) == ["e6"],
check(e6["investors"] == [{"id": "fund:inv-dead", "name": "Dead Deal LP"}], "graveyard email still shows its investor chip")
check(ids(_db.query_email_activity(conn, investor_id="fund:inv-dead")) == ["e6"],
"explicit investor_id filter still works for a graveyard investor")
check(ids(_db.query_email_activity(conn, search="deaddeal")) == ["e6"],
"graveyard email remains findable by free-text search")
@@ -161,10 +220,27 @@ def main():
tr = _db.query_email_activity(conn, limit=2)
check(tr["count"] == 2 and tr["truncated"] is True, "limit=2 -> truncated")
# --- detail view (full body + recipients + attachments + identity) ---
d = _db.query_email_detail(conn, "e2")
check(d is not None and d["body_text"] == "Thanks for the deck — one question on the carry.",
"detail returns the full body")
check(d["direction"] == "inbound" and set(d["mailboxes"]) == {"grant@ten31.xyz", "jonathan@ten31.xyz"},
"detail direction + mailboxes")
check([(r["address"], r["kind"]) for r in d["recipients"]] ==
[("grant@ten31.xyz", "to"), ("jonathan@ten31.xyz", "cc")],
"detail recipients = to/cc only (from is excluded)")
check([a["filename"] for a in d["attachments"]] == ["term_sheet.pdf"], "detail lists attachments")
check(d["investors"] == [{"id": "fund:inv-harbor", "name": "Harbor & Vine"}], "detail resolves investor identity")
check(_db.query_email_detail(conn, "e5") is None,
"detail of a tombstoned-only email -> None (soft-delete on the sighting)")
check(_db.query_email_detail(conn, "nope") is None, "detail of a missing id -> None")
conn.close()
# --- route enforces admin server-side ---
test_route_admin_only()
# --- semantic content-search route (hydrate + soft-delete + 503) ---
test_search_route()
if FAILS:
print(f"\nFAILED ({len(FAILS)})")
@@ -174,6 +250,28 @@ def main():
print("\nALL PASS (email-activity panel)")
class FakeHandler:
def __init__(self, user, params=None):
self._user = user
self._params = params or {}
self.json = None
self.err = None
self.code = None
def get_user(self):
return self._user
def get_query_params(self):
return self._params
def send_json(self, obj):
self.json = obj
def send_error_json(self, msg, code):
self.err = msg
self.code = code
def test_route_admin_only():
try:
from email_integration import routes
@@ -181,26 +279,6 @@ def test_route_admin_only():
print(f" SKIP route admin test (routes import failed: {e})")
return
class FakeHandler:
def __init__(self, user):
self._user = user
self.json = None
self.err = None
self.code = None
def get_user(self):
return self._user
def get_query_params(self):
return {}
def send_json(self, obj):
self.json = obj
def send_error_json(self, msg, code):
self.err = msg
self.code = code
h = FakeHandler(None)
routes._h_activity(h)
check(h.code == 401 and h.json is None, "route: no user -> 401")
@@ -210,5 +288,48 @@ def test_route_admin_only():
check(h.code == 403 and h.json is None, "route: member -> 403 (admin enforced server-side)")
def test_search_route():
try:
from email_integration import routes
except Exception as e: # pragma: no cover
print(f" SKIP search route test (routes import failed: {e})")
return
# Hydration source = a fresh fully-populated in-memory DB each call (the handler
# opens + closes its own conn). Retrieval is stubbed — no Spark/Qdrant in tests.
routes._conn = make_db
routes._semantic_email_search = lambda query, top_k: [
{"score": 0.91, "text": "carry discussion\nand terms", "payload": {"source_id": "e2", "lp_name": "Harbor & Vine"}},
{"score": 0.80, "text": "gone", "payload": {"source_id": "e5", "lp_name": "Ghost"}}, # tombstoned -> drop
{"score": 0.70, "text": "n/a", "payload": {"source_id": "missing", "lp_name": "Nobody"}}, # missing -> drop
]
h = FakeHandler({"role": "admin"}, {"q": "carry"})
routes._h_search(h)
check(h.json and [r["email_id"] for r in h.json["results"]] == ["e2"],
f"content search drops tombstoned + missing, keeps live e2; got {h.json and [r['email_id'] for r in h.json['results']]}")
top = h.json["results"][0]
check(top["lp_name"] == "Harbor & Vine" and top["score"] == 0.91 and top["subject"] == "Re: Fund III update",
"hit carries lp_name + score + hydrated subject")
check("\n" not in top["excerpt"], "excerpt is newline-flattened")
# empty query short-circuits (no retrieval call)
h = FakeHandler({"role": "admin"}, {"q": ""})
routes._h_search(h)
check(h.json == {"query": "", "results": []}, "empty query -> empty results")
# retrieval failure -> clean 503 (Spark/Qdrant down)
def _boom(query, top_k):
raise RuntimeError("spark down")
routes._semantic_email_search = _boom
h = FakeHandler({"role": "admin"}, {"q": "x"})
routes._h_search(h)
check(h.code == 503, f"retrieval failure -> 503, got {h.code}")
# admin enforced
h = FakeHandler({"role": "member"}, {"q": "x"})
routes._h_search(h)
check(h.code == 403, "content search admin-enforced server-side")
if __name__ == "__main__":
main()