Remove Instructions/Feedback + lp_profiles; sync retry, purge, mobile fixes (v0.1.0:104)

Removals (net -570 lines):
- Delete the Instructions and Feedback (feature_requests) pages + backend.
- Retire lp_profiles + investor_type across server, ingest, and seeds; migration
  0008 drops both empty tables (a sanctioned one-off exception to
  never-hard-delete). 0001's lp_profiles ALTER is removed so a fresh DB doesn't
  break the migration chain (live DBs already applied it).

Fixes:
- Email sync: a transient timeout no longer terminally parks a mailbox; the
  scheduler retries 'retrying' each cycle and re-includes errored accounts on an
  hourly backoff, so stuck mailboxes self-heal.
- Mobile Contacts: page through the full directory (server caps 500/page) -- one
  fetch silently truncated at 720, hiding people from the list and from search.
- Mobile email review: clock icon to set a reminder inline; approval cards show
  date/time.

New:
- Admin-only purge of soft-deleted rows (Settings -> Admin; type-to-confirm,
  refuses any row still linked to live data).

Tests: 45/45 (adds test_sync_ready + test_purge_soft_deleted). Reviewer pass
applied (NULL reminders.contact_id on contact purge). Bumped to v0.1.0:104.
This commit is contained in:
Keysat
2026-06-20 20:06:11 -05:00
parent 985cba3c81
commit 1564c087bf
21 changed files with 629 additions and 694 deletions
+11 -1
View File
@@ -57,10 +57,20 @@ def _json(v) -> str:
# ------------------------------------------------------------------ email_accounts
def list_sync_ready_accounts(conn: sqlite3.Connection) -> list[sqlite3.Row]:
# Ready = healthy ('pending'/'active') + transient-failing ('retrying', retried every
# cycle for fast recovery) + 'error' accounts whose last attempt was over an hour ago.
# The hour-backoff on 'error' means a terminal failure (auth/permanent) self-heals once
# the operator fixes it WITHOUT hammering Google, and un-sticks any mailbox parked by the
# pre-v0.1.0:104 bug where one timeout dark-listed it forever. (last_synced_at is stamped
# on every attempt, success or fail, so it doubles as the last-attempt clock here.)
cur = conn.cursor()
cur.execute(
"SELECT * FROM email_accounts "
"WHERE sync_enabled = 1 AND sync_status IN ('pending','active') "
"WHERE sync_enabled = 1 AND ("
" sync_status IN ('pending','active','retrying') "
" OR (sync_status = 'error' AND (last_synced_at IS NULL "
" OR last_synced_at < datetime('now','-1 hour')))"
") "
"ORDER BY last_synced_at IS NOT NULL, last_synced_at"
)
return cur.fetchall()
+10
View File
@@ -23,6 +23,7 @@ import logging
import sqlite3
import traceback
from typing import Optional
from urllib.error import URLError
from . import attachments as _attach
from . import config as _cfg
@@ -112,6 +113,15 @@ def sync_account(conn_factory, credential_provider, account,
error_str = "history expired; fallback to date backfill"
status = "partial"
_fallback_date_backfill(conn_factory, client, account, index, run_stats)
except (_errors.RateLimitError, _errors.TransientError, URLError, TimeoutError) as e:
# A network / 5xx / rate-limit error that outlived the in-pass retry loop.
# This is TRANSIENT, not terminal: park it as 'retrying' (which the scheduler
# still picks up every cycle) instead of 'error' (which it excludes). Fixes the
# v<=0.1.0:103 bug where a single timeout dark-listed a mailbox until a manual
# kick. Terminal causes (auth, permanent, unexpected) still fall through to 'error'.
error_str = f"transient: {type(e).__name__}: {e}"
status = "retrying"
log.warning("transient error during sync of %s: %s", email_addr, e)
except Exception as e:
error_str = f"unexpected: {type(e).__name__}: {e}"
status = "error"
@@ -0,0 +1,72 @@
#!/usr/bin/env python3
"""Regression test for list_sync_ready_accounts (v0.1.0:104).
Guards the Bug-B fix: a transient network timeout used to flip a mailbox to terminal
sync_status='error', which the old `IN ('pending','active')` filter excluded forever —
so a single blip dark-listed a mailbox until a manual kick. The new filter:
* always includes 'pending' / 'active' / 'retrying' (the transient-retry state), and
* re-includes 'error' accounts whose last attempt was over an hour ago (gentle backoff,
so a fixed credential self-heals and the pre-fix stuck mailboxes recover on deploy).
Synthetic data only (guardrail #9).
Run: cd backend && python3 email_integration/test_sync_ready.py
"""
import os
import sqlite3
import sys
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from email_integration import db as edb # noqa: E402
FAILS = []
def check(cond, msg):
print((" PASS " if cond else " FAIL ") + msg)
if not cond:
FAILS.append(msg)
def make_account(conn, email, status, *, enabled=1, last_synced_sql="NULL"):
aid = edb.upsert_account(conn, user_id="u-" + email,
email_address=email, auth_method="dwd")
conn.execute(
f"UPDATE email_accounts SET sync_status=?, sync_enabled=?, "
f"last_synced_at={last_synced_sql} WHERE id=?",
(status, enabled, aid),
)
conn.commit()
return aid
def main():
conn = sqlite3.connect(":memory:")
conn.row_factory = sqlite3.Row
edb.apply_migrations(conn.cursor())
make_account(conn, "active@t.xyz", "active", last_synced_sql="datetime('now','-5 minutes')")
make_account(conn, "retrying@t.xyz", "retrying", last_synced_sql="datetime('now','-5 minutes')")
make_account(conn, "pending@t.xyz", "pending", last_synced_sql="NULL")
make_account(conn, "error_stale@t.xyz", "error", last_synced_sql="datetime('now','-2 hours')")
make_account(conn, "error_recent@t.xyz", "error", last_synced_sql="datetime('now','-5 minutes')")
make_account(conn, "error_neversync@t.xyz", "error", last_synced_sql="NULL")
make_account(conn, "disabled@t.xyz", "active", enabled=0, last_synced_sql="datetime('now','-5 minutes')")
ready = {r["email_address"] for r in edb.list_sync_ready_accounts(conn)}
check("active@t.xyz" in ready, "healthy 'active' is ready")
check("retrying@t.xyz" in ready, "transient 'retrying' is ready (fast retry)")
check("pending@t.xyz" in ready, "'pending' is ready")
check("error_stale@t.xyz" in ready, "'error' last attempted >1h ago is ready (backoff elapsed → recovers stuck mailbox)")
check("error_neversync@t.xyz" in ready, "'error' never synced (NULL last attempt) is ready")
check("error_recent@t.xyz" not in ready, "'error' attempted <1h ago is NOT ready (gentle backoff, no hammering)")
check("disabled@t.xyz" not in ready, "sync_enabled=0 is never ready")
print()
if FAILS:
print(f"{len(FAILS)} FAILED")
sys.exit(1)
print("ALL PASS (sync ready filter)")
if __name__ == "__main__":
main()
+1 -8
View File
@@ -4,7 +4,7 @@ Maps each CRM record type to one or more chunks per docs/EMBEDDINGS.md:
* one chunk per communications row (doc_type = the comm type)
* one chunk per MATCHED email (doc_type = email; body only when matched)
* one chunk per fundraising_investors notes LINE (the outreach log; split per line)
* one chunk each for free-text fields: contacts.notes, lp_profiles.notes,
* one chunk each for free-text fields: contacts.notes,
opportunities (description + next_step), organizations.description
Each chunk carries a canonical `lp_id` (resolved via entity_links) and a `date_ts`
@@ -104,13 +104,6 @@ def build_chunks(conn):
chunks.append(_mk(f"contacts.notes:{r['id']}", lp, lp_name, person,
"contact_note", to_epoch(r["updated_at"]), r["notes"], "contacts", r["id"]))
# lp_profiles.notes
for r in conn.execute("""SELECT lp.id, lp.contact_id, lp.notes, lp.updated_at
FROM lp_profiles lp WHERE lp.notes IS NOT NULL AND lp.notes <> '' AND lp.deleted_at IS NULL"""):
lp, lp_name, person = _contact_lp(r["contact_id"], person_canon, org_canon, name, contact_org)
chunks.append(_mk(f"lp_profiles.notes:{r['id']}", lp, lp_name, person,
"lp_note", to_epoch(r["updated_at"]), r["notes"], "lp_profiles", r["id"]))
# opportunities (description + next_step)
for r in conn.execute("""SELECT id, contact_id, name, description, next_step, updated_at
FROM opportunities WHERE deleted_at IS NULL"""):
+1 -8
View File
@@ -8,7 +8,6 @@ layer created by migration 0001:
fundraising_investors ─┴─► canonical_entities (entity_kind = lp | organization)
contacts ─┐
fundraising_contacts ─┴─► canonical_entities (entity_kind = person)
lp_profiles ───► linked to its contact's person entity
Every source row is recorded in `entity_links` so any name variant resolves to
one canonical id. This is the DETERMINISTIC tier — it merges only what we can
@@ -184,7 +183,7 @@ def resolve_people(conn, org_canon_by_orgid, org_canon_by_fundinv, merge_map=Non
people — each is matched to a contact-person and recorded only as a member_of
edge to its investor entity (the grid's 'Contacts' column says who belongs to
which investor). This is what stops the double-count.
Returns contact_id -> person canonical id (for lp_profiles)."""
Returns contact_id -> person canonical id."""
merge_map = merge_map or {}
contact_to_person = {}
person_meta = {}
@@ -245,12 +244,6 @@ def resolve_people(conn, org_canon_by_orgid, org_canon_by_fundinv, merge_map=Non
_link(conn, cid, "fundraising_contacts", r["id"], email or name_norm, mk, 0.95 if mk == "grid_link" else 0.9)
_member_of(conn, cid, inv_canon)
# lp_profiles -> the person entity of its contact
for r in conn.execute("SELECT id, contact_id FROM lp_profiles WHERE deleted_at IS NULL"):
cid = contact_to_person.get(r["contact_id"])
if cid:
_link(conn, cid, "lp_profiles", r["id"], r["contact_id"], "contact_fk", 1.0)
return person_meta
+2 -2
View File
@@ -34,7 +34,7 @@ import entity_resolution as er
import qdrant_io
_CHANGE_TABLES = [("communications", "communications"), ("contacts", "contacts"),
("lp_profiles", "lp_profiles"), ("opportunities", "opportunities"),
("opportunities", "opportunities"),
("organizations", "organizations"), ("fundraising_investors", "fundraising_investors")]
@@ -63,7 +63,7 @@ def _state_set(conn, key, value):
def _deleted_source_ids(conn, since):
"""CRM records soft-deleted since the watermark — their chunks get pruned."""
ids = set()
for tbl in ("contacts", "organizations", "opportunities", "communications", "lp_profiles"):
for tbl in ("contacts", "organizations", "opportunities", "communications"):
try:
for r in conn.execute(f"SELECT id FROM {tbl} WHERE deleted_at IS NOT NULL AND deleted_at > ?", (since,)):
ids.add(r["id"])
+9 -12
View File
@@ -12,7 +12,7 @@ Asserts the SAFE fix:
3. a grid contact that can't be PROVABLY matched mints NOTHING (no duplicate
person, no cross-firm name guess) — the count stays correct,
4. targeted cleanup soft-deletes a stale grid-only "twin" (person with no
contacts link) and a superseded 'lp'/'organization' row, with no enrichment,
contacts link), with no enrichment,
5. cleanup PRESERVES a grid-only person that carries enrichment (guardrail #3),
6. a re-emitted id is UN-tombstoned (no permanent burial),
7. re-running is idempotent.
@@ -58,10 +58,9 @@ CREATE TABLE contacts (
CREATE TABLE organizations (id TEXT PRIMARY KEY, name TEXT, email TEXT);
CREATE TABLE fundraising_investors (id TEXT PRIMARY KEY, investor_name TEXT);
CREATE TABLE fundraising_contacts (id TEXT PRIMARY KEY, full_name TEXT, email TEXT, investor_id TEXT, contact_id TEXT);
CREATE TABLE lp_profiles (id TEXT PRIMARY KEY, contact_id TEXT, deleted_at TEXT);
"""
SEEDED = ("per_TWIN", "per_ENR", "lp_OLD")
SEEDED = ("per_TWIN", "per_ENR")
def seed(db):
@@ -94,16 +93,14 @@ def seed(db):
"('per_ENR','person','Enriched Orphan','entity_resolution','warm')")
c.execute("INSERT INTO entity_links (id, canonical_id, source_model, source_id, match_value, match_kind, confidence, created_at) "
"VALUES ('l_enr','per_ENR','fundraising_contacts','gy','enr','name_org',0.8,'t')")
# Superseded pre-:48 kind -> prune
c.execute("INSERT INTO canonical_entities (id, entity_kind, display_name, source) VALUES "
"('lp_OLD','lp','Old LP Row','entity_resolution')")
c.commit()
c.close()
def resolved_persons(db):
c = sqlite3.connect(db)
q = "SELECT COUNT(*) FROM canonical_entities WHERE entity_kind='person' AND deleted_at IS NULL AND id NOT IN (?,?,?)"
ph = ",".join("?" * len(SEEDED))
q = f"SELECT COUNT(*) FROM canonical_entities WHERE entity_kind='person' AND deleted_at IS NULL AND id NOT IN ({ph})"
n = c.execute(q, SEEDED).fetchone()[0]
c.close()
return n
@@ -127,10 +124,11 @@ def grid_match_kinds(db):
def minted_from_grid(db):
"""Persons minted directly from a grid row (the bug). Should be 0 after the fix."""
c = sqlite3.connect(db)
n = c.execute("""SELECT COUNT(DISTINCT l.canonical_id) FROM entity_links l
ph = ",".join("?" * len(SEEDED))
n = c.execute(f"""SELECT COUNT(DISTINCT l.canonical_id) FROM entity_links l
JOIN canonical_entities ce ON ce.id=l.canonical_id AND ce.deleted_at IS NULL
WHERE l.source_model='fundraising_contacts' AND l.match_kind IN ('name_org','exact_email')
AND l.canonical_id NOT IN (?,?,?)""", SEEDED).fetchone()[0]
AND l.canonical_id NOT IN ({ph})""", SEEDED).fetchone()[0]
c.close()
return n
@@ -162,12 +160,11 @@ def main():
check(mk.get("grid_assoc", 0) == 2, f"two grid contacts matched back via grid_assoc (got {mk.get('grid_assoc',0)})")
check(mk.get("grid_link", 0) == 1, f"one grid contact linked via explicit contact_id (grid_link==1, got {mk.get('grid_link',0)})")
# Targeted cleanup: stale grid-only twin + superseded 'lp' row tombstoned...
# Targeted cleanup: stale grid-only twin tombstoned...
check(deleted_at(db, "per_TWIN") is not None, "stale grid-only twin 'per_TWIN' tombstoned")
check(deleted_at(db, "lp_OLD") is not None, "superseded 'lp' row 'lp_OLD' tombstoned")
# ...enriched grid-only person preserved.
check(deleted_at(db, "per_ENR") is None, "enriched grid-only person 'per_ENR' PRESERVED (has segment)")
check(counts1.get("pruned_stale", 0) == 2, f"exactly 2 stale rows pruned (got {counts1.get('pruned_stale')})")
check(counts1.get("pruned_stale", 0) == 1, f"exactly 1 stale row pruned (got {counts1.get('pruned_stale')})")
# Un-tombstone: soft-delete a real contact-person, then re-run -> it comes back.
alice = er._eid("per", "e|alice@x.com")
@@ -113,4 +113,8 @@ ALTER TABLE contacts ADD COLUMN deleted_at TEXT;
ALTER TABLE organizations ADD COLUMN deleted_at TEXT;
ALTER TABLE opportunities ADD COLUMN deleted_at TEXT;
ALTER TABLE communications ADD COLUMN deleted_at TEXT;
ALTER TABLE lp_profiles ADD COLUMN deleted_at TEXT;
-- lp_profiles ALTER removed (v0.1.0:104): the lp_profiles table is dropped in
-- 0008_drop_retired_tables.sql and is no longer created by init_db(), so this
-- ALTER would fail "no such table" on a fresh install. Live DBs already applied
-- this migration (with the original ALTER) before lp_profiles was dropped, so
-- removing the line here only affects fresh DBs — same end state either way.
@@ -0,0 +1,42 @@
-- 0008_drop_retired_tables.down.sql (manual rollback only — never auto-applied)
--
-- Recreates the two dropped tables as EMPTY shells, matching the schema that existed
-- immediately before 0008 (lp_profiles includes the deleted_at column that migration
-- 0001 had added). Data is not restored — both tables were empty when dropped.
CREATE TABLE IF NOT EXISTS lp_profiles (
id TEXT PRIMARY KEY,
contact_id TEXT NOT NULL UNIQUE REFERENCES contacts(id) ON DELETE CASCADE,
commitment_amount REAL DEFAULT 0,
funded_amount REAL DEFAULT 0,
commitment_date TEXT,
fund_name TEXT,
investor_type TEXT,
accredited INTEGER DEFAULT 0,
legal_docs_signed INTEGER DEFAULT 0,
signed_date TEXT,
wire_received INTEGER DEFAULT 0,
wire_date TEXT,
k1_sent INTEGER DEFAULT 0,
preferred_communication TEXT DEFAULT 'email',
notes TEXT,
created_at TEXT DEFAULT (datetime('now')),
updated_at TEXT DEFAULT (datetime('now')),
deleted_at TEXT
);
CREATE INDEX IF NOT EXISTS idx_lp_profiles_contact ON lp_profiles(contact_id);
CREATE TABLE IF NOT EXISTS feature_requests (
id TEXT PRIMARY KEY,
title TEXT NOT NULL,
description TEXT,
page TEXT,
category TEXT DEFAULT 'general',
priority TEXT DEFAULT 'medium',
status TEXT DEFAULT 'new',
requested_by TEXT,
requested_by_user_id TEXT REFERENCES users(id),
created_at TEXT DEFAULT (datetime('now')),
updated_at TEXT DEFAULT (datetime('now'))
);
CREATE INDEX IF NOT EXISTS idx_feature_requests_status ON feature_requests(status);
CREATE INDEX IF NOT EXISTS idx_feature_requests_created_at ON feature_requests(created_at);
@@ -0,0 +1,15 @@
-- 0008_drop_retired_tables.sql (v0.1.0:104)
--
-- ONE-OFF DESTRUCTIVE EXCEPTION to the never-hard-delete rule, explicitly approved.
-- Both tables are EMPTY and fully removed from the application code:
-- * lp_profiles — the legacy single-fund LP model, retired v0.1.0:78; the
-- fundraising_* grid is the canonical commitment record now.
-- * feature_requests — backed the in-app Feedback page, which was removed.
--
-- The never-hard-delete policy STILL STANDS for all real CRM and thesis data — this
-- is a deliberate, documented exception for two empty, retired tables so they don't
-- linger as dead schema. init_db() no longer creates either table, and migration
-- 0001's lp_profiles ALTER was removed, so a fresh DB never creates them and this
-- DROP is a harmless no-op there; on the live box it removes the existing empties.
DROP TABLE IF EXISTS lp_profiles;
DROP TABLE IF EXISTS feature_requests;
+4 -17
View File
@@ -11,8 +11,8 @@ What it builds (into a SEPARATE dev DB, never crm.db):
core migration (backend/migrations/), so the canonical/interaction/graph
tables exist.
* A classic-model dataset: organizations, contacts (investors + prospects),
opportunities across pipeline stages, communications with entity-rich prose
notes, and lp_profiles.
opportunities across pipeline stages, and communications with entity-rich
prose notes.
* A fundraising grid (fundraising_state.grid_json) populated via the real
sync_fundraising_relational() code path, so the normalized mirror + the
grid->classic bridge behave exactly as in production.
@@ -179,7 +179,7 @@ def main():
f"Prospect sourced via {random.choice(['X DM', 'warm intro', 'podcast'])}.", uid, now()))
contacts.append((cid, first, last, org_name, "prospect"))
# ── opportunities + lp_profiles + communications ──
# ── opportunities + communications ──
stages = server.PIPELINE_STAGES
for idx, (cid, first, last, org_name, ctype) in enumerate(contacts):
person = f"{first} {last}"
@@ -199,19 +199,6 @@ def main():
random.choice(["Send deck", "Schedule call", "Await IC", "Send subdocs"]),
uid, random.choice(["low", "medium", "high"]), now()))
# lp_profile for ~closed investors
if ctype == "investor" and idx % 2 == 0:
amt = random.choice(AMOUNTS)
conn.execute(
"INSERT INTO lp_profiles (id, contact_id, commitment_amount, funded_amount, commitment_date, "
"fund_name, investor_type, accredited, legal_docs_signed, wire_received, k1_sent, notes, updated_at) "
"VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)",
(gen(), cid, amt, amt if idx % 4 == 0 else 0, past(120),
random.choice(list(FUND_LABELS.values())),
random.choice(["family_office", "institutional", "endowment", "individual"]),
1, 1 if idx % 3 else 0, 1 if idx % 4 == 0 else 0, 0,
f"Closed LP. Accreditation on file. Primary contact {person}.", now()))
# 2-4 communications each, entity-rich prose
for k in range(random.randint(2, 4)):
ctype_comm, subj, body = random.choice(COMM_TEMPLATES)
@@ -275,7 +262,7 @@ def main():
print(f"\nSynthetic dev DB written to: {db}")
print(" Classic model:")
for t in ("organizations", "contacts", "opportunities", "communications", "lp_profiles"):
for t in ("organizations", "contacts", "opportunities", "communications"):
print(f" {t:<24} {count(t)}")
print(" Fundraising grid (after real sync):")
for t in ("fundraising_investors", "fundraising_contacts", "fundraising_funds",
+95 -137
View File
@@ -215,26 +215,6 @@ def init_db():
updated_at TEXT DEFAULT (datetime('now'))
);
CREATE TABLE IF NOT EXISTS lp_profiles (
id TEXT PRIMARY KEY,
contact_id TEXT NOT NULL UNIQUE REFERENCES contacts(id) ON DELETE CASCADE,
commitment_amount REAL DEFAULT 0,
funded_amount REAL DEFAULT 0,
commitment_date TEXT,
fund_name TEXT,
investor_type TEXT,
accredited INTEGER DEFAULT 0,
legal_docs_signed INTEGER DEFAULT 0,
signed_date TEXT,
wire_received INTEGER DEFAULT 0,
wire_date TEXT,
k1_sent INTEGER DEFAULT 0,
preferred_communication TEXT DEFAULT 'email',
notes TEXT,
created_at TEXT DEFAULT (datetime('now')),
updated_at TEXT DEFAULT (datetime('now'))
);
CREATE TABLE IF NOT EXISTS custom_fields (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
@@ -273,20 +253,6 @@ def init_db():
created_at TEXT DEFAULT (datetime('now'))
);
CREATE TABLE IF NOT EXISTS feature_requests (
id TEXT PRIMARY KEY,
title TEXT NOT NULL,
description TEXT,
page TEXT,
category TEXT DEFAULT 'general',
priority TEXT DEFAULT 'medium',
status TEXT DEFAULT 'new',
requested_by TEXT,
requested_by_user_id TEXT REFERENCES users(id),
created_at TEXT DEFAULT (datetime('now')),
updated_at TEXT DEFAULT (datetime('now'))
);
CREATE TABLE IF NOT EXISTS fundraising_state (
id TEXT PRIMARY KEY,
grid_json TEXT NOT NULL,
@@ -422,9 +388,6 @@ def init_db():
CREATE INDEX IF NOT EXISTS idx_communications_contact ON communications(contact_id);
CREATE INDEX IF NOT EXISTS idx_communications_date ON communications(communication_date);
CREATE INDEX IF NOT EXISTS idx_audit_entity ON audit_log(entity_type, entity_id);
CREATE INDEX IF NOT EXISTS idx_lp_profiles_contact ON lp_profiles(contact_id);
CREATE INDEX IF NOT EXISTS idx_feature_requests_status ON feature_requests(status);
CREATE INDEX IF NOT EXISTS idx_feature_requests_created_at ON feature_requests(created_at);
CREATE INDEX IF NOT EXISTS idx_fr_investor_name ON fundraising_investors(investor_name);
CREATE INDEX IF NOT EXISTS idx_fr_investor_lead ON fundraising_investors(lead);
CREATE INDEX IF NOT EXISTS idx_fr_contacts_investor ON fundraising_contacts(investor_id);
@@ -2381,10 +2344,6 @@ class CRMHandler(BaseHTTPRequestHandler):
if path == '/api/export/contacts':
return self.handle_export_contacts(user, params)
# Feature requests
if path == '/api/feature-requests':
return self.handle_list_feature_requests(user, params)
# Fundraising grid state
if path == '/api/fundraising/state':
return self.handle_get_fundraising_state(user)
@@ -2398,6 +2357,8 @@ class CRMHandler(BaseHTTPRequestHandler):
return self.handle_get_backup_policy(user)
if path == '/api/admin/digest/policy':
return self.handle_get_digest_policy(user)
if path == '/api/admin/soft-deleted':
return self.handle_list_soft_deleted(user)
if path == '/api/fundraising/relational-summary':
return self.handle_get_fundraising_relational_summary(user)
if path == '/api/fundraising/automations':
@@ -2503,8 +2464,6 @@ class CRMHandler(BaseHTTPRequestHandler):
return self.handle_create_communication(user, body)
if path == '/api/import/csv':
return self.handle_import_csv(user, body)
if path == '/api/feature-requests':
return self.handle_create_feature_request(user, body)
if path == '/api/fundraising/log-communication':
return self.handle_log_fundraising_communication(user, body)
if path == '/api/fundraising/update-row':
@@ -2525,6 +2484,8 @@ class CRMHandler(BaseHTTPRequestHandler):
return self.handle_admin_create_user(user, body)
if path == '/api/admin/reset-all-data':
return self.handle_admin_reset_all_data(user, body)
if path == '/api/admin/soft-deleted/purge':
return self.handle_purge_soft_deleted(user, body)
if path == '/api/admin/digest/test-email':
return self.handle_admin_send_test_email(user, body)
if path == '/api/admin/digest/send-now':
@@ -2623,9 +2584,6 @@ class CRMHandler(BaseHTTPRequestHandler):
if re.match(r'^/api/opportunities/[^/]+/stage$', path):
opp_id = path.split('/')[-2]
return self.handle_update_stage(user, opp_id, body)
if re.match(r'^/api/feature-requests/[^/]+$', path):
fr_id = path.split('/')[-1]
return self.handle_update_feature_request(user, fr_id, body)
if re.match(r'^/api/reminders/[^/]+$', path):
return self.handle_update_reminder(user, path.split('/')[-1], body)
if re.match(r'^/api/admin/users/[^/]+$', path):
@@ -2963,12 +2921,11 @@ class CRMHandler(BaseHTTPRequestHandler):
_sync_contact_to_fundraising_state(conn, row_to_dict(existing), actor_user_id=user['user_id'], remove=True)
# Soft-delete (guardrail #3 — never hard-delete): mark deleted_at and
# cascade to the contact's opportunities, communications, and lp_profile.
# cascade to the contact's opportunities and communications.
_ts = now()
conn.execute("UPDATE contacts SET deleted_at = ?, updated_at = ? WHERE id = ?", (_ts, _ts, contact_id))
conn.execute("UPDATE opportunities SET deleted_at = ? WHERE contact_id = ? AND deleted_at IS NULL", (_ts, contact_id))
conn.execute("UPDATE communications SET deleted_at = ? WHERE contact_id = ? AND deleted_at IS NULL", (_ts, contact_id))
conn.execute("UPDATE lp_profiles SET deleted_at = ? WHERE contact_id = ? AND deleted_at IS NULL", (_ts, contact_id))
log_audit(conn, user['user_id'], 'contact', contact_id, 'delete')
conn.commit()
conn.close()
@@ -4821,6 +4778,96 @@ class CRMHandler(BaseHTTPRequestHandler):
finally:
conn.close()
# ─── Soft-deleted purge (admin maintenance) ──────────────────────────────────
# Lists soft-deleted rows and HARD-deletes them — a deliberate, admin-only, type-to-confirm
# exception to the never-hard-delete rule, for clearing out dummy/test data. A purge can only
# ever touch a soft-deleted row, and refuses any contact/org whose delete would CASCADE or
# SET NULL onto LIVE data, so it can never remove or mutate a live record.
_PURGE_TABLES = ('contacts', 'organizations', 'opportunities', 'communications')
def handle_list_soft_deleted(self, user):
if not require_admin(user):
return self.send_error_json("Admin required", 403)
conn = get_db()
try:
groups = {}
groups['contacts'] = [
{"id": r["id"],
"label": (f"{r['first_name'] or ''} {r['last_name'] or ''}".strip() or r["email"] or r["id"]),
"deleted_at": r["deleted_at"]}
for r in conn.execute(
"SELECT id, first_name, last_name, email, deleted_at FROM contacts "
"WHERE deleted_at IS NOT NULL ORDER BY deleted_at DESC").fetchall()]
groups['organizations'] = [
{"id": r["id"], "label": (r["name"] or r["id"]), "deleted_at": r["deleted_at"]}
for r in conn.execute(
"SELECT id, name, deleted_at FROM organizations "
"WHERE deleted_at IS NOT NULL ORDER BY deleted_at DESC").fetchall()]
groups['opportunities'] = [
{"id": r["id"], "label": (r["name"] or r["id"]), "deleted_at": r["deleted_at"]}
for r in conn.execute(
"SELECT id, name, deleted_at FROM opportunities "
"WHERE deleted_at IS NOT NULL ORDER BY deleted_at DESC").fetchall()]
groups['communications'] = [
{"id": r["id"],
"label": ((r["subject"] or (r["type"] or "note"))
+ (f" · {(r['communication_date'] or '')[:10]}" if r["communication_date"] else "")),
"deleted_at": r["deleted_at"]}
for r in conn.execute(
"SELECT id, type, subject, communication_date, deleted_at FROM communications "
"WHERE deleted_at IS NOT NULL ORDER BY deleted_at DESC").fetchall()]
return self.send_json({"groups": groups, "total": sum(len(v) for v in groups.values())})
finally:
conn.close()
def handle_purge_soft_deleted(self, user, body):
if not require_admin(user):
return self.send_error_json("Admin required", 403)
body = body or {}
table = str(body.get('table') or '').strip()
row_id = str(body.get('id') or '').strip()
if table not in self._PURGE_TABLES: # validated -> safe to interpolate below
return self.send_error_json("Unknown table", 400)
if not row_id:
return self.send_error_json("id is required", 400)
conn = get_db()
try:
row = conn.execute(f"SELECT id, deleted_at FROM {table} WHERE id = ?", (row_id,)).fetchone()
if not row:
return self.send_error_json("Not found", 404)
if not row["deleted_at"]:
return self.send_error_json("Only soft-deleted rows can be purged", 400)
# A contacts/organizations delete cascades / SET NULLs onto children. Refuse if any LIVE
# child exists so a purge can never touch live data (a soft-deleted parent's children were
# soft-deleted with it, so normally there are none).
if table == 'contacts':
live = conn.execute(
"SELECT (SELECT COUNT(*) FROM opportunities WHERE contact_id=? AND deleted_at IS NULL) + "
"(SELECT COUNT(*) FROM communications WHERE contact_id=? AND deleted_at IS NULL) AS n",
(row_id, row_id)).fetchone()["n"]
if live:
return self.send_error_json("This contact still has live communications or opportunities — cannot purge", 409)
# Drop the optional logical FKs that have no ON DELETE (so a purged contact leaves no
# dangling reference): the derived grid link (fundraising_contacts.contact_id, migration
# 0004) and any reminder's contact_id (migration 0006 — the reminder's real link is
# investor_id, which is unaffected). Both are bare TEXT columns, not declared FKs.
conn.execute("UPDATE fundraising_contacts SET contact_id = NULL WHERE contact_id = ?", (row_id,))
conn.execute("UPDATE reminders SET contact_id = NULL WHERE contact_id = ?", (row_id,))
elif table == 'organizations':
live = conn.execute(
"SELECT (SELECT COUNT(*) FROM contacts WHERE organization_id=? AND deleted_at IS NULL) + "
"(SELECT COUNT(*) FROM opportunities WHERE organization_id=? AND deleted_at IS NULL) AS n",
(row_id, row_id)).fetchone()["n"]
if live:
return self.send_error_json("This organization is still linked to live contacts or opportunities — cannot purge", 409)
# CASCADE removes the (soft-deleted) children for contacts; the rest are leaves.
conn.execute(f"DELETE FROM {table} WHERE id = ?", (row_id,))
log_audit(conn, user['user_id'], table, row_id, 'purge', {"table": table})
conn.commit()
return self.send_json({"data": {"purged": True, "table": table, "id": row_id}})
finally:
conn.close()
def handle_decide_activity_proposal(self, user, proposal_id, decision, body):
if not require_admin(user):
return self.send_error_json("Admin required", 403)
@@ -5393,10 +5440,8 @@ class CRMHandler(BaseHTTPRequestHandler):
conn.execute("DELETE FROM communications")
conn.execute("DELETE FROM opportunities")
conn.execute("DELETE FROM lp_profiles")
conn.execute("DELETE FROM custom_field_values")
conn.execute("DELETE FROM custom_fields")
conn.execute("DELETE FROM feature_requests")
conn.execute("DELETE FROM contacts")
conn.execute("DELETE FROM organizations")
@@ -5882,93 +5927,6 @@ class CRMHandler(BaseHTTPRequestHandler):
}
})
# ═══════════════════════════════════════════════════════════════════════════
# FEATURE REQUESTS
# ═══════════════════════════════════════════════════════════════════════════
def handle_list_feature_requests(self, user, params):
conn = get_db()
query = """
SELECT fr.*, u.full_name as requested_by_name
FROM feature_requests fr
LEFT JOIN users u ON fr.requested_by_user_id = u.id
WHERE 1=1
"""
args = []
if params.get('status'):
query += " AND fr.status = ?"
args.append(params['status'])
if params.get('search'):
search = f"%{params['search']}%"
query += " AND (fr.title LIKE ? OR fr.description LIKE ? OR fr.requested_by LIKE ?)"
args.extend([search, search, search])
query += " ORDER BY fr.created_at DESC"
rows = rows_to_list(conn.execute(query, args).fetchall())
conn.close()
return self.send_json({"data": rows, "total": len(rows)})
def handle_create_feature_request(self, user, body):
title = str(body.get('title', '')).strip()
if not title:
return self.send_error_json("title is required")
req_id = generate_id()
requested_by = str(body.get('requested_by') or user.get('username') or '').strip()
conn = get_db()
conn.execute("""
INSERT INTO feature_requests (
id, title, description, page, category, priority, status,
requested_by, requested_by_user_id
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
""", (
req_id,
title,
body.get('description'),
body.get('page'),
body.get('category', 'general'),
body.get('priority', 'medium'),
body.get('status', 'new'),
requested_by,
user['user_id']
))
log_audit(conn, user['user_id'], 'feature_request', req_id, 'create', {"title": title})
conn.commit()
row = row_to_dict(conn.execute("SELECT * FROM feature_requests WHERE id = ?", (req_id,)).fetchone())
conn.close()
return self.send_json({"data": row}, 201)
def handle_update_feature_request(self, user, req_id, body):
conn = get_db()
existing = conn.execute("SELECT * FROM feature_requests WHERE id = ?", (req_id,)).fetchone()
if not existing:
conn.close()
return self.send_error_json("Feature request not found", 404)
updatable = ['title', 'description', 'page', 'category', 'priority', 'status', 'requested_by']
sets = []
args = []
for field in updatable:
if field in body:
sets.append(f"{field} = ?")
args.append(body[field])
if not sets:
conn.close()
return self.send_error_json("No fields to update")
sets.append("updated_at = ?")
args.append(now())
args.append(req_id)
conn.execute(f"UPDATE feature_requests SET {', '.join(sets)} WHERE id = ?", args)
log_audit(conn, user['user_id'], 'feature_request', req_id, 'update', body)
conn.commit()
row = row_to_dict(conn.execute("SELECT * FROM feature_requests WHERE id = ?", (req_id,)).fetchone())
conn.close()
return self.send_json({"data": row})
# ═══════════════════════════════════════════════════════════════════════════
# FUNDRAISING STATE (AIRTABLE-LIKE GRID)
# ═══════════════════════════════════════════════════════════════════════════
+157
View File
@@ -0,0 +1,157 @@
#!/usr/bin/env python3
"""Test for the admin soft-deleted purge (v0.1.0:104).
The purge is a deliberate, admin-only, type-to-confirm exception to never-hard-delete, for
clearing dummy/test data. It must be SAFE: only ever touch a soft-deleted row, and never
remove or mutate LIVE data via a cascade/SET-NULL. This boots the real server, seeds live +
soft-deleted graphs, and drives /api/admin/soft-deleted[/purge] over HTTP. Synthetic only.
Run: cd backend && python3 test_purge_soft_deleted.py
"""
import http.client
import json
import os
import sqlite3
import sys
import tempfile
import threading
from http.server import ThreadingHTTPServer
_DATA = tempfile.mkdtemp()
os.environ["CRM_DATA_DIR"] = _DATA
os.environ["CRM_DB_PATH"] = os.path.join(_DATA, "crm.db")
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import server # noqa: E402
FAILS = []
DEL = "2026-06-01T00:00:00"
def check(cond, msg):
print((" PASS " if cond else " FAIL ") + msg)
if not cond:
FAILS.append(msg)
class _Quiet(server.CRMHandler):
def log_message(self, *a):
pass
def _post(port, path, token, payload):
conn = http.client.HTTPConnection("127.0.0.1", port, timeout=10)
conn.request("POST", path, body=json.dumps(payload),
headers={"Authorization": "Bearer " + token, "Content-Type": "application/json"})
resp = conn.getresponse()
body = resp.read().decode("utf-8", "replace")
conn.close()
try:
return resp.status, (json.loads(body) if body else None)
except ValueError:
return resp.status, None
def _get(port, path, token):
conn = http.client.HTTPConnection("127.0.0.1", port, timeout=10)
conn.request("GET", path, headers={"Authorization": "Bearer " + token})
resp = conn.getresponse()
body = resp.read().decode("utf-8", "replace")
conn.close()
try:
return resp.status, (json.loads(body) if body else None)
except ValueError:
return resp.status, None
def exists(table, rid):
c = sqlite3.connect(os.environ["CRM_DB_PATH"])
n = c.execute(f"SELECT COUNT(*) FROM {table} WHERE id = ?", (rid,)).fetchone()[0]
c.close()
return n > 0
def seed():
c = sqlite3.connect(os.environ["CRM_DB_PATH"])
c.execute("INSERT INTO users (id,username,email,password_hash,full_name,role,is_active) "
"VALUES ('u1','grant','grant@ten31.example','x','Grant','admin',1)")
# Soft-deleted contact with ONLY soft-deleted children -> purgeable; cascade should remove them.
c.execute("INSERT INTO contacts (id,first_name,last_name,deleted_at) VALUES ('cClean','Dummy','Clean',?)", (DEL,))
c.execute("INSERT INTO opportunities (id,name,contact_id,owner_id,deleted_at) VALUES ('opC','Opp','cClean','u1',?)", (DEL,))
c.execute("INSERT INTO communications (id,contact_id,communication_date,created_by,subject,deleted_at) VALUES ('cmC','cClean','2026-05-01','u1','note',?)", (DEL,))
# A reminder pointing at the purge target (reminders.contact_id is a bare logical FK, no ON DELETE):
# the purge must NULL it, not leave it dangling and not delete the reminder.
c.execute("INSERT INTO reminders (id,contact_id,investor_id,title) VALUES ('remC','cClean','inv-x','Follow up dummy')")
# Soft-deleted contact WITH a live child -> must refuse (cascade would kill live data).
c.execute("INSERT INTO contacts (id,first_name,last_name,deleted_at) VALUES ('cLiveKid','Has','Livekid',?)", (DEL,))
c.execute("INSERT INTO communications (id,contact_id,communication_date,created_by,subject) VALUES ('cmLive','cLiveKid','2026-05-02','u1','live note')")
# A live contact -> must refuse (not soft-deleted).
c.execute("INSERT INTO contacts (id,first_name,last_name) VALUES ('cLive','Real','Person')")
# Soft-deleted org with no live refs -> purgeable.
c.execute("INSERT INTO organizations (id,name,deleted_at) VALUES ('orgClean','Dead Org',?)", (DEL,))
# Soft-deleted org referenced by a LIVE contact -> must refuse (SET NULL would mutate live data).
c.execute("INSERT INTO organizations (id,name,deleted_at) VALUES ('orgRef','Ref Org',?)", (DEL,))
c.execute("INSERT INTO contacts (id,first_name,last_name,organization_id) VALUES ('cRef','Org','Member','orgRef')")
c.commit()
c.close()
def main():
server.init_db()
seed()
token = server.create_token("u1", "grant", "admin")
httpd = ThreadingHTTPServer(("127.0.0.1", 0), _Quiet)
port = httpd.server_address[1]
threading.Thread(target=httpd.serve_forever, daemon=True).start()
try:
print("\n[list soft-deleted]")
st, body = _get(port, "/api/admin/soft-deleted", token)
groups = (body or {}).get("groups", {})
cids = {x["id"] for x in groups.get("contacts", [])}
oids = {x["id"] for x in groups.get("organizations", [])}
check(st == 200, f"GET soft-deleted -> 200 (got {st})")
check({"cClean", "cLiveKid"} <= cids and "cLive" not in cids, f"lists soft-deleted contacts only (got {cids})")
check({"orgClean", "orgRef"} <= oids, f"lists soft-deleted orgs (got {oids})")
check("opC" in {x["id"] for x in groups.get("opportunities", [])}, "lists the soft-deleted opportunity")
print("\n[purge guards]")
st, _ = _post(port, "/api/admin/soft-deleted/purge", token, {"table": "contacts", "id": "cLive"})
check(st == 400, f"purge a LIVE contact -> 400 (got {st})")
check(exists("contacts", "cLive"), "live contact still present after refused purge")
st, _ = _post(port, "/api/admin/soft-deleted/purge", token, {"table": "contacts", "id": "cLiveKid"})
check(st == 409, f"purge contact with a LIVE child -> 409 (got {st})")
check(exists("contacts", "cLiveKid") and exists("communications", "cmLive"), "contact + its live child preserved")
st, _ = _post(port, "/api/admin/soft-deleted/purge", token, {"table": "organizations", "id": "orgRef"})
check(st == 409, f"purge org referenced by a LIVE contact -> 409 (got {st})")
check(exists("organizations", "orgRef") and exists("contacts", "cRef"), "org + its live referencing contact preserved")
st, _ = _post(port, "/api/admin/soft-deleted/purge", token, {"table": "bogus", "id": "x"})
check(st == 400, f"unknown table -> 400 (got {st})")
st, _ = _post(port, "/api/admin/soft-deleted/purge", token, {"table": "contacts", "id": "nope"})
check(st == 404, f"missing id -> 404 (got {st})")
print("\n[purge happy path + cascade]")
st, _ = _post(port, "/api/admin/soft-deleted/purge", token, {"table": "contacts", "id": "cClean"})
check(st == 200, f"purge a clean soft-deleted contact -> 200 (got {st})")
check(not exists("contacts", "cClean"), "purged contact is gone")
check(not exists("opportunities", "opC") and not exists("communications", "cmC"),
"its soft-deleted children were cascade-removed")
_rc = sqlite3.connect(os.environ["CRM_DB_PATH"])
_rem = _rc.execute("SELECT contact_id FROM reminders WHERE id = 'remC'").fetchone()
_rc.close()
check(_rem is not None and _rem[0] is None,
"a reminder on the purged contact is KEPT but its contact_id is NULL'd (no dangling ref)")
st, _ = _post(port, "/api/admin/soft-deleted/purge", token, {"table": "organizations", "id": "orgClean"})
check(st == 200, f"purge a clean soft-deleted org -> 200 (got {st})")
check(not exists("organizations", "orgClean"), "purged org is gone")
finally:
httpd.shutdown()
print()
if FAILS:
print(f"{len(FAILS)} FAILED")
sys.exit(1)
print("ALL PASS (soft-deleted purge)")
if __name__ == "__main__":
main()