Soft-delete + source-count diagnostics; thesis v4 (0.1.0:47)

- DELETE handlers soft-delete (set deleted_at) + cascade contact -> opps/comms/lp
  instead of hard-deleting (guardrail #3); list queries filter deleted rows.
- ingest: chunking excludes soft-deleted records; qdrant delete-by-source-id;
  sync prunes soft-deleted records' vectors incrementally.
- /api/system/status returns raw source-record counts for sanity-checking.
- docs/thesis-seed-v4.md (no "bet" language, scarcity-forward, freedom-tech as
  a banner option, tightened pillars, reworked segments + edge).

Soft-delete verified via the running HTTP server (delete -> hidden + row kept).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

backend/ingest/chunking.py

@@ -91,7 +91,7 @@ def build_chunks(conn):
 
     # communications
     for r in conn.execute("""SELECT id, contact_id, type, subject, body, outcome, next_action, communication_date
-                             FROM communications"""):
+                             FROM communications WHERE deleted_at IS NULL"""):
         lp, lp_name, person = _contact_lp(r["contact_id"], person_canon, org_canon, name, contact_org)
         parts = [p for p in (r["subject"], r["body"], r["outcome"], r["next_action"]) if (p or "").strip()]
         chunks.append(_mk(f"communications:{r['id']}", lp, lp_name, person,
@@ -99,21 +99,21 @@ def build_chunks(conn):
                           "\n".join(parts), "communications", r["id"]))
 
     # contacts.notes
-    for r in conn.execute("SELECT id, notes, updated_at FROM contacts WHERE notes IS NOT NULL AND notes <> ''"):
+    for r in conn.execute("SELECT id, notes, updated_at FROM contacts WHERE notes IS NOT NULL AND notes <> '' AND deleted_at IS NULL"):
         lp, lp_name, person = _contact_lp(r["id"], person_canon, org_canon, name, contact_org)
         chunks.append(_mk(f"contacts.notes:{r['id']}", lp, lp_name, person,
                           "contact_note", to_epoch(r["updated_at"]), r["notes"], "contacts", r["id"]))
 
     # lp_profiles.notes
     for r in conn.execute("""SELECT lp.id, lp.contact_id, lp.notes, lp.updated_at
-                             FROM lp_profiles lp WHERE lp.notes IS NOT NULL AND lp.notes <> ''"""):
+                             FROM lp_profiles lp WHERE lp.notes IS NOT NULL AND lp.notes <> '' AND lp.deleted_at IS NULL"""):
         lp, lp_name, person = _contact_lp(r["contact_id"], person_canon, org_canon, name, contact_org)
         chunks.append(_mk(f"lp_profiles.notes:{r['id']}", lp, lp_name, person,
                           "lp_note", to_epoch(r["updated_at"]), r["notes"], "lp_profiles", r["id"]))
 
     # opportunities (description + next_step)
     for r in conn.execute("""SELECT id, contact_id, name, description, next_step, updated_at
-                             FROM opportunities"""):
+                             FROM opportunities WHERE deleted_at IS NULL"""):
         lp, lp_name, person = _contact_lp(r["contact_id"], person_canon, org_canon, name, contact_org)
         parts = [p for p in (r["name"], r["description"], r["next_step"]) if (p or "").strip()]
         chunks.append(_mk(f"opportunities:{r['id']}", lp, lp_name, person,
@@ -121,7 +121,7 @@ def build_chunks(conn):
 
     # organizations.description
     for r in conn.execute("""SELECT id, description, updated_at FROM organizations
-                             WHERE description IS NOT NULL AND description <> ''"""):
+                             WHERE description IS NOT NULL AND description <> '' AND deleted_at IS NULL"""):
         lp = org_canon.get(r["id"])
         chunks.append(_mk(f"organizations.description:{r['id']}", lp, name.get(lp), None,
                           "org_note", to_epoch(r["updated_at"]), r["description"], "organizations", r["id"]))

backend/ingest/qdrant_io.py

@@ -48,3 +48,16 @@ def upsert(points):
 def count():
     status, data = _req("POST", f"/collections/{COL}/points/count", {"exact": True})
     return (data or {}).get("result", {}).get("count")
+
+
+def delete_by_source_ids(source_ids):
+    """Delete all chunks belonging to the given CRM source records (by payload
+    source_id) — used to prune soft-deleted records from the index."""
+    ids = list(source_ids)
+    if not ids:
+        return None
+    status, data = _req("POST", f"/collections/{COL}/points/delete?wait=true",
+                        {"filter": {"must": [{"key": "source_id", "match": {"any": ids}}]}})
+    if status not in (200, 201):
+        raise RuntimeError(f"delete points -> {status}: {data}")
+    return data

backend/ingest/sync.py

@@ -60,6 +60,18 @@ def _state_set(conn, key, value):
                  (key, value, _now()))
 
 
+def _deleted_source_ids(conn, since):
+    """CRM records soft-deleted since the watermark — their chunks get pruned."""
+    ids = set()
+    for tbl in ("contacts", "organizations", "opportunities", "communications", "lp_profiles"):
+        try:
+            for r in conn.execute(f"SELECT id FROM {tbl} WHERE deleted_at IS NOT NULL AND deleted_at > ?", (since,)):
+                ids.add(r["id"])
+        except Exception:
+            pass
+    return ids
+
+
 def _changed_source_ids(conn, since):
     changed = set()
     for tbl, model in _CHANGE_TABLES:
@@ -91,6 +103,10 @@ def run(db, recreate=False, fuzzy=False, batch=32):
     if last is None or recreate:
         mode, target = "full", all_chunks
     else:
+        # Prune chunks of records soft-deleted since the last sync.
+        deleted = _deleted_source_ids(conn, last)
+        if deleted:
+            qdrant_io.delete_by_source_ids(deleted)
         changed = _changed_source_ids(conn, last)
         mode, target = "incremental", [c for c in all_chunks
                                        if (c["source_model"], c["source_id"]) in changed]