#!/usr/bin/env python3
"""
Venture Fund CRM — Local Self-Hosted Server
Backend API server using Python stdlib + SQLite + bcrypt + PyJWT
"""

import json
import os
import sys
import sqlite3
import hashlib
import hmac
import time
import uuid
import csv
import io
import re
import difflib
import base64
import threading
from datetime import datetime, timedelta
from http.server import HTTPServer, ThreadingHTTPServer, BaseHTTPRequestHandler
from urllib.parse import urlparse, parse_qs, unquote
from functools import wraps

# Available system packages
try:
    import bcrypt  # type: ignore
    BCRYPT_AVAILABLE = True
except Exception:
    bcrypt = None
    BCRYPT_AVAILABLE = False

try:
    import jwt  # type: ignore
    JWT_AVAILABLE = True
except Exception:
    jwt = None
    JWT_AVAILABLE = False

# Phase-1 Architect: human-gated thesis approval logic (pure stdlib; guarded).
try:
    import thesis_review  # type: ignore
except Exception:
    thesis_review = None

# Phase-1: entity-merge review + UI-triggered index jobs (guarded).
try:
    import entity_merge  # type: ignore
except Exception:
    entity_merge = None
try:
    import entity_jobs  # type: ignore
except Exception:
    entity_jobs = None

# Phase-1: the Architect agent (runs on Claude) + its tools live in backend/mcp.
# (Compute the path inline — this runs before BASE_DIR is defined below.)
try:
    sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "mcp"))
    import architect_tools as _architect_tools  # type: ignore
    import architect_agent as _architect_agent  # type: ignore
    import architect_grounding as _architect_grounding  # type: ignore
    import outreach_agent as _outreach_agent  # type: ignore
except Exception:
    _architect_tools = None
    _architect_agent = None
    _architect_grounding = None
    _outreach_agent = None

# ─── Configuration ────────────────────────────────────────────────────────────

BASE_DIR = os.path.dirname(os.path.abspath(__file__))
PROJECT_DIR = os.path.dirname(BASE_DIR)
DATA_DIR = os.environ.get("CRM_DATA_DIR", os.path.join(PROJECT_DIR, "data"))
FRONTEND_DIR = os.environ.get("CRM_FRONTEND_DIR", os.path.join(PROJECT_DIR, "frontend"))
FRONTEND_ROOT = os.path.realpath(FRONTEND_DIR)  # resolved once; the /assets/ containment boundary
DB_PATH = os.environ.get("CRM_DB_PATH", os.path.join(DATA_DIR, "crm.db"))
SECRET_KEY = os.environ.get("CRM_SECRET_KEY", "venture-crm-secret-change-in-production-" + str(uuid.uuid4()))
TOKEN_EXPIRY_HOURS = 24
HOST = os.environ.get("CRM_HOST", "0.0.0.0")
PORT = int(os.environ.get("CRM_PORT", "8080"))
CORS_ORIGIN = os.environ.get("CRM_CORS_ORIGIN", "*")
ENV = os.environ.get("CRM_ENV", "development")
LOGIN_RATE_LIMIT_PER_MIN = int(os.environ.get("CRM_LOGIN_RATE_LIMIT_PER_MIN", "20"))
WRITE_RATE_LIMIT_PER_MIN = int(os.environ.get("CRM_WRITE_RATE_LIMIT_PER_MIN", "300"))
GET_RATE_LIMIT_PER_MIN = int(os.environ.get("CRM_GET_RATE_LIMIT_PER_MIN", "600"))
# Auto-ban any IP that racks up too many 404s in a short window — almost always
# a vulnerability scanner blasting common paths (/.env, /.git/config, /swagger,
# /actuator/env, wp-json, etc.). Banned IPs get instant 429s with no DB or
# filesystem work, so they can't keep the single SQLite writer busy.
ABUSE_404_THRESHOLD = int(os.environ.get("CRM_ABUSE_404_THRESHOLD", "15"))
ABUSE_404_WINDOW_SEC = int(os.environ.get("CRM_ABUSE_404_WINDOW_SEC", "60"))
ABUSE_BAN_SEC = int(os.environ.get("CRM_ABUSE_BAN_SEC", "900"))  # 15 minutes
BACKUP_POLICY_SETTING_KEY = "fundraising_backup_policy"
DEFAULT_BACKUP_POLICY = {
    "enabled": True,
    "interval_hours": 24,
    "retention_days": 30,
    "max_backups": 60,
    "last_run_at": None
}
SEED_DEMO_DATA = os.environ.get("CRM_SEED_DEMO_DATA", "").strip().lower() in ("1", "true", "yes", "on")

os.makedirs(DATA_DIR, exist_ok=True)

# ─── Database Setup ───────────────────────────────────────────────────────────

def get_db():
    """Get a database connection with WAL mode and foreign keys enabled."""
    conn = sqlite3.connect(DB_PATH)
    conn.execute("PRAGMA journal_mode=WAL")
    conn.execute("PRAGMA foreign_keys=ON")
    conn.execute("PRAGMA busy_timeout=5000")
    conn.row_factory = sqlite3.Row
    return conn

def init_db():
    """Initialize all database tables."""
    conn = get_db()
    cursor = conn.cursor()

    cursor.executescript("""
        CREATE TABLE IF NOT EXISTS users (
            id TEXT PRIMARY KEY,
            username TEXT NOT NULL UNIQUE,
            email TEXT NOT NULL UNIQUE,
            password_hash TEXT NOT NULL,
            full_name TEXT NOT NULL,
            role TEXT NOT NULL DEFAULT 'member',
            is_active INTEGER DEFAULT 1,
            created_at TEXT DEFAULT (datetime('now')),
            updated_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS organizations (
            id TEXT PRIMARY KEY,
            name TEXT NOT NULL,
            type TEXT DEFAULT 'other',
            industry TEXT,
            website TEXT,
            phone TEXT,
            email TEXT,
            address TEXT,
            city TEXT,
            state TEXT,
            country TEXT,
            description TEXT,
            tags TEXT DEFAULT '[]',
            created_by TEXT REFERENCES users(id),
            created_at TEXT DEFAULT (datetime('now')),
            updated_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS contacts (
            id TEXT PRIMARY KEY,
            first_name TEXT NOT NULL,
            last_name TEXT NOT NULL,
            email TEXT,
            phone TEXT,
            mobile TEXT,
            title TEXT,
            organization_id TEXT REFERENCES organizations(id) ON DELETE SET NULL,
            -- RETIRED (v0.1.0:106), inert: the Investors/Prospects distinction is now derived live
            -- from the grid (contact_grid_signals → committed/pipeline_stage), not this column. No
            -- code reads or writes it; the DEFAULT keeps NOT NULL satisfied for inserts that omit it.
            -- Physical drop deferred to a signed-off table-rebuild migration (SQLite has no DROP COLUMN
            -- here; contacts is FK-referenced) — same retire-then-drop path lp_profiles took (v78→v104).
            contact_type TEXT NOT NULL DEFAULT 'prospect',
            status TEXT NOT NULL DEFAULT 'active',
            source TEXT,
            tags TEXT DEFAULT '[]',
            notes TEXT,
            linkedin_url TEXT,
            city TEXT,
            state TEXT,
            country TEXT,
            location_query TEXT,
            preferred_contact TEXT DEFAULT 'email',
            created_by TEXT REFERENCES users(id),
            created_at TEXT DEFAULT (datetime('now')),
            updated_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS opportunities (
            id TEXT PRIMARY KEY,
            name TEXT NOT NULL,
            contact_id TEXT NOT NULL REFERENCES contacts(id) ON DELETE CASCADE,
            organization_id TEXT REFERENCES organizations(id) ON DELETE SET NULL,
            stage TEXT NOT NULL DEFAULT 'lead',
            commitment_amount REAL DEFAULT 0,
            expected_amount REAL DEFAULT 0,
            probability INTEGER DEFAULT 10,
            expected_close_date TEXT,
            fund_name TEXT,
            description TEXT,
            next_step TEXT,
            owner_id TEXT NOT NULL REFERENCES users(id),
            priority TEXT DEFAULT 'medium',
            lost_reason TEXT,
            created_at TEXT DEFAULT (datetime('now')),
            updated_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS communications (
            id TEXT PRIMARY KEY,
            contact_id TEXT NOT NULL REFERENCES contacts(id) ON DELETE CASCADE,
            opportunity_id TEXT REFERENCES opportunities(id) ON DELETE SET NULL,
            type TEXT NOT NULL DEFAULT 'note',
            subject TEXT,
            body TEXT,
            communication_date TEXT NOT NULL,
            duration_minutes INTEGER,
            outcome TEXT,
            next_action TEXT,
            next_action_date TEXT,
            attendees TEXT DEFAULT '[]',
            created_by TEXT NOT NULL REFERENCES users(id),
            created_at TEXT DEFAULT (datetime('now')),
            updated_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS custom_fields (
            id TEXT PRIMARY KEY,
            name TEXT NOT NULL,
            entity_type TEXT NOT NULL,
            field_type TEXT NOT NULL DEFAULT 'text',
            options TEXT DEFAULT '[]',
            required INTEGER DEFAULT 0,
            display_order INTEGER DEFAULT 0,
            created_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS custom_field_values (
            id TEXT PRIMARY KEY,
            custom_field_id TEXT NOT NULL REFERENCES custom_fields(id) ON DELETE CASCADE,
            entity_id TEXT NOT NULL,
            entity_type TEXT NOT NULL,
            value TEXT,
            updated_at TEXT DEFAULT (datetime('now')),
            UNIQUE(custom_field_id, entity_id, entity_type)
        );

        CREATE TABLE IF NOT EXISTS audit_log (
            id TEXT PRIMARY KEY,
            user_id TEXT REFERENCES users(id),
            entity_type TEXT NOT NULL,
            entity_id TEXT NOT NULL,
            action TEXT NOT NULL,
            changes TEXT,
            created_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS tags (
            id TEXT PRIMARY KEY,
            name TEXT NOT NULL UNIQUE,
            color TEXT DEFAULT '#6366f1',
            created_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS fundraising_state (
            id TEXT PRIMARY KEY,
            grid_json TEXT NOT NULL,
            views_json TEXT NOT NULL,
            version INTEGER NOT NULL DEFAULT 1,
            updated_by TEXT REFERENCES users(id),
            created_at TEXT DEFAULT (datetime('now')),
            updated_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS fundraising_investors (
            id TEXT PRIMARY KEY,
            investor_name TEXT NOT NULL,
            notes TEXT,
            lead TEXT,
            lead_source TEXT,
            priority INTEGER DEFAULT 0,
            follow_up INTEGER DEFAULT 0,
            graveyard INTEGER DEFAULT 0,
            longshot_followup INTEGER DEFAULT 0,
            source_row_id TEXT NOT NULL UNIQUE,
            total_invested REAL DEFAULT 0,
            updated_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS fundraising_contacts (
            id TEXT PRIMARY KEY,
            investor_id TEXT NOT NULL REFERENCES fundraising_investors(id) ON DELETE CASCADE,
            full_name TEXT NOT NULL,
            email TEXT,
            title TEXT,
            city TEXT,
            state TEXT,
            country TEXT,
            location_query TEXT,
            sort_order INTEGER DEFAULT 0,
            updated_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS fundraising_funds (
            id TEXT PRIMARY KEY,
            column_id TEXT NOT NULL UNIQUE,
            fund_name TEXT NOT NULL,
            display_order INTEGER DEFAULT 0,
            active INTEGER DEFAULT 1,
            updated_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS fundraising_commitments (
            id TEXT PRIMARY KEY,
            investor_id TEXT NOT NULL REFERENCES fundraising_investors(id) ON DELETE CASCADE,
            fund_id TEXT NOT NULL REFERENCES fundraising_funds(id) ON DELETE CASCADE,
            amount REAL DEFAULT 0,
            updated_at TEXT DEFAULT (datetime('now')),
            UNIQUE(investor_id, fund_id)
        );

        CREATE TABLE IF NOT EXISTS fundraising_views (
            id TEXT PRIMARY KEY,
            name TEXT NOT NULL,
            filters_json TEXT NOT NULL,
            quick_search TEXT,
            hidden_columns_json TEXT NOT NULL DEFAULT '[]',
            column_filters_json TEXT NOT NULL DEFAULT '[]',
            updated_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS fundraising_automation_rules (
            id TEXT PRIMARY KEY,
            name TEXT NOT NULL,
            trigger_type TEXT NOT NULL DEFAULT 'flag_change',
            condition_json TEXT NOT NULL,
            action_json TEXT NOT NULL,
            enabled INTEGER DEFAULT 1,
            updated_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS fundraising_automation_runs (
            id TEXT PRIMARY KEY,
            rule_id TEXT REFERENCES fundraising_automation_rules(id) ON DELETE SET NULL,
            investor_id TEXT REFERENCES fundraising_investors(id) ON DELETE SET NULL,
            status TEXT NOT NULL DEFAULT 'applied',
            result_json TEXT,
            created_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS fundraising_list_memberships (
            id TEXT PRIMARY KEY,
            investor_id TEXT NOT NULL REFERENCES fundraising_investors(id) ON DELETE CASCADE,
            list_key TEXT NOT NULL,
            source TEXT NOT NULL DEFAULT 'automation',
            updated_at TEXT DEFAULT (datetime('now')),
            UNIQUE(investor_id, list_key)
        );

        CREATE TABLE IF NOT EXISTS fundraising_presence (
            user_id TEXT PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE,
            username TEXT NOT NULL,
            full_name TEXT,
            active_view TEXT,
            row_id TEXT,
            col_id TEXT,
            is_editing INTEGER DEFAULT 0,
            cell_key TEXT,
            last_seen_at TEXT DEFAULT (datetime('now')),
            expires_at_epoch INTEGER NOT NULL
        );

        CREATE TABLE IF NOT EXISTS fundraising_cell_locks (
            cell_key TEXT PRIMARY KEY,
            row_id TEXT NOT NULL,
            col_id TEXT NOT NULL,
            locked_by_user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
            locked_by_username TEXT NOT NULL,
            locked_by_full_name TEXT,
            last_seen_at TEXT DEFAULT (datetime('now')),
            expires_at_epoch INTEGER NOT NULL
        );

        CREATE TABLE IF NOT EXISTS app_settings (
            key TEXT PRIMARY KEY,
            value_json TEXT NOT NULL,
            updated_at TEXT DEFAULT (datetime('now'))
        );

        -- Indexes for performance
        CREATE INDEX IF NOT EXISTS idx_contacts_type ON contacts(contact_type);
        CREATE INDEX IF NOT EXISTS idx_contacts_status ON contacts(status);
        CREATE INDEX IF NOT EXISTS idx_contacts_org ON contacts(organization_id);
        CREATE INDEX IF NOT EXISTS idx_opportunities_stage ON opportunities(stage);
        CREATE INDEX IF NOT EXISTS idx_opportunities_owner ON opportunities(owner_id);
        CREATE INDEX IF NOT EXISTS idx_opportunities_contact ON opportunities(contact_id);
        CREATE INDEX IF NOT EXISTS idx_communications_contact ON communications(contact_id);
        CREATE INDEX IF NOT EXISTS idx_communications_date ON communications(communication_date);
        CREATE INDEX IF NOT EXISTS idx_audit_entity ON audit_log(entity_type, entity_id);
        CREATE INDEX IF NOT EXISTS idx_fr_investor_name ON fundraising_investors(investor_name);
        CREATE INDEX IF NOT EXISTS idx_fr_investor_lead ON fundraising_investors(lead);
        CREATE INDEX IF NOT EXISTS idx_fr_contacts_investor ON fundraising_contacts(investor_id);
        CREATE INDEX IF NOT EXISTS idx_fr_commitments_investor ON fundraising_commitments(investor_id);
        CREATE INDEX IF NOT EXISTS idx_fr_commitments_fund ON fundraising_commitments(fund_id);
        CREATE INDEX IF NOT EXISTS idx_fr_automation_runs_created ON fundraising_automation_runs(created_at);
        CREATE INDEX IF NOT EXISTS idx_fr_memberships_list ON fundraising_list_memberships(list_key);
        CREATE INDEX IF NOT EXISTS idx_fr_presence_expires ON fundraising_presence(expires_at_epoch);
        CREATE INDEX IF NOT EXISTS idx_fr_locks_expires ON fundraising_cell_locks(expires_at_epoch);
    """)

    # Lightweight schema migrations for existing databases.
    for stmt in [
        "ALTER TABLE contacts ADD COLUMN city TEXT",
        "ALTER TABLE contacts ADD COLUMN state TEXT",
        "ALTER TABLE contacts ADD COLUMN country TEXT",
        "ALTER TABLE contacts ADD COLUMN location_query TEXT",
        "ALTER TABLE fundraising_investors ADD COLUMN lead_source TEXT",
    ]:
        try:
            conn.execute(stmt)
        except sqlite3.OperationalError:
            pass

    # ─── Gmail integration migrations (feature-flag-guarded import) ───
    try:
        from email_integration.db import apply_migrations as _email_apply_migrations
        _email_apply_migrations(cursor)
    except ImportError:
        pass
    except Exception as _e:
        print(f"[email_integration] migration warning: {_e}")

    conn.commit()

    # ─── Core schema migrations (Phase 0+; ordered .sql files w/ ledger) ───
    # Additive/reversible only; tracked in schema_migrations. See core_migrations.py.
    try:
        from core_migrations import apply_core_migrations as _apply_core_migrations
        _apply_core_migrations(conn)
    except Exception as _e:
        print(f"[migrations] core migration warning: {_e}")

    # One-time: populate the new fundraising_contacts.contact_id (migration 0004)
    # by re-running the grid→relational sync. No-op once every row is linked.
    try:
        _backfill_grid_contact_ids(conn)
    except Exception as _e:
        print(f"[backfill] grid contact_id backfill warning: {_e}")

    # One-time: seed the v5 thesis into the Architect's Workshop if it is empty.
    try:
        from thesis_seed import ensure_thesis_seed as _ensure_thesis_seed
        _ensure_thesis_seed(conn)
    except Exception as _e:
        print(f"[thesis] seed warning: {_e}")

    # One-time: add the 2026-06-05 Architect positioning framings as candidate options.
    try:
        from thesis_seed import ensure_positioning_framings as _ensure_positioning_framings
        _ensure_positioning_framings(conn)
    except Exception as _e:
        print(f"[thesis] positioning framings warning: {_e}")

    # One-time: stage the v2.0 reserve-asset spine (signal-engine workstream) as candidates.
    try:
        from thesis_seed import ensure_thesis_v2_candidate as _ensure_thesis_v2_candidate
        _ensure_thesis_v2_candidate(conn)
    except Exception as _e:
        print(f"[thesis] v2 candidate warning: {_e}")

    # One-time: promote the v2.0 spine to the WORKING (approved) thesis and soft-retire the old
    # settlement throughline + Pillar 1, so the live agents stop emitting the dead spine. Node-level
    # only; the canonical thesis_version freeze stays the partners' dual-approval action (guardrail #4).
    try:
        from thesis_seed import ensure_thesis_v2_promoted as _ensure_thesis_v2_promoted
        _ensure_thesis_v2_promoted(conn)
    except Exception as _e:
        print(f"[thesis] v2 promote warning: {_e}")

    conn.close()
    print(f"Database initialized at {DB_PATH}")

# ─── Auth Helpers ─────────────────────────────────────────────────────────────

def hash_password(password):
    if BCRYPT_AVAILABLE:
        return bcrypt.hashpw(password.encode('utf-8'), bcrypt.gensalt()).decode('utf-8')

    # Stdlib fallback (PBKDF2) when bcrypt is unavailable
    salt = os.urandom(16).hex()
    digest = hashlib.pbkdf2_hmac('sha256', password.encode('utf-8'), bytes.fromhex(salt), 200_000).hex()
    return f"pbkdf2_sha256${salt}${digest}"

def verify_password(password, hashed):
    if BCRYPT_AVAILABLE and not str(hashed).startswith("pbkdf2_sha256$"):
        return bcrypt.checkpw(password.encode('utf-8'), hashed.encode('utf-8'))

    try:
        scheme, salt_hex, digest_hex = str(hashed).split('$', 2)
        if scheme != 'pbkdf2_sha256':
            return False
        check = hashlib.pbkdf2_hmac('sha256', password.encode('utf-8'), bytes.fromhex(salt_hex), 200_000).hex()
        return hmac.compare_digest(check, digest_hex)
    except Exception:
        return False

def _b64url_encode(data: bytes) -> str:
    return base64.urlsafe_b64encode(data).decode('utf-8').rstrip('=')

def _b64url_decode(data: str) -> bytes:
    padding = '=' * (-len(data) % 4)
    return base64.urlsafe_b64decode((data + padding).encode('utf-8'))

def create_token(user_id, username, role):
    payload = {
        "user_id": user_id,
        "username": username,
        "role": role,
        "exp": int(time.time()) + TOKEN_EXPIRY_HOURS * 3600,
        "iat": int(time.time())
    }
    if JWT_AVAILABLE:
        return jwt.encode(payload, SECRET_KEY, algorithm="HS256")

    # Stdlib fallback token format: base64url(payload).hmac_sha256_signature
    payload_bytes = json.dumps(payload, separators=(',', ':'), sort_keys=True).encode('utf-8')
    payload_part = _b64url_encode(payload_bytes)
    signature = hmac.new(SECRET_KEY.encode('utf-8'), payload_part.encode('utf-8'), hashlib.sha256).digest()
    return f"{payload_part}.{_b64url_encode(signature)}"

def decode_token(token):
    if JWT_AVAILABLE:
        try:
            return jwt.decode(token, SECRET_KEY, algorithms=["HS256"])
        except jwt.ExpiredSignatureError:
            return None
        except jwt.InvalidTokenError:
            return None

    try:
        payload_part, sig_part = str(token).split('.', 1)
        expected_sig = hmac.new(SECRET_KEY.encode('utf-8'), payload_part.encode('utf-8'), hashlib.sha256).digest()
        actual_sig = _b64url_decode(sig_part)
        if not hmac.compare_digest(expected_sig, actual_sig):
            return None
        payload = json.loads(_b64url_decode(payload_part).decode('utf-8'))
        exp = int(payload.get('exp', 0))
        if exp and time.time() > exp:
            return None
        return payload
    except Exception:
        return None

# ─── Helper Functions ─────────────────────────────────────────────────────────

def row_to_dict(row):
    if row is None:
        return None
    d = dict(row)
    # Parse JSON fields
    for key in ['tags', 'attendees', 'options']:
        if key in d and isinstance(d[key], str):
            try:
                d[key] = json.loads(d[key])
            except (json.JSONDecodeError, TypeError):
                pass
    return d

def rows_to_list(rows):
    return [row_to_dict(r) for r in rows]

def generate_id():
    return str(uuid.uuid4())[:8]

def now():
    return datetime.utcnow().isoformat() + "Z"

def deep_copy_json(value):
    return json.loads(json.dumps(value))

def parse_iso_utc(ts):
    if not ts or not isinstance(ts, str):
        return None
    try:
        if ts.endswith('Z'):
            ts = ts[:-1] + '+00:00'
        return datetime.fromisoformat(ts)
    except Exception:
        return None

def require_admin(user):
    return bool(user and user.get('role') == 'admin')

def require_bot_or_admin(user):
    """Gate for agent/bot-facing endpoints: a dedicated 'bot' service account OR an admin
    (admins keep parity for debugging/curl). The 'bot' role is authenticated-but-not-admin —
    it never passes require_admin, so a bot credential cannot reach user-management, security,
    or settings. Reach (which endpoints) is controlled here; autonomy (acting without a human)
    stays governed by the per-feature draft->approve gate, independent of role."""
    return bool(user and user.get('role') in ('admin', 'bot'))

def log_audit(conn, user_id, entity_type, entity_id, action, changes=None):
    conn.execute(
        "INSERT INTO audit_log (id, user_id, entity_type, entity_id, action, changes) VALUES (?, ?, ?, ?, ?, ?)",
        (generate_id(), user_id, entity_type, entity_id, action, json.dumps(changes) if changes else None)
    )

def get_app_setting(conn, key, default_value=None):
    row = conn.execute("SELECT value_json FROM app_settings WHERE key = ?", (key,)).fetchone()
    if not row:
        return deep_copy_json(default_value) if default_value is not None else None
    try:
        return json.loads(row['value_json'])
    except Exception:
        return deep_copy_json(default_value) if default_value is not None else None

def set_app_setting(conn, key, value):
    payload = json.dumps(value)
    conn.execute("""
        INSERT INTO app_settings (key, value_json, updated_at)
        VALUES (?, ?, ?)
        ON CONFLICT(key) DO UPDATE SET value_json = excluded.value_json, updated_at = excluded.updated_at
    """, (key, payload, now()))

def load_backup_policy(conn):
    raw = get_app_setting(conn, BACKUP_POLICY_SETTING_KEY, DEFAULT_BACKUP_POLICY)
    if not isinstance(raw, dict):
        raw = {}
    policy = deep_copy_json(DEFAULT_BACKUP_POLICY)
    policy.update(raw)
    policy['enabled'] = bool(policy.get('enabled'))
    policy['interval_hours'] = max(1, min(168, int(policy.get('interval_hours') or 24)))
    policy['retention_days'] = max(1, min(365, int(policy.get('retention_days') or 30)))
    policy['max_backups'] = max(1, min(1000, int(policy.get('max_backups') or 60)))
    policy['last_run_at'] = policy.get('last_run_at') if isinstance(policy.get('last_run_at'), str) else None
    return policy

def save_backup_policy(conn, policy):
    normalized = {
        "enabled": bool(policy.get('enabled')),
        "interval_hours": max(1, min(168, int(policy.get('interval_hours') or 24))),
        "retention_days": max(1, min(365, int(policy.get('retention_days') or 30))),
        "max_backups": max(1, min(1000, int(policy.get('max_backups') or 60))),
        "last_run_at": policy.get('last_run_at') if isinstance(policy.get('last_run_at'), str) else None
    }
    set_app_setting(conn, BACKUP_POLICY_SETTING_KEY, normalized)
    return normalized

def _to_bool(value):
    if isinstance(value, bool):
        return value
    if isinstance(value, (int, float)):
        return value != 0
    if isinstance(value, str):
        v = value.strip().lower()
        return v in ('1', 'true', 'yes', 'y', 'on')
    return False

def _to_number(value):
    if value is None:
        return 0.0
    if isinstance(value, (int, float)):
        return float(value)
    if isinstance(value, str):
        cleaned = value.replace(',', '').replace('$', '').strip()
        if cleaned == '':
            return 0.0
        try:
            return float(cleaned)
        except Exception:
            return 0.0
    return 0.0

def _split_full_name(full_name):
    parts = [p for p in str(full_name or '').strip().split() if p]
    if not parts:
        return '', ''
    if len(parts) == 1:
        return parts[0], ''
    return parts[0], ' '.join(parts[1:])

def _normalize_text(value):
    return str(value or '').strip().lower()

def _parse_location_text(text):
    raw = str(text or '').strip()
    if not raw:
        return '', '', '', ''
    parts = [p.strip() for p in raw.split(',') if p.strip()]
    city = parts[0] if len(parts) >= 1 else ''
    state = parts[1] if len(parts) >= 2 else ''
    country = parts[2] if len(parts) >= 3 else ''
    return city, state, country, raw

def ensure_default_automation_rules(conn):
    defaults = [
        {
            "id": "auto-graveyard-route",
            "name": "Route Graveyard Investors",
            "trigger_type": "flag_change",
            "condition_json": {"field": "graveyard", "equals": True},
            "action_json": {"set_list": "graveyard"},
            "enabled": 1
        },
        {
            "id": "auto-followup-route",
            "name": "Route Follow-up Investors",
            "trigger_type": "flag_change",
            "condition_json": {"field": "follow_up", "equals": True},
            "action_json": {"set_list": "follow_up"},
            "enabled": 1
        }
    ]
    for r in defaults:
        conn.execute("""
            INSERT INTO fundraising_automation_rules (id, name, trigger_type, condition_json, action_json, enabled, updated_at)
            VALUES (?, ?, ?, ?, ?, ?, ?)
            ON CONFLICT(id) DO UPDATE SET
                name = excluded.name,
                trigger_type = excluded.trigger_type,
                condition_json = excluded.condition_json,
                action_json = excluded.action_json,
                updated_at = excluded.updated_at
        """, (r["id"], r["name"], r["trigger_type"], json.dumps(r["condition_json"]), json.dumps(r["action_json"]), r["enabled"], now()))

def run_fundraising_automations(conn):
    ensure_default_automation_rules(conn)
    rules = rows_to_list(conn.execute("SELECT * FROM fundraising_automation_rules WHERE enabled = 1 ORDER BY id").fetchall())
    investors = rows_to_list(conn.execute("SELECT * FROM fundraising_investors").fetchall())
    previous = {}
    for m in rows_to_list(conn.execute("SELECT investor_id, list_key FROM fundraising_list_memberships").fetchall()):
        previous.setdefault(m['investor_id'], set()).add(m['list_key'])

    desired = {}
    for inv in investors:
        inv_id = inv['id']
        lists = {'all'}
        if _to_bool(inv.get('graveyard')):
            lists.add('graveyard')
        else:
            lists.add('main')
        if _to_bool(inv.get('follow_up')):
            lists.add('follow_up')
        desired[inv_id] = lists

    conn.execute("DELETE FROM fundraising_list_memberships")
    for inv_id, lists in desired.items():
        for key in sorted(lists):
            conn.execute("""
                INSERT INTO fundraising_list_memberships (id, investor_id, list_key, source, updated_at)
                VALUES (?, ?, ?, 'automation', ?)
            """, (generate_id(), inv_id, key, now()))

    for inv in investors:
        inv_id = inv['id']
        before = previous.get(inv_id, set())
        after = desired.get(inv_id, set())
        if before == after:
            continue
        added = sorted(list(after - before))
        removed = sorted(list(before - after))
        if added or removed:
            conn.execute("""
                INSERT INTO fundraising_automation_runs (id, rule_id, investor_id, status, result_json, created_at)
                VALUES (?, ?, ?, 'applied', ?, ?)
            """, (
                generate_id(),
                None,
                inv_id,
                json.dumps({"lists_added": added, "lists_removed": removed}),
                now()
            ))

def _ensure_organization_by_name(conn, org_name, actor_user_id=None):
    name = str(org_name or '').strip()
    if not name:
        return None
    existing = conn.execute("SELECT id FROM organizations WHERE lower(name) = lower(?) LIMIT 1", (name,)).fetchone()
    if existing:
        return existing['id']
    org_id = generate_id()
    conn.execute(
        "INSERT INTO organizations (id, name, type, created_by, updated_at) VALUES (?, ?, 'other', ?, ?)",
        (org_id, name, actor_user_id, now())
    )
    return org_id

def _upsert_contact_from_fundraising(conn, investor_name, contact, actor_user_id=None):
    if not isinstance(contact, dict):
        return None
    full_name = str(contact.get('name') or '').strip()
    email = str(contact.get('email') or '').strip()
    title = str(contact.get('title') or '').strip()
    source = str(contact.get('source') or '').strip()
    city = str(contact.get('city') or '').strip()
    state = str(contact.get('state') or '').strip()
    country = str(contact.get('country') or '').strip()
    location_query = str(contact.get('location_query') or '').strip()
    linkedin_url = str(contact.get('linkedin_url') or '').strip()
    phone = str(contact.get('phone') or '').strip()
    mobile = str(contact.get('mobile') or '').strip()
    if not full_name and not email:
        return None
    first_name, last_name = _split_full_name(full_name)
    if not first_name and email:
        first_name = email.split('@')[0]
    org_id = _ensure_organization_by_name(conn, investor_name, actor_user_id)

    existing = None
    if email:
        existing = conn.execute(
            "SELECT * FROM contacts WHERE lower(email) = lower(?) ORDER BY updated_at DESC LIMIT 1",
            (email,)
        ).fetchone()
    if not existing and first_name:
        if org_id:
            existing = conn.execute(
                """
                SELECT * FROM contacts
                WHERE lower(first_name) = lower(?) AND lower(last_name) = lower(?) AND organization_id = ?
                ORDER BY updated_at DESC LIMIT 1
                """,
                (first_name, last_name, org_id)
            ).fetchone()
        else:
            existing = conn.execute(
                """
                SELECT * FROM contacts
                WHERE lower(first_name) = lower(?) AND lower(last_name) = lower(?) AND organization_id IS NULL
                ORDER BY updated_at DESC LIMIT 1
                """,
                (first_name, last_name)
            ).fetchone()

    if existing:
        next_first = first_name or str(existing['first_name'] or '')
        next_last = last_name if (last_name or full_name) else str(existing['last_name'] or '')
        next_email = email or str(existing['email'] or '')
        next_title = title or str(existing['title'] or '')
        next_source = source or str(existing['source'] or '')
        next_city = city or str(existing['city'] or '')
        next_state = state or str(existing['state'] or '')
        next_country = country or str(existing['country'] or '')
        next_location_query = location_query or str(existing['location_query'] or '')
        next_linkedin = linkedin_url or str(existing['linkedin_url'] or '')
        next_phone = phone or str(existing['phone'] or '')
        next_mobile = mobile or str(existing['mobile'] or '')
        next_org = org_id or existing['organization_id']
        conn.execute("""
            UPDATE contacts
            SET first_name = ?, last_name = ?, email = ?, title = ?,
                organization_id = ?, source = ?, city = ?, state = ?, country = ?, location_query = ?, linkedin_url = ?, phone = ?, mobile = ?, updated_at = ?
            WHERE id = ?
        """, (next_first, next_last, next_email, next_title, next_org, next_source, next_city, next_state, next_country, next_location_query, next_linkedin, next_phone, next_mobile, now(), existing['id']))
        return existing['id']

    contact_id = generate_id()
    conn.execute("""
        INSERT INTO contacts (
            id, first_name, last_name, email, title, organization_id, source, status, city, state, country, location_query, linkedin_url, phone, mobile, created_by, updated_at
        ) VALUES (?, ?, ?, ?, ?, ?, ?, 'active', ?, ?, ?, ?, ?, ?, ?, ?, ?)
    """, (
        contact_id,
        first_name or 'Unknown',
        last_name or '',
        email,
        title,
        org_id,
        source,
        city,
        state,
        country,
        location_query,
        linkedin_url,
        phone,
        mobile,
        actor_user_id,
        now()
    ))
    return contact_id

def _sync_contact_to_fundraising_state(conn, contact_row, actor_user_id=None, remove=False):
    if not isinstance(contact_row, dict):
        return
    row = conn.execute("SELECT * FROM fundraising_state WHERE id = 'main'").fetchone()
    if not row:
        return
    try:
        grid = json.loads(row['grid_json']) if row['grid_json'] else {}
    except Exception:
        grid = {}
    grid = sanitize_fundraising_grid(grid)
    columns = grid.get('columns', [])
    rows = grid.get('rows', [])
    if not isinstance(columns, list) or not isinstance(rows, list):
        return
    if not any(isinstance(c, dict) and c.get('id') == 'contacts' for c in columns):
        return

    org_name = str(contact_row.get('organization_name') or '').strip()
    email = str(contact_row.get('email') or '').strip()
    full_name = ' '.join([str(contact_row.get('first_name') or '').strip(), str(contact_row.get('last_name') or '').strip()]).strip()
    title = str(contact_row.get('title') or '').strip()
    source = str(contact_row.get('source') or '').strip()
    city = str(contact_row.get('city') or '').strip()
    state = str(contact_row.get('state') or '').strip()
    country = str(contact_row.get('country') or '').strip()
    location_query = str(contact_row.get('location_query') or '').strip()
    if not full_name and not email:
        return

    target_row_indexes = []
    org_norm = _normalize_text(org_name)
    for idx, inv in enumerate(rows):
        if not isinstance(inv, dict):
            continue
        if org_norm and _normalize_text(inv.get('investor_name')) == org_norm:
            target_row_indexes.append(idx)
    if not target_row_indexes:
        for idx, inv in enumerate(rows):
            if not isinstance(inv, dict):
                continue
            contacts = inv.get('contacts')
            if not isinstance(contacts, list):
                continue
            for c in contacts:
                if not isinstance(c, dict):
                    continue
                if email and _normalize_text(c.get('email')) == _normalize_text(email):
                    target_row_indexes.append(idx)
                    break
                if full_name and _normalize_text(c.get('name')) == _normalize_text(full_name):
                    target_row_indexes.append(idx)
                    break

    if not target_row_indexes:
        return

    changed = False
    email_norm = _normalize_text(email)
    name_norm = _normalize_text(full_name)
    for idx in target_row_indexes:
        inv = rows[idx]
        contacts = inv.get('contacts')
        if not isinstance(contacts, list):
            contacts = []
        next_contacts = list(contacts)
        match_index = -1
        for c_idx, c in enumerate(next_contacts):
            if not isinstance(c, dict):
                continue
            c_email_norm = _normalize_text(c.get('email'))
            c_name_norm = _normalize_text(c.get('name'))
            if email_norm and c_email_norm and c_email_norm == email_norm:
                match_index = c_idx
                break
            if name_norm and c_name_norm == name_norm:
                match_index = c_idx
                break

        if remove:
            if match_index >= 0:
                next_contacts.pop(match_index)
                inv['contacts'] = next_contacts
                changed = True
            continue

        if match_index >= 0:
            existing = next_contacts[match_index] if isinstance(next_contacts[match_index], dict) else {}
            next_contacts[match_index] = {
                **existing,
                "name": full_name or existing.get('name') or '',
                "email": email or existing.get('email') or '',
                "title": title or existing.get('title') or '',
                "city": city or str(existing.get('city') or ''),
                "state": state or str(existing.get('state') or ''),
                "country": country or str(existing.get('country') or ''),
                "location_query": location_query or str(existing.get('location_query') or '')
            }
        else:
            next_contacts.append({
                "name": full_name,
                "email": email,
                "title": title,
                "city": city,
                "state": state,
                "country": country,
                "location_query": location_query
            })
        inv['contacts'] = next_contacts
        if source and not str(inv.get('lead_source') or '').strip():
            inv['lead_source'] = source
        changed = True

    if not changed:
        return

    next_views = []
    try:
        next_views = json.loads(row['views_json']) if row['views_json'] else []
    except Exception:
        next_views = []
    next_views = sanitize_grid_views(next_views)
    next_version = int(row['version'] or 1) + 1
    conn.execute("""
        UPDATE fundraising_state
        SET grid_json = ?, views_json = ?, version = ?, updated_by = ?, updated_at = ?
        WHERE id = 'main'
    """, (json.dumps(grid), json.dumps(next_views), next_version, actor_user_id, now()))
    sync_fundraising_relational(conn, grid, next_views, actor_user_id=actor_user_id)

def _backfill_grid_contact_ids(conn):
    """One-time backfill for migration 0004: populate fundraising_contacts.contact_id
    by re-running the grid→relational sync once. Fires only when the column exists
    AND some row still lacks a contact_id, so it runs once after the migration and is
    a no-op thereafter. Safe + idempotent: the fundraising_* tables are derived and
    rebuilt on every sync, and _upsert_contact_from_fundraising matches existing
    contacts by email/name (never creates a duplicate on re-run)."""
    try:
        need = conn.execute("SELECT 1 FROM fundraising_contacts WHERE contact_id IS NULL LIMIT 1").fetchone()
    except sqlite3.OperationalError:
        return  # contact_id column not present (migration 0004 not applied)
    if not need:
        return
    row = conn.execute("SELECT grid_json, views_json FROM fundraising_state WHERE id = 'main'").fetchone()
    if not row or not row[0]:
        return
    try:
        grid = json.loads(row[0])
        views = json.loads(row[1]) if row[1] else []
    except Exception:
        return
    sync_fundraising_relational(conn, sanitize_fundraising_grid(grid), views)
    conn.commit()
    print("[backfill] populated fundraising_contacts.contact_id from grid sync")


def sync_fundraising_relational(conn, grid, views, actor_user_id=None):
    columns = grid.get('columns', []) if isinstance(grid, dict) else []
    rows = grid.get('rows', []) if isinstance(grid, dict) else []
    views = views if isinstance(views, list) else []

    fund_columns = []
    for idx, col in enumerate(columns):
        if not isinstance(col, dict):
            continue
        col_id = str(col.get('id') or '').strip()
        if not col_id:
            continue
        is_fund = bool(col.get('isFund')) or col.get('type') == 'currency'
        if is_fund:
            fund_columns.append((idx, col))

    seen_fund_col_ids = set()
    fund_id_by_col = {}
    for idx, col in fund_columns:
        col_id = str(col.get('id'))
        label = str(col.get('label') or col_id).strip()
        seen_fund_col_ids.add(col_id)
        existing = conn.execute("SELECT id FROM fundraising_funds WHERE column_id = ?", (col_id,)).fetchone()
        fund_id = existing['id'] if existing else generate_id()
        conn.execute("""
            INSERT INTO fundraising_funds (id, column_id, fund_name, display_order, active, updated_at)
            VALUES (?, ?, ?, ?, 1, ?)
            ON CONFLICT(column_id) DO UPDATE SET
                fund_name = excluded.fund_name,
                display_order = excluded.display_order,
                active = 1,
                updated_at = excluded.updated_at
        """, (fund_id, col_id, label, idx, now()))
        fund_id_by_col[col_id] = fund_id

    if seen_fund_col_ids:
        placeholders = ','.join(['?'] * len(seen_fund_col_ids))
        conn.execute(f"UPDATE fundraising_funds SET active = 0, updated_at = ? WHERE column_id NOT IN ({placeholders})", [now(), *list(seen_fund_col_ids)])
    else:
        conn.execute("UPDATE fundraising_funds SET active = 0, updated_at = ?", (now(),))

    seen_source_row_ids = set()
    for row in rows:
        if not isinstance(row, dict):
            continue
        source_row_id = str(row.get('id') or '').strip()
        if not source_row_id:
            continue
        seen_source_row_ids.add(source_row_id)
        investor_name = str(row.get('investor_name') or '').strip() or 'Untitled Investor'
        notes = str(row.get('notes') or '')
        lead = str(row.get('lead') or '')
        lead_source = str(row.get('lead_source') or row.get('combined_lead_source') or '').strip()
        total_invested = 0.0
        for _, col in fund_columns:
            total_invested += _to_number(row.get(str(col.get('id'))))

        existing = conn.execute("SELECT id FROM fundraising_investors WHERE source_row_id = ?", (source_row_id,)).fetchone()
        investor_id = existing['id'] if existing else generate_id()
        conn.execute("""
            INSERT INTO fundraising_investors (
                id, investor_name, notes, lead, lead_source, priority, follow_up, graveyard,
                source_row_id, total_invested, updated_at
            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
            ON CONFLICT(source_row_id) DO UPDATE SET
                investor_name = excluded.investor_name,
                notes = excluded.notes,
                lead = excluded.lead,
                lead_source = CASE
                    WHEN COALESCE(trim(fundraising_investors.lead_source), '') = '' THEN excluded.lead_source
                    ELSE fundraising_investors.lead_source
                END,
                priority = excluded.priority,
                follow_up = excluded.follow_up,
                graveyard = excluded.graveyard,
                total_invested = excluded.total_invested,
                updated_at = excluded.updated_at
        """, (
            investor_id,
            investor_name,
            notes,
            lead,
            lead_source,
            1 if _to_bool(row.get('priority')) else 0,
            1 if _to_bool(row.get('follow_up')) else 0,
            1 if _to_bool(row.get('graveyard')) else 0,
            source_row_id,
            total_invested,
            now()
        ))

        fresh = conn.execute("SELECT id FROM fundraising_investors WHERE source_row_id = ?", (source_row_id,)).fetchone()
        investor_id = fresh['id']

        conn.execute("DELETE FROM fundraising_contacts WHERE investor_id = ?", (investor_id,))
        contacts = row.get('contacts')
        if isinstance(contacts, list):
            for i, c in enumerate(contacts):
                if not isinstance(c, dict):
                    continue
                full_name = str(c.get('name') or '').strip()
                email = str(c.get('email') or '').strip()
                if not full_name and not email:
                    continue
                contact_payload = dict(c)
                if lead_source and not str(contact_payload.get('source') or '').strip():
                    contact_payload['source'] = lead_source
                linked_contact_id = _upsert_contact_from_fundraising(conn, investor_name, contact_payload, actor_user_id=actor_user_id)
                conn.execute("""
                    INSERT INTO fundraising_contacts (
                        id, investor_id, full_name, email, title, city, state, country, location_query, sort_order, contact_id, updated_at
                    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                """, (
                    generate_id(), investor_id, full_name, email, str(c.get('title') or ''),
                    str(c.get('city') or ''), str(c.get('state') or ''), str(c.get('country') or ''),
                    str(c.get('location_query') or ''), i, linked_contact_id, now()
                ))
        elif isinstance(contacts, str) and contacts.strip():
            linked_contact_id = _upsert_contact_from_fundraising(conn, investor_name, {"name": contacts.strip(), "email": "", "title": "", "source": lead_source}, actor_user_id=actor_user_id)
            conn.execute("""
                INSERT INTO fundraising_contacts (
                    id, investor_id, full_name, email, title, city, state, country, location_query, sort_order, contact_id, updated_at
                ) VALUES (?, ?, ?, '', '', '', '', '', '', 0, ?, ?)
            """, (generate_id(), investor_id, contacts.strip(), linked_contact_id, now()))

        conn.execute("DELETE FROM fundraising_commitments WHERE investor_id = ?", (investor_id,))
        for _, col in fund_columns:
            col_id = str(col.get('id'))
            fund_id = fund_id_by_col.get(col_id)
            if not fund_id:
                continue
            amount = _to_number(row.get(col_id))
            if abs(amount) < 1e-9:
                continue
            conn.execute("""
                INSERT INTO fundraising_commitments (id, investor_id, fund_id, amount, updated_at)
                VALUES (?, ?, ?, ?, ?)
            """, (generate_id(), investor_id, fund_id, amount, now()))

    if seen_source_row_ids:
        placeholders = ','.join(['?'] * len(seen_source_row_ids))
        conn.execute(f"DELETE FROM fundraising_investors WHERE source_row_id NOT IN ({placeholders})", list(seen_source_row_ids))
    else:
        conn.execute("DELETE FROM fundraising_investors")

    conn.execute("DELETE FROM fundraising_views")
    for v in views:
        if not isinstance(v, dict):
            continue
        view_id = str(v.get('id') or '').strip()
        if not view_id:
            continue
        conn.execute("""
            INSERT INTO fundraising_views (
                id, name, filters_json, quick_search, hidden_columns_json, column_filters_json, updated_at
            ) VALUES (?, ?, ?, ?, ?, ?, ?)
        """, (
            view_id,
            str(v.get('name') or view_id),
            json.dumps(v.get('filters') if isinstance(v.get('filters'), dict) else {}),
            str(v.get('quickSearch') or ''),
            json.dumps(v.get('hiddenColumns') if isinstance(v.get('hiddenColumns'), list) else []),
            json.dumps(v.get('columnFilters') if isinstance(v.get('columnFilters'), list) else []),
            now()
        ))
    run_fundraising_automations(conn)

def find_intake_match(conn, q, email=None):
    """Find an existing fundraising-grid investor for the intake bot's new-vs-existing hint.

    Scans the canonical grid blob (not the derived tables) so the returned `id` is the grid
    row id that handle_log_fundraising_communication matches on — keeping the bot's proposal
    consistent with where the write actually lands (no duplicate-investor risk). Matches by
    normalized investor_name first (the write's own key), then falls back to a contact email.
    Deleted investors are absent from the blob; graveyarded ones remain (a note on them is
    still valid), so no extra filtering is needed."""
    row = conn.execute("SELECT grid_json FROM fundraising_state WHERE id = 'main'").fetchone()
    if not row or not row['grid_json']:
        return None
    try:
        grid = json.loads(row['grid_json'])
    except Exception:
        return None
    rows = grid.get('rows', []) if isinstance(grid, dict) else []
    wanted_name = _normalize_text(q) if q else ''
    wanted_email = (email or '').strip().lower()
    email_hit = None
    for r in rows:
        if not isinstance(r, dict):
            continue
        rid = str(r.get('id') or '').strip()
        if not rid:
            continue
        name = str(r.get('investor_name') or '').strip()
        if wanted_name and _normalize_text(name) == wanted_name:
            return {"id": rid, "investor_name": name, "matched_on": "name"}
        if wanted_email and email_hit is None:
            contacts = r.get('contacts')
            if isinstance(contacts, list):
                for c in contacts:
                    if isinstance(c, dict) and str(c.get('email') or '').strip().lower() == wanted_email:
                        email_hit = {"id": rid, "investor_name": name, "matched_on": "email"}
                        break
    return email_hit


def _email_edit_distance(a, b):
    """Levenshtein distance between two short strings (emails). Stdlib-only DP; used to flag
    near-miss emails (a one- or two-character typo) for the intake fuzzy matcher."""
    a = (a or '').strip().lower()
    b = (b or '').strip().lower()
    if a == b:
        return 0
    if not a or not b:
        return max(len(a), len(b))
    prev = list(range(len(b) + 1))
    for i, ca in enumerate(a, 1):
        cur = [i]
        for j, cb in enumerate(b, 1):
            cost = 0 if ca == cb else 1
            cur.append(min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + cost))
        prev = cur
    return prev[-1]


# Legal-entity suffixes stripped before name comparison so 'Acme Capital' ~ 'Acme Capital LLC'.
# Deliberately tight: only true entity types, NOT name-type words (Capital/Ventures/Partners),
# which are distinctive enough to keep. Intentionally EXCLUDES 'sa'/'ag' (Société Anonyme /
# Aktiengesellschaft) — niche for this portfolio and ambiguous enough as coincidental trailing
# tokens that stripping them inflates false 1.0 matches across distinct firms.
_LEGAL_SUFFIX = {"llc", "lp", "llp", "inc", "incorporated", "ltd", "limited", "co", "corp",
                 "corporation", "company", "plc", "gmbh", "pte"}


def _strip_legal_suffix(normalized_name):
    """Drop trailing legal-entity suffix tokens (llc/lp/inc/…) from an already-normalized name."""
    toks = re.findall(r"[a-z0-9]+", normalized_name)
    while toks and toks[-1] in _LEGAL_SUFFIX:
        toks.pop()
    return " ".join(toks)


# Generic firm-descriptor words that carry almost no identifying signal: nearly every firm name
# contains one ("… Investment Group", "… Capital", "… Family Office"). Two names that overlap ONLY
# on these are NOT duplicates — 'Fortitude Investment Group' is not 'Aether Investment Group'. We
# compare on the DISTINCTIVE remainder so a shared descriptor can't inflate the score (the earlier
# "Capital/Ventures/Partners are distinctive enough to keep" assumption produced false shortlists —
# Grant, 2026-06-20). If a name is ALL descriptor ('Family Office'), we fall back to its full tokens
# so there's still something to compare.
_GENERIC_ORG_WORDS = frozenset({
    "investment", "investments", "investing", "investor", "investors",
    "capital", "ventures", "venture", "partners", "partner", "group",
    "fund", "funds", "management", "advisors", "advisers", "advisory",
    "asset", "assets", "holdings", "holding", "family", "office",
    "trust", "associates", "equity", "financial", "finance", "global",
    "international", "company", "enterprises", "wealth", "the", "and", "of",
})


def _distinctive_tokens(normalized_name):
    """Tokens of a (legal-suffix-stripped) name with generic firm descriptors removed. Falls back to
    the full token list when the name is nothing but descriptors, so an all-generic name still compares."""
    toks = re.findall(r"[a-z0-9]+", normalized_name)
    keep = [t for t in toks if t not in _GENERIC_ORG_WORDS]
    return keep or toks


def _name_similarity(a, b):
    """0..1 fuzzy similarity between two investor names: the max of difflib's sequence ratio
    (catches near-spellings — 'Charlie'/'Charles') and token-set Jaccard overlap (catches
    word-order differences). Legal-entity suffixes are stripped first, so two names differing
    only by 'LLC'/'LP'/'Inc' score 1.0 (a near-certain duplicate to surface — find_intake_match
    won't have caught it, since it compares the full string). Both the ratio and the Jaccard run on
    the DISTINCTIVE tokens (generic descriptors like 'Investment Group'/'Capital' removed), so firms
    that share only a descriptor don't surface as look-alikes; 'Aether Capital' ~ 'Aether Capital
    Partners' still scores 1.0 on the distinctive 'aether'. Still recall-favoring on real overlap."""
    a = _normalize_text(a)
    b = _normalize_text(b)
    if not a or not b:
        return 0.0
    if a == b:
        return 1.0
    sa = _strip_legal_suffix(a) or a
    sb = _strip_legal_suffix(b) or b
    if sa == sb:
        return 1.0
    da = _distinctive_tokens(sa)  # order-preserving for the sequence ratio
    db = _distinctive_tokens(sb)
    ratio = difflib.SequenceMatcher(None, " ".join(da), " ".join(db)).ratio()
    ta, tb = set(da), set(db)
    jaccard = len(ta & tb) / len(ta | tb) if (ta or tb) else 0.0
    return max(ratio, jaccard)


def find_intake_candidates(conn, q, email=None, limit=5, min_score=0.62, max_email_distance=2):
    """Ranked fuzzy near-matches for the intake bot's disambiguation prompt.

    Complements find_intake_match (which is exact-after-normalization): when the exact matcher
    misses, this returns the closest existing grid investors so the bot can surface them
    in-thread and the human can attach to one — instead of unknowingly creating a duplicate.
    Deterministic (stdlib difflib + token overlap + email edit distance), no LLM. Scans the same
    canonical grid blob as find_intake_match, so candidate ids are grid row ids the write targets.
    EXCLUDES exact matches (score 1.0 — those belong to find_intake_match) and ranks by score."""
    row = conn.execute("SELECT grid_json FROM fundraising_state WHERE id = 'main'").fetchone()
    if not row or not row['grid_json']:
        return []
    try:
        grid = json.loads(row['grid_json'])
    except Exception:
        return []
    rows = grid.get('rows', []) if isinstance(grid, dict) else []
    wanted_name = _normalize_text(q) if q else ''
    wanted_email = (email or '').strip().lower()
    scored = {}
    for r in rows:
        if not isinstance(r, dict):
            continue
        rid = str(r.get('id') or '').strip()
        if not rid:
            continue
        name = str(r.get('investor_name') or '').strip()
        # An exact name match belongs to find_intake_match — never echo it back as a candidate.
        if wanted_name and _normalize_text(name) == wanted_name:
            continue
        name_score = _name_similarity(wanted_name, name) if (wanted_name and name) else 0.0
        email_score = 0.0
        if wanted_email:
            contacts = r.get('contacts')
            if isinstance(contacts, list):
                for c in contacts:
                    if not isinstance(c, dict):
                        continue
                    ce = str(c.get('email') or '').strip().lower()
                    if not ce:
                        continue
                    dist = _email_edit_distance(wanted_email, ce)
                    # dist 0 is an exact email (find_intake_match's); 1→0.9, 2→0.8 are near-misses
                    if 0 < dist <= max_email_distance:
                        email_score = max(email_score, 1.0 - 0.1 * dist)
        score = max(name_score, email_score)
        if score < min_score:  # too weak to be a useful suggestion
            continue
        matched_on = 'email' if email_score >= name_score else 'name'
        # a row can match on both name and email — keep its highest-scoring read
        if rid not in scored or score > scored[rid]['score']:
            scored[rid] = {"id": rid, "investor_name": name,
                           "score": round(score, 3), "matched_on": matched_on}
    out = sorted(scored.values(), key=lambda x: x['score'], reverse=True)
    return out[:limit]


def ensure_fundraising_state_row(conn):
    existing = conn.execute("SELECT * FROM fundraising_state WHERE id = 'main'").fetchone()
    if not existing:
        default_grid = {
            "columns": deep_copy_json(DEFAULT_FUNDRAISING_COLUMNS),
            "rows": deep_copy_json(DEFAULT_FUNDRAISING_ROWS)
        }
        default_views = sanitize_grid_views(deep_copy_json(DEFAULT_GRID_VIEWS))
        conn.execute("""
            INSERT INTO fundraising_state (id, grid_json, views_json, version, updated_by)
            VALUES ('main', ?, ?, 1, NULL)
        """, (json.dumps(default_grid), json.dumps(default_views)))
        sync_fundraising_relational(conn, default_grid, default_views)
        conn.commit()
        return

    try:
        grid = json.loads(existing['grid_json']) if existing['grid_json'] else {}
    except Exception:
        grid = {}
    grid = sanitize_fundraising_grid(grid)
    try:
        views = json.loads(existing['views_json']) if existing['views_json'] else []
    except Exception:
        views = []
    rel_count = conn.execute("SELECT COUNT(*) AS c FROM fundraising_investors").fetchone()['c']
    if rel_count == 0:
        sync_fundraising_relational(conn, grid, sanitize_grid_views(views if isinstance(views, list) else deep_copy_json(DEFAULT_GRID_VIEWS)))
        conn.commit()

def list_backups():
    backup_dir = os.path.join(DATA_DIR, "backups")
    os.makedirs(backup_dir, exist_ok=True)
    entries = []
    for name in os.listdir(backup_dir):
        if not name.endswith('.json'):
            continue
        path = os.path.join(backup_dir, name)
        if not os.path.isfile(path):
            continue
        st = os.stat(path)
        if "pre_restore" in name:
            kind = "pre_restore"
        elif "_auto_" in name:
            kind = "auto"
        else:
            kind = "backup"
        entries.append({
            "filename": name,
            "path": path,
            "size_bytes": st.st_size,
            "modified_at": datetime.utcfromtimestamp(st.st_mtime).isoformat() + "Z",
            "kind": kind
        })
    entries.sort(key=lambda x: x["modified_at"], reverse=True)
    return entries

def create_fundraising_backup_file(state_row, kind="backup"):
    backup_dir = os.path.join(DATA_DIR, "backups")
    os.makedirs(backup_dir, exist_ok=True)
    ts = datetime.utcnow().strftime('%Y%m%dT%H%M%SZ')
    if kind == "pre_restore":
        filename = f"fundraising_state_pre_restore_{ts}.json"
    elif kind == "auto":
        filename = f"fundraising_state_auto_{ts}.json"
    else:
        filename = f"fundraising_state_{ts}.json"
    path = os.path.join(backup_dir, filename)
    try:
        grid = json.loads(state_row['grid_json']) if state_row and state_row['grid_json'] else {}
    except Exception:
        grid = {}
    try:
        views = json.loads(state_row['views_json']) if state_row and state_row['views_json'] else []
    except Exception:
        views = []
    payload = {
        "backup_at": now(),
        "version": state_row['version'] if state_row else 1,
        "updated_at": state_row['updated_at'] if state_row else None,
        "grid": grid,
        "views": views
    }
    with open(path, 'w', encoding='utf-8') as f:
        json.dump(payload, f, ensure_ascii=True, indent=2)
    return {
        "filename": filename,
        "path": path,
        "version": payload["version"],
        "kind": kind
    }

def apply_backup_retention(policy):
    entries = [b for b in list_backups() if b.get('kind') in ('backup', 'auto')]
    keep = max(1, int(policy.get('max_backups') or 60))
    cutoff = datetime.utcnow() - timedelta(days=max(1, int(policy.get('retention_days') or 30)))

    for b in entries:
        ts = parse_iso_utc(b.get('modified_at'))
        if ts and ts.replace(tzinfo=None) < cutoff and b.get('kind') == 'auto':
            try:
                os.remove(b['path'])
            except Exception:
                pass

    entries = [b for b in list_backups() if b.get('kind') in ('backup', 'auto')]
    for stale in entries[keep:]:
        if stale.get('kind') != 'auto':
            continue
        try:
            os.remove(stale['path'])
        except Exception:
            pass

def compute_restore_diff(current_grid, current_views, next_grid, next_views):
    current_columns = current_grid.get('columns', []) if isinstance(current_grid, dict) else []
    next_columns = next_grid.get('columns', []) if isinstance(next_grid, dict) else []
    current_rows = current_grid.get('rows', []) if isinstance(current_grid, dict) else []
    next_rows = next_grid.get('rows', []) if isinstance(next_grid, dict) else []

    curr_col_map = {str(c.get('id')): c for c in current_columns if isinstance(c, dict) and c.get('id')}
    next_col_map = {str(c.get('id')): c for c in next_columns if isinstance(c, dict) and c.get('id')}
    curr_row_map = {str(r.get('id')): r for r in current_rows if isinstance(r, dict) and r.get('id')}
    next_row_map = {str(r.get('id')): r for r in next_rows if isinstance(r, dict) and r.get('id')}

    added_columns = [next_col_map[k] for k in next_col_map.keys() - curr_col_map.keys()]
    removed_columns = [curr_col_map[k] for k in curr_col_map.keys() - next_col_map.keys()]

    changed_column_types = []
    for key in curr_col_map.keys() & next_col_map.keys():
        before_type = curr_col_map[key].get('type')
        after_type = next_col_map[key].get('type')
        if before_type != after_type:
            changed_column_types.append({
                "id": key,
                "label": next_col_map[key].get('label') or curr_col_map[key].get('label') or key,
                "before_type": before_type,
                "after_type": after_type
            })

    common_cols = list(curr_col_map.keys() & next_col_map.keys())
    cell_changes_count = 0
    changed_rows = set()
    for rid in curr_row_map.keys() & next_row_map.keys():
        before = curr_row_map[rid]
        after = next_row_map[rid]
        row_changed = False
        for cid in common_cols:
            if before.get(cid) != after.get(cid):
                cell_changes_count += 1
                row_changed = True
        if row_changed:
            changed_rows.add(rid)

    curr_view_ids = {str(v.get('id')) for v in current_views if isinstance(v, dict) and v.get('id')}
    next_view_ids = {str(v.get('id')) for v in next_views if isinstance(v, dict) and v.get('id')}
    curr_view_names = {str(v.get('id')): v.get('name') for v in current_views if isinstance(v, dict) and v.get('id')}
    next_view_names = {str(v.get('id')): v.get('name') for v in next_views if isinstance(v, dict) and v.get('id')}

    return {
        "columns_added": [{"id": c.get('id'), "label": c.get('label')} for c in added_columns[:25]],
        "columns_removed": [{"id": c.get('id'), "label": c.get('label')} for c in removed_columns[:25]],
        "columns_type_changed": changed_column_types[:25],
        "rows_added_count": len(next_row_map.keys() - curr_row_map.keys()),
        "rows_removed_count": len(curr_row_map.keys() - next_row_map.keys()),
        "rows_changed_count": len(changed_rows),
        "cell_changes_count": cell_changes_count,
        "views_added": [{"id": i, "name": next_view_names.get(i)} for i in list(next_view_ids - curr_view_ids)[:25]],
        "views_removed": [{"id": i, "name": curr_view_names.get(i)} for i in list(curr_view_ids - next_view_ids)[:25]]
    }

def sanitize_grid_views(views):
    if not isinstance(views, list):
        return deep_copy_json(DEFAULT_GRID_VIEWS)
    out = []
    for view in views:
        if not isinstance(view, dict) or str(view.get('id') or '') == 'view-longshot':
            continue
        next_view = deep_copy_json(view)
        filters = next_view.get('filters')
        if isinstance(filters, dict) and 'longshotOnly' in filters:
            filters.pop('longshotOnly', None)
        if not isinstance(filters, dict):
            filters = {}
            next_view['filters'] = filters
        view_id = str(next_view.get('id') or '')
        if view_id == 'view-graveyard':
            filters['graveyardOnly'] = True
            filters['includeGraveyard'] = True
        else:
            filters['graveyardOnly'] = bool(filters.get('graveyardOnly', False))
        if view_id == 'view-followup':
            filters['followUpOnly'] = True
        else:
            filters['followUpOnly'] = bool(filters.get('followUpOnly', False))
        out.append(next_view)
    return out if out else deep_copy_json(DEFAULT_GRID_VIEWS)

def sanitize_fundraising_grid(grid):
    if not isinstance(grid, dict):
        return {
            "columns": deep_copy_json(DEFAULT_FUNDRAISING_COLUMNS),
            "rows": deep_copy_json(DEFAULT_FUNDRAISING_ROWS)
        }
    columns = grid.get('columns')
    rows = grid.get('rows')
    if not isinstance(columns, list):
        columns = deep_copy_json(DEFAULT_FUNDRAISING_COLUMNS)
    if not isinstance(rows, list):
        rows = deep_copy_json(DEFAULT_FUNDRAISING_ROWS)

    # `pipeline` / `pipeline_stage` are read-only columns whose VALUES are derived from the
    # linked opportunity and injected on read — never persisted as row data (the GET handler
    # re-injects them after sanitize). The column DEFINITIONS persist like any other column
    # so their position / width / hidden state is kept.
    _computed_row_values = ('longshot_followup', 'pipeline', 'pipeline_stage', 'opportunity_id',
                            'reminder_status', 'existing_investor', 'last_activity_at', 'staleness')

    clean_columns = []
    seen = set()
    for col in columns:
        if not isinstance(col, dict):
            continue
        col_id = str(col.get('id') or '').strip()
        if not col_id or col_id == 'longshot_followup' or col_id in seen:
            continue
        seen.add(col_id)
        clean_columns.append(col)

    clean_rows = []
    for row in rows:
        if not isinstance(row, dict):
            continue
        next_row = dict(row)
        for _k in _computed_row_values:
            next_row.pop(_k, None)
        clean_rows.append(next_row)

    return {"columns": clean_columns, "rows": clean_rows}

# The full set of fields a fundraising contact "pill" carries (what the desktop contacts-cell
# editor produces). The mobile pill editor edits name/email/title and carries the rest through
# untouched; this whitelist sanitizes an inbound contacts array on the single-row update path.
_FUNDRAISING_CONTACT_FIELDS = ('name', 'email', 'title', 'source', 'city', 'state',
                               'country', 'location_query', 'linkedin_url')

def _sanitize_fundraising_contacts(raw):
    """Coerce a client-supplied contacts array to clean pill dicts: keep only the known pill
    fields, trim strings, and drop entries with neither a name nor an email (the same emptiness
    rule sync_fundraising_relational uses, so a sanitized list round-trips through sync without
    surprise drops). Dedup is the client's job (BRIEF §3a); this only guards blanks + stray keys."""
    out = []
    if not isinstance(raw, list):
        return out
    for c in raw:
        if not isinstance(c, dict):
            continue
        pill = {}
        for k in _FUNDRAISING_CONTACT_FIELDS:
            if k in c and c.get(k) is not None:
                pill[k] = str(c.get(k)).strip()
        if not pill.get('name') and not pill.get('email'):
            continue
        out.append(pill)
    return out

# ─── Grid ↔ Pipeline link (Adopt the Pipeline) ────────────────────────────────
# The fundraising grid is canonical; the Pipeline board is a view of the deals it
# drives. opportunities.fundraising_investor_id is the durable join. Two ownership
# rules keep them reconciled:
#   * Grid owns: whether the link exists, the investor, the primary contact, the seed.
#   * Pipeline owns: stage / probability / owner / close date / next step.
# So a grid save NEVER reseeds an existing linked opp (it would clobber funnel state),
# and "is in pipeline?" / "what stage?" are DERIVED from the live opp join — never a
# denormalized flag that could drift.

def _resolve_owner_from_lead(conn, lead_value, fallback_user_id):
    """Map a grid 'lead' cell (a team member's name/initials) to a users.id.
    opportunities.owner_id is NOT NULL, so a value is always returned — the acting user
    is the fallback when no confident match exists. Owner is reassignable on the board,
    so a forgiving prefix match is acceptable here."""
    lead = str(lead_value or '').strip().lower()
    if lead:
        row = conn.execute(
            "SELECT id FROM users WHERE is_active = 1 AND (lower(full_name) = ? OR lower(username) = ?) LIMIT 1",
            (lead, lead)
        ).fetchone()
        if row:
            return row['id']
        row = conn.execute(
            "SELECT id FROM users WHERE is_active = 1 AND (lower(full_name) LIKE ? OR lower(username) LIKE ?) "
            "ORDER BY length(full_name) LIMIT 1",
            (lead + '%', lead + '%')
        ).fetchone()
        if row:
            return row['id']
    return fallback_user_id


def reconcile_grid_pipeline_links(conn):
    """After a grid save + relational sync, archive (soft-delete) any pipeline
    opportunity whose linked grid investor row no longer exists — i.e. the investor was
    deleted from the grid. Creation is NEVER done here: a pipeline opp is only created
    via the explicit /api/fundraising/pipeline/link endpoint (which carries the seed
    fields), so this reconciler is a one-way orphan cleanup that can never spawn an
    empty opp or reseed a live one."""
    conn.execute(
        """
        UPDATE opportunities
        SET deleted_at = ?, updated_at = ?
        WHERE fundraising_investor_id IS NOT NULL
          AND deleted_at IS NULL
          AND fundraising_investor_id NOT IN (SELECT id FROM fundraising_investors)
        """,
        (now(), now())
    )


def reconcile_grid_reminders(conn):
    """After a grid save + relational sync, cancel any open/snoozed reminder whose linked
    grid investor row no longer exists (the investor was deleted from the grid) — the twin
    of reconcile_grid_pipeline_links. We CANCEL rather than soft-delete so the reminder stays
    visible in history with its denormalized investor_name; standalone reminders
    (investor_id IS NULL) are never touched."""
    conn.execute(
        """
        UPDATE reminders
        SET status = 'cancelled', updated_at = ?
        WHERE investor_id IS NOT NULL
          AND deleted_at IS NULL
          AND status IN ('open', 'snoozed')
          AND investor_id NOT IN (SELECT id FROM fundraising_investors)
        """,
        (now(),)
    )


def pipeline_stage_by_source_row(conn):
    """Return {grid source_row_id: current pipeline stage} for every investor with a
    live (non-deleted) linked opportunity. The opportunities table is the single source
    of truth, so this is always derived fresh and injected as read-only grid columns —
    never stored in the grid blob, where it could go stale."""
    out = {}
    # ORDER BY created_at DESC so the EARLIEST opp is processed last and wins the overwrite —
    # matching handle_pipeline_link's canonical choice (ORDER BY created_at LIMIT 1). The link
    # enforces one live opp per investor, so multi-opp is an out-of-band anomaly; this just keeps
    # the injection deterministic and consistent with opportunity_id_by_source_row if it occurs.
    for r in conn.execute(
        """
        SELECT fi.source_row_id AS srid, o.stage AS stage
        FROM opportunities o
        JOIN fundraising_investors fi ON o.fundraising_investor_id = fi.id
        WHERE o.deleted_at IS NULL
        ORDER BY o.created_at DESC
        """
    ).fetchall():
        srid = str(r['srid'] or '')
        if srid:
            out[srid] = r['stage']
    return out


def opportunity_id_by_source_row(conn):
    """Return {grid source_row_id: live opportunity id} for every investor with a non-deleted
    linked opportunity. Injected read-only alongside pipeline_stage so the mobile grid detail can
    PATCH /api/opportunities/{id}/stage directly (the same endpoint the Pipeline board uses) — the
    grid row otherwise carries no opp id. Never persisted; stripped on write like pipeline_stage."""
    out = {}
    # See pipeline_stage_by_source_row: earliest opp wins (ORDER BY created_at DESC + overwrite) so
    # opportunity_id and pipeline_stage always reference the SAME opp, matching the link's canonical pick.
    for r in conn.execute(
        """
        SELECT fi.source_row_id AS srid, o.id AS opp_id
        FROM opportunities o
        JOIN fundraising_investors fi ON o.fundraising_investor_id = fi.id
        WHERE o.deleted_at IS NULL
        ORDER BY o.created_at DESC
        """
    ).fetchall():
        srid = str(r['srid'] or '')
        if srid:
            out[srid] = r['opp_id']
    return out


def last_activity_by_investor(conn):
    """Return {fundraising_investors.id: latest activity ISO timestamp} across captured
    emails (grid-linked) and logged communications — the per-investor recency signal behind
    the reminders surface and (later) the "not nurtured in a while" query. The grid is the
    higher-signal side. Computed fresh on read (a few hundred investors); materialize only if
    it ever shows as slow. The email/comm tables can be absent on a minimal DB, so each leg
    is guarded independently."""
    out = {}
    def _bump(inv_id, ts):
        if inv_id and ts and (out.get(inv_id) is None or str(ts) > str(out[inv_id])):
            out[inv_id] = ts
    # Communications logged against any of the investor's people (via the grid contact link).
    try:
        for r in conn.execute(
            """
            SELECT fc.investor_id AS inv, MAX(cm.communication_date) AS last_ts
            FROM communications cm
            JOIN fundraising_contacts fc ON fc.contact_id = cm.contact_id
            WHERE cm.deleted_at IS NULL AND fc.contact_id IS NOT NULL
            GROUP BY fc.investor_id
            """
        ).fetchall():
            _bump(r["inv"], r["last_ts"])
    except sqlite3.OperationalError:
        pass
    # Captured emails matched to the grid investor (sent_at is set on matched + unmatched).
    try:
        for r in conn.execute(
            """
            SELECT eil.fundraising_investor_id AS inv, MAX(e.sent_at) AS last_ts
            FROM email_investor_links eil
            JOIN emails e ON e.id = eil.email_id
            WHERE eil.fundraising_investor_id IS NOT NULL
              -- soft-delete lives on the per-mailbox sighting, not emails/links; require a
              -- live sighting so a tombstoned email doesn't inflate recency (matches the digest).
              AND EXISTS (SELECT 1 FROM email_account_messages eam
                          WHERE eam.email_id = e.id AND eam.deleted_at IS NULL)
            GROUP BY eil.fundraising_investor_id
            """
        ).fetchall():
            _bump(r["inv"], r["last_ts"])
    except sqlite3.OperationalError:
        pass
    return out


REMINDER_DUE_SOON_DAYS = 7

def reminder_status_by_source_row(conn):
    """Return {grid source_row_id: 'overdue'|'due_soon'|'open'} for every investor with at
    least one open reminder, derived live from the reminders table (never stored in the grid
    blob — like pipeline_stage). Injected as a read-only grid column so a saved view can later
    filter "has an open reminder" instead of the binary follow_up checkbox. Most-urgent wins."""
    out = {}
    try:
        rows = conn.execute(
            """
            SELECT fi.source_row_id AS srid, r.due_date AS due
            FROM reminders r
            JOIN fundraising_investors fi ON r.investor_id = fi.id
            WHERE r.deleted_at IS NULL AND r.status = 'open'
            """
        ).fetchall()
    except sqlite3.OperationalError:
        return out
    today = now()[:10]
    soon = (datetime.utcnow() + timedelta(days=REMINDER_DUE_SOON_DAYS)).date().isoformat()
    rank = {'open': 0, 'due_soon': 1, 'overdue': 2}
    for r in rows:
        srid = str(r['srid'] or '')
        if not srid:
            continue
        due = str(r['due'] or '')[:10]
        if due and due < today:
            st = 'overdue'
        elif due and due <= soon:
            st = 'due_soon'
        else:
            st = 'open'
        if srid not in out or rank[st] > rank[out[srid]]:
            out[srid] = st
    return out


# Staleness ramp — one global threshold set (locked spec 2026-06-19): the last-contact recency
# value colors fresh (grey) -> aging (amber) >= STALE_AGING_DAYS -> stale (red) >= STALE_DAYS.
# Not stage-aware for v1. The same `staleness`/`last_activity_at` the grid injects also drives the
# mobile card and (a user-built) "Stale" view, so everything color-codes off one server signal.
STALE_AGING_DAYS = 30
STALE_DAYS = 60

def existing_investor_by_source_row(conn):
    """Return the set of grid source_row_ids whose investor has any committed capital
    (fundraising_investors.total_invested > 0) — the auto-derived "Existing Investor" flag
    (locked spec 2026-06-19). Injected read-only on grid read like pipeline_stage; never a
    maintained column. Orthogonal to stage: a re-solicited LP shows the star AND a live stage.
    Deliberately NOT filtered by `graveyard` (unlike the dashboard total-invested aggregate):
    committed capital makes an investor "existing" regardless of disposition; graveyard rows are
    muted/filtered by the view, not by this signal. Same intentional omission in staleness below."""
    out = set()
    for r in conn.execute(
        "SELECT source_row_id FROM fundraising_investors WHERE total_invested > 0"
    ).fetchall():
        srid = str(r['source_row_id'] or '')
        if srid:
            out.add(srid)
    return out


def fundraising_contact_emails_by_row(conn):
    """{ source_row_id: {'order': {sort_order: email}, 'name': {normalized_name: email}} } of the
    authoritative email per grid contact, for HEALING blank pill emails on read.

    The grid blob is canonical for the edit sheet, but an email can reach the linked classic
    contact (via email capture / a contact edit) without ever being written back into the blob
    pill — so the mobile "Edit investor" sheet shows an empty email for a contact the directory
    clearly has (Grant, 2026-06-20). We recover it from the relational mirror: prefer the synced
    fundraising_contacts.email, else the linked classic contacts.email (the source that actually
    holds the captured address). Keyed by sort_order (pills and fundraising_contacts share the
    blob order — the robust key) with a normalized-name fallback. Only non-blank emails are
    returned; filling is fill-only-when-blank in the handler, so it heals and converges (the next
    one-row save persists the recovered email into the blob)."""
    out = {}
    rows = conn.execute(
        """
        SELECT fi.source_row_id AS srid, fc.sort_order AS so, fc.full_name AS name,
               COALESCE(NULLIF(TRIM(fc.email), ''), c.email) AS email
        FROM fundraising_investors fi
        JOIN fundraising_contacts fc ON fc.investor_id = fi.id
        LEFT JOIN contacts c ON c.id = fc.contact_id AND c.deleted_at IS NULL
        """
    ).fetchall()
    for r in rows:
        email = str(r['email'] or '').strip()
        if not email:
            continue
        srid = str(r['srid'] or '')
        if not srid:
            continue
        bucket = out.setdefault(srid, {'order': {}, 'name': {}})
        if r['so'] is not None:
            bucket['order'][int(r['so'])] = email
        nm = _normalize_text(r['name'])
        if nm:
            bucket['name'][nm] = email
    return out


def contact_grid_signals(conn, contact_id=None):
    """Return {contacts.id: {'committed': float, 'pipeline_stage': str|None, 'priority': bool}} for
    every classic contact linked to a fundraising-grid investor (via fundraising_contacts.contact_id,
    migration 0004). Surfaces the canonical investor's committed rollup (total_invested → the mobile
    Contacts card's existing-LP avatar ring, committed > 0, mirroring existing_investor_by_source_row),
    its live derived pipeline stage (→ the card's stage pill), and its priority flag (→ the mobile
    Contacts Priority sort, 8d). Derived fresh on read like the grid's
    injected columns — never stored on the contact. A contact with no grid link gets nothing (a pure
    classic/legacy contact is not an investor). The grid relational tables are rebuilt from the blob
    on each save (no soft-delete axis), so no deleted_at filter is needed on the join — same basis as
    existing_investor_by_source_row. Pass `contact_id` to score a single contact (the detail path),
    avoiding a scan of the whole directory the list path needs."""
    out = {}
    where = "WHERE fc.contact_id IS NOT NULL"
    params = ()
    if contact_id is not None:
        where += " AND fc.contact_id = ?"
        params = (contact_id,)
    try:
        rows = conn.execute(
            f"""
            SELECT fc.contact_id AS cid, fi.total_invested AS committed, fi.source_row_id AS srid,
                   fi.priority AS priority
            FROM fundraising_contacts fc
            JOIN fundraising_investors fi ON fc.investor_id = fi.id
            {where}
            """,
            params,
        ).fetchall()
    except sqlite3.OperationalError:
        return out
    stage_by_srid = pipeline_stage_by_source_row(conn)
    for r in rows:
        cid = str(r['cid'] or '')
        if not cid:
            continue
        committed = float(r['committed'] or 0)
        prev = out.get(cid)
        # A contact normally links to exactly one investor; if it links to several, keep the
        # highest-committed one (its stage + priority) so the signals reflect the strongest link.
        if prev is None or committed > prev['committed']:
            out[cid] = {'committed': committed, 'pipeline_stage': stage_by_srid.get(str(r['srid'] or '')),
                        'priority': bool(r['priority']),
                        # None (not '') when unset, matching the caller's `... if sig else None`;
                        # source_row_id is NOT NULL on real investors so this is belt-and-suspenders.
                        'source_row_id': str(r['srid']) if r['srid'] else None}
    return out


def staleness_by_source_row(conn):
    """Return {grid source_row_id: (last_activity_iso_or_None, staleness)} where staleness is
    '' (fresh or no recorded activity), 'aging' (>= STALE_AGING_DAYS since last contact), or
    'stale' (>= STALE_DAYS). Derived from the SAME last_activity_by_investor signal the reminders
    surface uses, so desktop grid + mobile card color-code identically. A row with no recorded
    activity gets '' (no false "stale" on a brand-new lead); the W1b nurture-gap nudge handles
    in-pipeline-with-no-activity separately."""
    last_by_inv = last_activity_by_investor(conn)
    out = {}
    today = datetime.utcnow().date()
    for r in conn.execute("SELECT id, source_row_id FROM fundraising_investors").fetchall():
        srid = str(r['source_row_id'] or '')
        if not srid:
            continue
        ts = last_by_inv.get(r['id'])
        level = ''
        if ts:
            try:
                d = datetime.strptime(str(ts)[:10], '%Y-%m-%d').date()
                age = (today - d).days
                if age >= STALE_DAYS:
                    level = 'stale'
                elif age >= STALE_AGING_DAYS:
                    level = 'aging'
            except ValueError:
                pass
        out[srid] = (ts, level)
    return out


def maybe_run_scheduled_backup():
    conn = get_db()
    try:
        ensure_fundraising_state_row(conn)
        policy = load_backup_policy(conn)
        if not policy.get('enabled'):
            return
        interval = timedelta(hours=int(policy.get('interval_hours') or 24))
        last_run = parse_iso_utc(policy.get('last_run_at'))
        current = datetime.utcnow()
        if last_run and (current - last_run.replace(tzinfo=None)) < interval:
            return
        row = conn.execute("SELECT * FROM fundraising_state WHERE id = 'main'").fetchone()
        if not row:
            return
        create_fundraising_backup_file(row, kind="auto")
        policy['last_run_at'] = now()
        save_backup_policy(conn, policy)
        apply_backup_retention(policy)
        conn.commit()
    finally:
        conn.close()

def start_backup_scheduler():
    def _loop():
        while True:
            try:
                maybe_run_scheduled_backup()
            except Exception as exc:
                print(f"[scheduler] backup check failed: {exc}")
            time.sleep(60)
    thread = threading.Thread(target=_loop, daemon=True)
    thread.start()

# ─── Request Handler ──────────────────────────────────────────────────────────

# 4-stage per-investor funnel, terminal at 'commitment' (locked spec 2026-06-19). On commit the
# deal is handed to fund admin + the $ recorded in the grid fund cell — there is no 'funded'/'lost'
# stage (the grid's committed $ and the graveyard flag carry those). Migration 0007 remapped the
# legacy 6-stage values. Keep the frontend kanban + opp-form + nl_query rank in sync with this list.
PIPELINE_STAGES = ['lead', 'engaged', 'diligence', 'commitment']
CONTACT_TYPES = ['investor', 'prospect', 'advisor', 'other']
COMM_TYPES = ['email', 'call', 'meeting', 'note', 'text']

DEFAULT_GRID_VIEWS = [
    {"id": "view-main", "name": "Main Fundraising", "filters": {"includeGraveyard": False, "graveyardOnly": False, "followUpOnly": False, "lead": ""}, "quickSearch": "", "hiddenColumns": [], "columnFilters": []},
    {"id": "view-followup", "name": "Follow-up List", "filters": {"includeGraveyard": False, "graveyardOnly": False, "followUpOnly": True, "lead": ""}, "quickSearch": "", "hiddenColumns": [], "columnFilters": []},
    {"id": "view-graveyard", "name": "Graveyard", "filters": {"includeGraveyard": True, "graveyardOnly": True, "followUpOnly": False, "lead": ""}, "quickSearch": "", "hiddenColumns": [], "columnFilters": []},
    {"id": "view-all", "name": "All Investors", "filters": {"includeGraveyard": True, "graveyardOnly": False, "followUpOnly": False, "lead": ""}, "quickSearch": "", "hiddenColumns": [], "columnFilters": []}
]

DEFAULT_FUNDRAISING_COLUMNS = [
    {"id": "investor_name", "label": "Investor Name", "type": "text", "width": 220},
    {"id": "contacts", "label": "Contacts", "type": "contacts", "width": 260},
    {"id": "log_action", "label": "Log", "type": "action", "readOnly": True, "width": 90},
    {"id": "notes", "label": "Notes / Communication / Outreach", "type": "longtext", "width": 420},
    {"id": "lead_source", "label": "Lead Source", "type": "text", "width": 180},
    {"id": "notes_last_modified", "label": "Notes Last Modified", "type": "date", "readOnly": True, "width": 180},
    {"id": "last_communication_date", "label": "Last Communication Date", "type": "date", "readOnly": True, "width": 195},
    {"id": "priority", "label": "Priority", "type": "checkbox", "width": 110},
    {"id": "follow_up", "label": "Follow up", "type": "checkbox", "width": 110},
    {"id": "lead", "label": "Lead", "type": "select", "options": ["JK", "Grant", "MB", "Parker", "Other"], "width": 130},
    {"id": "graveyard", "label": "Graveyard", "type": "checkbox", "width": 115},
    {"id": "fund_i", "label": "Fund I", "type": "currency", "isFund": True, "width": 130},
    {"id": "fund_ii", "label": "Fund II", "type": "currency", "isFund": True, "width": 130},
    {"id": "fund_iii", "label": "Fund III", "type": "currency", "isFund": True, "width": 130},
    {"id": "tactical_fund", "label": "Tactical Fund", "type": "currency", "isFund": True, "width": 140},
    {"id": "pawn_to_e4", "label": "Pawn to E4", "type": "currency", "isFund": True, "width": 130},
    {"id": "ten31_terahash", "label": "Ten31 Terahash", "type": "currency", "isFund": True, "width": 150},
    {"id": "sats_and_stats", "label": "Sats and Stats", "type": "currency", "isFund": True, "width": 140},
    {"id": "pawn_to_f4", "label": "Pawn to f4", "type": "currency", "isFund": True, "width": 130},
    {"id": "join_the_fold", "label": "Join the Fold", "type": "currency", "isFund": True, "width": 130},
    {"id": "total_invested", "label": "Total invested", "type": "rollup", "readOnly": True, "width": 150},
    {"id": "tactical_fund_commit_date", "label": "Tactical Fund Commit Date", "type": "date", "width": 180}
]

DEFAULT_FUNDRAISING_ROWS = []

class CRMHandler(BaseHTTPRequestHandler):
    """Main HTTP request handler for the CRM API."""
    # Class-level state shared across all handler threads. Protected by
    # _abuse_lock; see rate_limited() and record_404() for usage.
    _rate_limit_buckets = {}   # (scope, ip) -> [timestamps]
    _404_buckets = {}          # ip -> [timestamps] of recent 404 responses
    _banned_ips = {}           # ip -> ban_until_epoch
    _abuse_lock = threading.Lock()

    def log_message(self, format, *args):
        """Override to use cleaner logging."""
        sys.stderr.write(f"[{datetime.now().strftime('%H:%M:%S')}] {args[0]}\n")

    # ── Request Parsing ──

    def get_body(self):
        # Cache parsed body on the request handler instance so repeated
        # calls don't try to re-read an already-consumed stream. Handler
        # instances are one-per-request in ThreadingHTTPServer, so the
        # cache is naturally request-scoped.
        if hasattr(self, '_cached_body'):
            return self._cached_body
        content_length = int(self.headers.get('Content-Length', 0))
        if content_length == 0:
            self._cached_body = {}
            return self._cached_body
        body = self.rfile.read(content_length)
        try:
            self._cached_body = json.loads(body.decode('utf-8'))
        except json.JSONDecodeError:
            self._cached_body = {}
        return self._cached_body

    def get_query_params(self):
        parsed = urlparse(self.path)
        return {k: v[0] if len(v) == 1 else v for k, v in parse_qs(parsed.query).items()}

    def get_path(self):
        return urlparse(self.path).path

    def get_user(self):
        auth = self.headers.get('Authorization', '')
        if not auth.startswith('Bearer '):
            return None
        token = auth[7:]
        payload = decode_token(token)
        if not payload:
            return None
        user_id = str(payload.get('user_id') or '').strip()
        if not user_id:
            return None
        conn = get_db()
        row = conn.execute(
            "SELECT id as user_id, username, role, is_active FROM users WHERE id = ?",
            (user_id,)
        ).fetchone()
        conn.close()
        if not row or int(row['is_active'] or 0) != 1:
            return None
        return {
            "user_id": row['user_id'],
            "username": row['username'],
            "role": row['role']
        }

    def get_client_ip(self):
        fwd = self.headers.get('X-Forwarded-For', '')
        if fwd:
            return fwd.split(',')[0].strip()
        return str(self.client_address[0])

    def rate_limited(self, scope, limit_per_minute):
        now_ts = time.time()
        bucket_key = f"{scope}:{self.get_client_ip()}"
        with self._abuse_lock:
            bucket = self._rate_limit_buckets.get(bucket_key, [])
            cutoff = now_ts - 60.0
            bucket = [t for t in bucket if t >= cutoff]
            if len(bucket) >= max(1, int(limit_per_minute)):
                self._rate_limit_buckets[bucket_key] = bucket
                return True
            bucket.append(now_ts)
            self._rate_limit_buckets[bucket_key] = bucket
            return False

    def is_banned(self):
        """Return True if the client IP is currently in the abuse blacklist.

        Uses a coarse class-level dict — fine for a handful of scanners hitting
        a small team CRM. Auto-expires entries when their ban window passes.
        """
        ip = self.get_client_ip()
        now_ts = time.time()
        with self._abuse_lock:
            until = self._banned_ips.get(ip)
            if until is None:
                return False
            if now_ts >= until:
                self._banned_ips.pop(ip, None)
                return False
            return True

    def record_404(self):
        """Track 404s per IP and auto-ban IPs that exceed the burst threshold.

        Called from send_error_json whenever we send a 404. A scanner probing
        /.env, /.git/config, /swagger, /actuator/env etc. will trip this fast
        and get parked on the blacklist for ABUSE_BAN_SEC seconds.
        """
        ip = self.get_client_ip()
        now_ts = time.time()
        with self._abuse_lock:
            bucket = self._404_buckets.get(ip, [])
            cutoff = now_ts - ABUSE_404_WINDOW_SEC
            bucket = [t for t in bucket if t >= cutoff]
            bucket.append(now_ts)
            if len(bucket) >= ABUSE_404_THRESHOLD:
                self._banned_ips[ip] = now_ts + ABUSE_BAN_SEC
                self._404_buckets.pop(ip, None)
                sys.stderr.write(
                    f"[abuse] Banning {ip} for {ABUSE_BAN_SEC}s after "
                    f"{len(bucket)} 404s in {ABUSE_404_WINDOW_SEC}s\n"
                )
            else:
                self._404_buckets[ip] = bucket

    # ── Response Helpers ──

    def send_json(self, data, status=200):
        self.send_response(status)
        self.send_header('Content-Type', 'application/json')
        self.send_header('Access-Control-Allow-Origin', CORS_ORIGIN)
        self.send_header('Access-Control-Allow-Methods', 'GET, POST, PUT, PATCH, DELETE, OPTIONS')
        self.send_header('Access-Control-Allow-Headers', 'Content-Type, Authorization')
        self.end_headers()
        self.wfile.write(json.dumps(data, default=str).encode('utf-8'))

    def send_error_json(self, message, status=400):
        # Record 404s for the abuser-detection blacklist before responding.
        # We do this on send (not on routing) so that any code path that 404s
        # contributes to the burst counter, including unknown POST paths.
        if status == 404:
            try:
                self.record_404()
            except Exception:
                pass
        self.send_json({"error": message}, status)

    def send_file(self, filepath, content_type='text/html'):
        try:
            with open(filepath, 'rb') as f:
                content = f.read()
            self.send_response(200)
            self.send_header('Content-Type', content_type)
            self.send_header('Content-Length', str(len(content)))
            self.end_headers()
            self.wfile.write(content)
        except FileNotFoundError:
            self.send_error_json("File not found", 404)

    # ── Routing ──

    def do_OPTIONS(self):
        if self.is_banned():
            return self.send_error_json("Too many requests", 429)
        self.send_response(200)
        self.send_header('Access-Control-Allow-Origin', CORS_ORIGIN)
        self.send_header('Access-Control-Allow-Methods', 'GET, POST, PUT, PATCH, DELETE, OPTIONS')
        self.send_header('Access-Control-Allow-Headers', 'Content-Type, Authorization')
        self.end_headers()

    def do_GET(self):
        # Short-circuit known abusers before any work, including file reads.
        if self.is_banned():
            return self.send_error_json("Too many requests", 429)
        # Generic per-IP GET rate limit. Generous enough for a normal user
        # (page load fans out ~15 GETs + heartbeats every ~6s) but blocks a
        # scanner blasting hundreds of paths per second.
        if self.rate_limited('get', GET_RATE_LIMIT_PER_MIN):
            return self.send_error_json("Too many requests", 429)

        path = self.get_path()

        # ─── Gmail integration routes (feature-flag-guarded) ─────────
        try:
            from email_integration.routes import try_handle as _email_try_handle
            if _email_try_handle(self):
                return
        except ImportError:
            pass

        # Serve frontend
        if path == '/' or path == '/index.html':
            return self.send_file(os.path.join(FRONTEND_DIR, 'index.html'))
        # PWA manifest — served at root (manifest scope = its own path's directory) and
        # pre-auth, since the browser fetches it (and the icons under /assets/) before login
        # to offer "Add to Home Screen". No service worker (iOS-first; see ROADMAP).
        if path == '/manifest.webmanifest':
            return self.send_file(os.path.join(FRONTEND_DIR, 'manifest.webmanifest'), 'application/manifest+json')
        if path.startswith('/assets/'):
            filepath = os.path.join(FRONTEND_DIR, path.lstrip('/'))
            # Containment check: get_path()/urlparse does NOT normalize '..', so without
            # this an unauthenticated GET /assets/../../data/crm.db (raw client) would read
            # any file the process can — the LP DB, the JWT secret, the Gmail key. Resolve
            # and require the target stay under FRONTEND_ROOT; 404 (not 403) so it looks like
            # any other miss and still trips the scanner abuse counter.
            _real = os.path.realpath(filepath)
            if _real != FRONTEND_ROOT and not _real.startswith(FRONTEND_ROOT + os.sep):
                return self.send_error_json("File not found", 404)
            ext = os.path.splitext(path)[1].lower()
            content_types = {
                '.css': 'text/css',
                '.js': 'application/javascript',
                '.svg': 'image/svg+xml',
                '.png': 'image/png',
                '.jpg': 'image/jpeg',
                '.jpeg': 'image/jpeg',
                '.gif': 'image/gif',
                '.webp': 'image/webp',
                '.ico': 'image/x-icon',
            }
            ct = content_types.get(ext, 'application/octet-stream')
            return self.send_file(filepath, ct)

        # API routes
        if path == '/api/health':
            return self.send_json({"status": "ok", "timestamp": now()})
        if path == '/api/bootstrap/status':
            return self.handle_bootstrap_status()

        # Auth required from here
        user = self.get_user()
        if not user and path.startswith('/api/'):
            return self.send_error_json("Authentication required", 401)

        params = self.get_query_params()

        # Contacts
        if path == '/api/contacts':
            return self.handle_list_contacts(user, params)
        if re.match(r'^/api/contacts/[^/]+$', path):
            contact_id = path.split('/')[-1]
            return self.handle_get_contact(user, contact_id)
        if re.match(r'^/api/contacts/[^/]+/communications$', path):
            contact_id = path.split('/')[-2]
            return self.handle_list_contact_communications(user, contact_id, params)

        # Organizations
        if path == '/api/organizations':
            return self.handle_list_organizations(user, params)
        if re.match(r'^/api/organizations/[^/]+$', path):
            org_id = path.split('/')[-1]
            return self.handle_get_organization(user, org_id)

        # Opportunities
        if path == '/api/opportunities':
            return self.handle_list_opportunities(user, params)
        if re.match(r'^/api/opportunities/[^/]+$', path):
            opp_id = path.split('/')[-1]
            return self.handle_get_opportunity(user, opp_id)

        # Communications
        if path == '/api/communications':
            return self.handle_list_communications(user, params)

        # Reports
        if path == '/api/reports/dashboard':
            return self.handle_dashboard_report(user)
        if path == '/api/reports/pipeline':
            return self.handle_pipeline_report(user)
        if path == '/api/reports/activity':
            return self.handle_activity_report(user, params)

        # Export
        if path == '/api/export/contacts':
            return self.handle_export_contacts(user, params)

        # Fundraising grid state
        if path == '/api/fundraising/state':
            return self.handle_get_fundraising_state(user)
        if path == '/api/fundraising/collab/state':
            return self.handle_get_fundraising_collab_state(user)
        if path == '/api/fundraising/export':
            return self.handle_export_fundraising_state(user)
        if path == '/api/fundraising/backups':
            return self.handle_list_fundraising_backups(user)
        if path == '/api/fundraising/backup-policy':
            return self.handle_get_backup_policy(user)
        if path == '/api/admin/digest/policy':
            return self.handle_get_digest_policy(user)
        if path == '/api/admin/soft-deleted':
            return self.handle_list_soft_deleted(user)
        if path == '/api/admin/contacts-census':   # TEMPORARY — remove after the consolidation census
            return self.handle_contacts_census(user)
        if path == '/api/fundraising/relational-summary':
            return self.handle_get_fundraising_relational_summary(user)
        if path == '/api/fundraising/automations':
            return self.handle_list_fundraising_automations(user)
        if path == '/api/fundraising/automation-runs':
            return self.handle_list_fundraising_automation_runs(user, params)
        if path == '/api/fundraising/activity':
            return self.handle_get_fundraising_activity(user, params)
        if path == '/api/security/status':
            return self.handle_security_status(user)
        if path == '/api/system/status':
            return self.handle_system_status(user)
        if path == '/api/activity/proposals':
            return self.handle_list_activity_proposals(user)
        if path == '/api/outreach/investors':
            return self.handle_list_outreach_investors(user)
        if path == '/api/outreach/radar':
            return self.handle_outreach_radar(user)

        # Reminders / follow-ups
        if path == '/api/reminders':
            return self.handle_list_reminders(user, params)

        # Natural-language query (W2) — the askable catalog
        if path == '/api/query/catalog':
            return self.handle_nl_query_catalog(user)

        # Matrix intake bot — new-vs-existing lookup for its in-thread proposal
        if path == '/api/intake/match':
            return self.handle_intake_match(user, params)
        # Matrix review bot — email-activity proposal work-lists (to_post/open/to_close)
        if path == '/api/intake/email-proposals':
            return self.handle_list_bot_email_proposals(user)

        # Users
        if path == '/api/users':
            return self.handle_list_users(user)

        # Audit log
        if path == '/api/audit-log':
            return self.handle_list_audit_log(user, params)

        # ─── Architect thesis (Phase 1) ───
        if path == '/api/thesis/lines':
            return self.handle_list_thesis_lines(user)
        if path == '/api/thesis/versions':
            return self.handle_list_thesis_review_queue(user)
        if re.match(r'^/api/thesis/versions/[^/]+$', path):
            return self.handle_get_thesis_version(user, path.split('/')[-1])
        if re.match(r'^/api/thesis/[^/]+/canonical$', path):
            return self.handle_get_canonical_thesis(user, path.split('/')[-2])
        if path == '/api/architect/status':
            return self.handle_architect_status(user)
        if re.match(r'^/api/thesis/nodes/[^/]+/variants$', path):
            return self.handle_get_node_variants(user, path.split('/')[-2])
        if re.match(r'^/api/thesis/[^/]+/tree$', path):
            return self.handle_get_thesis_tree(user, path.split('/')[-2])

        # ─── Entity-merge review queue ───
        if path == '/api/entities/merge-candidates':
            return self.handle_list_merge_candidates(user, params)

        self.send_error_json("Not found", 404)

    def do_POST(self):
        if self.is_banned():
            return self.send_error_json("Too many requests", 429)

        path = self.get_path()
        body = self.get_body()

        if self.rate_limited('write', WRITE_RATE_LIMIT_PER_MIN):
            return self.send_error_json("Too many requests", 429)

        # ─── Gmail integration routes (feature-flag-guarded) ─────────
        try:
            from email_integration.routes import try_handle as _email_try_handle
            if _email_try_handle(self):
                return
        except ImportError:
            pass

        # Auth (no token needed)
        if path == '/api/auth/login':
            if self.rate_limited('login', LOGIN_RATE_LIMIT_PER_MIN):
                return self.send_error_json("Too many login attempts", 429)
            return self.handle_login(body)
        if path == '/api/auth/register':
            return self.handle_register(body)

        # Auth required
        user = self.get_user()
        if not user:
            return self.send_error_json("Authentication required", 401)

        if path == '/api/contacts':
            return self.handle_create_contact(user, body)
        if path == '/api/organizations':
            return self.handle_create_organization(user, body)
        if path == '/api/opportunities':
            return self.handle_create_opportunity(user, body)
        if path == '/api/communications':
            return self.handle_create_communication(user, body)
        if path == '/api/import/csv':
            return self.handle_import_csv(user, body)
        if path == '/api/fundraising/log-communication':
            return self.handle_log_fundraising_communication(user, body)
        if path == '/api/fundraising/update-row':
            return self.handle_update_fundraising_row(user, body)
        if path == '/api/fundraising/pipeline/link':
            return self.handle_pipeline_link(user, body)
        if path == '/api/fundraising/pipeline/unlink':
            return self.handle_pipeline_unlink(user, body)
        if path == '/api/reminders':
            return self.handle_create_reminder(user, body)
        if path == '/api/query/nl':
            return self.handle_nl_query(user, body)
        if path == '/api/intake/card':
            return self.handle_intake_card(user, body)
        if path == '/api/fundraising/collab/heartbeat':
            return self.handle_fundraising_collab_heartbeat(user, body)
        if path == '/api/admin/users':
            return self.handle_admin_create_user(user, body)
        if path == '/api/admin/reset-all-data':
            return self.handle_admin_reset_all_data(user, body)
        if path == '/api/admin/soft-deleted/purge':
            return self.handle_purge_soft_deleted(user, body)
        if path == '/api/admin/digest/test-email':
            return self.handle_admin_send_test_email(user, body)
        if path == '/api/admin/digest/send-now':
            return self.handle_admin_send_digest_now(user, body)
        if path == '/api/admin/digest/preview':
            return self.handle_admin_digest_preview(user, body)
        if path == '/api/fundraising/backup':
            return self.handle_backup_fundraising_state(user)
        if path == '/api/fundraising/restore-preview':
            return self.handle_preview_fundraising_restore(user, body)
        if path == '/api/fundraising/restore':
            return self.handle_restore_fundraising_state(user, body)
        if path == '/api/fundraising/backup-verify':
            return self.handle_verify_fundraising_backups(user)

        # ─── Architect thesis review (Phase 1, human approval gate) ───
        if re.match(r'^/api/thesis/versions/[^/]+/review$', path):
            return self.handle_thesis_review(user, path.split('/')[-2], body)
        # ─── Architect generation (Claude) ───
        if re.match(r'^/api/thesis/nodes/[^/]+/generate$', path):
            return self.handle_generate_options(user, path.split('/')[-2], body)
        if re.match(r'^/api/thesis/nodes/[^/]+/feedback$', path):
            return self.handle_node_feedback(user, path.split('/')[-2], body)
        if path == '/api/architect/ground':
            return self.handle_architect_ground(user, body)
        if path == '/api/outreach/draft':
            return self.handle_outreach_draft(user, body)
        if path == '/api/outreach/gmail-draft':
            return self.handle_outreach_gmail_draft(user, body)
        if re.match(r'^/api/activity/proposals/[^/]+/approve$', path):
            return self.handle_decide_activity_proposal(user, path.split('/')[-2], 'approve', body)
        if re.match(r'^/api/activity/proposals/[^/]+/dismiss$', path):
            return self.handle_decide_activity_proposal(user, path.split('/')[-2], 'dismiss', body)
        if re.match(r'^/api/intake/email-proposals/[^/]+/matrix$', path):
            return self.handle_mark_email_proposal_matrix(user, path.split('/')[-2], body)
        if re.match(r'^/api/intake/email-proposals/[^/]+/decide$', path):
            return self.handle_decide_email_proposal_matrix(user, path.split('/')[-2], body)
        if re.match(r'^/api/thesis/nodes/[^/]+/choose$', path):
            return self.handle_choose_variant(user, path.split('/')[-2])
        if re.match(r'^/api/thesis/lines/[^/]+/approve$', path):
            return self.handle_approve_line(user, path.split('/')[-2])
        if path == '/api/thesis/lines':
            return self.handle_create_thesis_line(user, body)
        if re.match(r'^/api/thesis/lines/[^/]+/nodes$', path):
            return self.handle_add_thesis_node(user, path.split('/')[-2], body)

        # ─── UI-triggered index jobs + entity-merge decisions (Phase 1) ───
        if path == '/api/index/rebuild':
            return self.handle_index_job(user, 'rebuild_index')
        if path == '/api/index/update':
            return self.handle_index_job(user, 'update_index')
        if path == '/api/entities/find-duplicates':
            return self.handle_index_job(user, 'find_duplicates')
        if re.match(r'^/api/entities/merge-candidates/[^/]+$', path):
            return self.handle_decide_merge_candidate(user, path.split('/')[-1], body)

        self.send_error_json("Not found", 404)

    def do_PUT(self):
        if self.is_banned():
            return self.send_error_json("Too many requests", 429)
        path = self.get_path()
        body = self.get_body()
        if self.rate_limited('write', WRITE_RATE_LIMIT_PER_MIN):
            return self.send_error_json("Too many requests", 429)
        user = self.get_user()
        if not user:
            return self.send_error_json("Authentication required", 401)

        if re.match(r'^/api/contacts/[^/]+$', path):
            return self.handle_update_contact(user, path.split('/')[-1], body)
        if re.match(r'^/api/organizations/[^/]+$', path):
            return self.handle_update_organization(user, path.split('/')[-1], body)
        if re.match(r'^/api/opportunities/[^/]+$', path):
            return self.handle_update_opportunity(user, path.split('/')[-1], body)
        if re.match(r'^/api/communications/[^/]+$', path):
            return self.handle_update_communication(user, path.split('/')[-1], body)
        if path == '/api/fundraising/state':
            return self.handle_update_fundraising_state(user, body)
        if re.match(r'^/api/thesis/nodes/[^/]+$', path):
            return self.handle_edit_thesis_node(user, path.split('/')[-1], body)

        self.send_error_json("Not found", 404)

    def do_PATCH(self):
        if self.is_banned():
            return self.send_error_json("Too many requests", 429)
        path = self.get_path()
        body = self.get_body()
        if self.rate_limited('write', WRITE_RATE_LIMIT_PER_MIN):
            return self.send_error_json("Too many requests", 429)
        user = self.get_user()
        if not user:
            return self.send_error_json("Authentication required", 401)

        if re.match(r'^/api/opportunities/[^/]+/stage$', path):
            opp_id = path.split('/')[-2]
            return self.handle_update_stage(user, opp_id, body)
        if re.match(r'^/api/reminders/[^/]+$', path):
            return self.handle_update_reminder(user, path.split('/')[-1], body)
        if re.match(r'^/api/admin/users/[^/]+$', path):
            target_user_id = path.split('/')[-1]
            return self.handle_admin_update_user(user, target_user_id, body)
        if path == '/api/fundraising/backup-policy':
            return self.handle_update_backup_policy(user, body)
        if path == '/api/admin/digest/policy':
            return self.handle_update_digest_policy(user, body)
        if re.match(r'^/api/fundraising/automations/[^/]+$', path):
            rule_id = path.split('/')[-1]
            return self.handle_update_fundraising_automation_rule(user, rule_id, body)

        self.send_error_json("Not found", 404)

    def do_DELETE(self):
        if self.is_banned():
            return self.send_error_json("Too many requests", 429)
        path = self.get_path()
        if self.rate_limited('write', WRITE_RATE_LIMIT_PER_MIN):
            return self.send_error_json("Too many requests", 429)
        user = self.get_user()
        if not user:
            return self.send_error_json("Authentication required", 401)

        if re.match(r'^/api/contacts/[^/]+$', path):
            return self.handle_delete_contact(user, path.split('/')[-1])
        if re.match(r'^/api/organizations/[^/]+$', path):
            return self.handle_delete_organization(user, path.split('/')[-1])
        if re.match(r'^/api/opportunities/[^/]+$', path):
            return self.handle_delete_opportunity(user, path.split('/')[-1])
        if re.match(r'^/api/communications/[^/]+$', path):
            return self.handle_delete_communication(user, path.split('/')[-1])
        if re.match(r'^/api/reminders/[^/]+$', path):
            return self.handle_delete_reminder(user, path.split('/')[-1])
        if re.match(r'^/api/thesis/nodes/[^/]+$', path):
            return self.handle_retire_thesis_node(user, path.split('/')[-1])
        self.send_error_json("Not found", 404)

    # ═══════════════════════════════════════════════════════════════════════════
    # AUTH HANDLERS
    # ═══════════════════════════════════════════════════════════════════════════

    def handle_bootstrap_status(self):
        conn = get_db()
        count = int(conn.execute("SELECT COUNT(*) as c FROM users").fetchone()['c'])
        conn.close()
        return self.send_json({
            "data": {
                "user_count": count,
                "setup_required": count == 0
            }
        })

    def handle_login(self, body):
        username = body.get('username', '').strip()
        password = body.get('password', '')
        if not username or not password:
            return self.send_error_json("Username and password required")

        conn = get_db()
        user = conn.execute("SELECT * FROM users WHERE username = ? AND is_active = 1", (username,)).fetchone()
        conn.close()

        if not user or not verify_password(password, user['password_hash']):
            return self.send_error_json("Invalid credentials", 401)

        token = create_token(user['id'], user['username'], user['role'])
        return self.send_json({
            "token": token,
            "user": {
                "id": user['id'],
                "username": user['username'],
                "email": user['email'],
                "full_name": user['full_name'],
                "role": user['role']
            }
        })

    def handle_register(self, body):
        required = ['username', 'password', 'email', 'full_name']
        for field in required:
            if not body.get(field, '').strip():
                return self.send_error_json(f"{field} is required")
        if len(str(body.get('password') or '').strip()) < 8:
            return self.send_error_json("password must be at least 8 characters")

        conn = get_db()
        user_count = conn.execute("SELECT COUNT(*) as c FROM users").fetchone()['c']
        if user_count > 0:
            conn.close()
            return self.send_error_json("Registration is disabled. Ask an admin for an invite.", 403)

        existing = conn.execute("SELECT id FROM users WHERE username = ? OR email = ?",
                                (body['username'], body['email'])).fetchone()
        if existing:
            conn.close()
            return self.send_error_json("Username or email already exists")

        user_id = generate_id()
        # Bootstrap first user as admin
        role = 'admin' if user_count == 0 else body.get('role', 'member')

        conn.execute(
            "INSERT INTO users (id, username, email, password_hash, full_name, role) VALUES (?, ?, ?, ?, ?, ?)",
            (user_id, body['username'], body['email'], hash_password(body['password']), body['full_name'], role)
        )
        conn.commit()
        conn.close()

        token = create_token(user_id, body['username'], role)
        return self.send_json({
            "token": token,
            "user": {
                "id": user_id,
                "username": body['username'],
                "email": body['email'],
                "full_name": body['full_name'],
                "role": role
            }
        }, 201)

    # ═══════════════════════════════════════════════════════════════════════════
    # CONTACT HANDLERS
    # ═══════════════════════════════════════════════════════════════════════════

    def handle_list_contacts(self, user, params):
        conn = get_db()
        query = """
            SELECT c.*, o.name as organization_name,
                   (SELECT COUNT(*) FROM communications WHERE contact_id = c.id AND deleted_at IS NULL) as comm_count,
                   (SELECT MAX(communication_date) FROM communications WHERE contact_id = c.id AND deleted_at IS NULL) as last_contact_date
            FROM contacts c
            LEFT JOIN organizations o ON c.organization_id = o.id
            WHERE 1=1 AND c.deleted_at IS NULL
        """
        args = []

        if params.get('status'):
            query += " AND c.status = ?"
            args.append(params['status'])
        if params.get('search'):
            search = f"%{params['search']}%"
            query += " AND (c.first_name LIKE ? OR c.last_name LIKE ? OR c.email LIKE ? OR o.name LIKE ? OR c.source LIKE ?)"
            args.extend([search, search, search, search, search])
        if params.get('organization_id'):
            query += " AND c.organization_id = ?"
            args.append(params['organization_id'])
        if params.get('tag'):
            query += " AND c.tags LIKE ?"
            args.append(f'%"{params["tag"]}"%')

        sort = params.get('sort', 'updated_at')
        order = 'DESC' if params.get('order', 'desc').lower() == 'desc' else 'ASC'
        allowed_sorts = ['first_name', 'last_name', 'email', 'created_at', 'updated_at', 'source']
        if sort in allowed_sorts:
            query += f" ORDER BY c.{sort} {order}"
        else:
            query += f" ORDER BY c.updated_at DESC"

        # Pagination
        limit = min(int(params.get('limit', 50)), 500)
        offset = int(params.get('offset', 0))
        count_query = f"SELECT COUNT(*) as total FROM ({query})"
        total = conn.execute(count_query, args).fetchone()['total']

        query += " LIMIT ? OFFSET ?"
        args.extend([limit, offset])

        contacts = rows_to_list(conn.execute(query, args).fetchall())
        # Enrich with read-only, live-derived grid signals (committed → existing-LP avatar ring,
        # pipeline_stage → stage pill, source_row_id → the detail's "Open investor in Grid"
        # deep-link target) for the mobile Contacts card. Harmless extra fields for the desktop
        # page, which ignores them. Never persisted on the contact (the list is read-only).
        signals = contact_grid_signals(conn)
        for c in contacts:
            sig = signals.get(str(c.get('id') or ''))
            c['committed'] = sig['committed'] if sig else 0
            c['pipeline_stage'] = sig['pipeline_stage'] if sig else None
            c['priority'] = bool(sig['priority']) if sig else False
            c['source_row_id'] = sig['source_row_id'] if sig else None
        conn.close()

        return self.send_json({
            "data": contacts,
            "total": total,
            "limit": limit,
            "offset": offset
        })

    def handle_get_contact(self, user, contact_id):
        conn = get_db()
        contact = conn.execute("""
            SELECT c.*, o.name as organization_name
            FROM contacts c
            LEFT JOIN organizations o ON c.organization_id = o.id
            WHERE c.id = ? AND c.deleted_at IS NULL
        """, (contact_id,)).fetchone()

        if not contact:
            conn.close()
            return self.send_error_json("Contact not found", 404)

        result = row_to_dict(contact)

        # Get related data
        result['communications'] = rows_to_list(conn.execute(
            """SELECT cm.*, u.full_name as created_by_name
               FROM communications cm LEFT JOIN users u ON cm.created_by = u.id
               WHERE cm.contact_id = ? AND cm.deleted_at IS NULL ORDER BY cm.communication_date DESC LIMIT 20""",
            (contact_id,)
        ).fetchall())

        result['opportunities'] = rows_to_list(conn.execute(
            "SELECT * FROM opportunities WHERE contact_id = ? AND deleted_at IS NULL ORDER BY updated_at DESC",
            (contact_id,)
        ).fetchall())

        # Same read-only grid signals the list injects (committed → existing-LP ring/pill,
        # pipeline_stage → stage pill, source_row_id → "Open investor in Grid" deep-link), so the
        # mobile detail sheet can render them without a second round-trip. Derived live; never stored.
        sig = contact_grid_signals(conn, contact_id).get(contact_id)
        result['committed'] = sig['committed'] if sig else 0
        result['pipeline_stage'] = sig['pipeline_stage'] if sig else None
        result['priority'] = bool(sig['priority']) if sig else False
        result['source_row_id'] = sig['source_row_id'] if sig else None

        conn.close()
        return self.send_json({"data": result})

    def handle_create_contact(self, user, body):
        if not body.get('first_name') or not body.get('last_name'):
            return self.send_error_json("first_name and last_name are required")

        contact_id = generate_id()
        conn = get_db()
        organization_id = body.get('organization_id')
        if not organization_id and body.get('organization'):
            organization_id = _ensure_organization_by_name(conn, body.get('organization'), user['user_id'])

        tags = json.dumps(body.get('tags', []))
        conn.execute("""
            INSERT INTO contacts (id, first_name, last_name, email, phone, mobile, title,
                organization_id, status, source, tags, notes, linkedin_url,
                city, state, country, location_query, preferred_contact, created_by)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
        """, (
            contact_id, body['first_name'], body['last_name'],
            body.get('email'), body.get('phone'), body.get('mobile'),
            body.get('title'), organization_id,
            body.get('status', 'active'),
            body.get('source'), tags, body.get('notes'),
            body.get('linkedin_url'), body.get('city'), body.get('state'),
            body.get('country'), body.get('location_query'),
            body.get('preferred_contact', 'email'),
            user['user_id']
        ))

        contact = row_to_dict(conn.execute("""
            SELECT c.*, o.name as organization_name
            FROM contacts c LEFT JOIN organizations o ON c.organization_id = o.id
            WHERE c.id = ?
        """, (contact_id,)).fetchone())
        _sync_contact_to_fundraising_state(conn, contact, actor_user_id=user['user_id'], remove=False)
        log_audit(conn, user['user_id'], 'contact', contact_id, 'create')
        conn.commit()
        conn.close()
        return self.send_json({"data": contact}, 201)

    def handle_update_contact(self, user, contact_id, body):
        conn = get_db()
        existing = conn.execute("""
            SELECT c.*, o.name as organization_name
            FROM contacts c LEFT JOIN organizations o ON c.organization_id = o.id
            WHERE c.id = ?
        """, (contact_id,)).fetchone()
        if not existing:
            conn.close()
            return self.send_error_json("Contact not found", 404)
        previous_contact = row_to_dict(existing)

        updatable = ['first_name', 'last_name', 'email', 'phone', 'mobile', 'title',
                     'organization_id', 'status', 'source', 'notes',
                     'linkedin_url', 'city', 'state', 'country', 'location_query', 'preferred_contact']
        sets = []
        args = []
        for field in updatable:
            if field in body:
                sets.append(f"{field} = ?")
                args.append(body[field])
        if 'organization' in body and 'organization_id' not in body:
            org_id = _ensure_organization_by_name(conn, body.get('organization'), user['user_id'])
            sets.append("organization_id = ?")
            args.append(org_id)

        if 'tags' in body:
            sets.append("tags = ?")
            args.append(json.dumps(body['tags']))

        if not sets:
            conn.close()
            return self.send_error_json("No fields to update")

        sets.append("updated_at = ?")
        args.append(now())
        args.append(contact_id)

        conn.execute(f"UPDATE contacts SET {', '.join(sets)} WHERE id = ?", args)

        contact = row_to_dict(conn.execute("""
            SELECT c.*, o.name as organization_name
            FROM contacts c LEFT JOIN organizations o ON c.organization_id = o.id
            WHERE c.id = ?
        """, (contact_id,)).fetchone())
        _sync_contact_to_fundraising_state(conn, previous_contact, actor_user_id=user['user_id'], remove=True)
        _sync_contact_to_fundraising_state(conn, contact, actor_user_id=user['user_id'], remove=False)
        log_audit(conn, user['user_id'], 'contact', contact_id, 'update', body)
        conn.commit()
        conn.close()
        return self.send_json({"data": contact})

    def handle_delete_contact(self, user, contact_id):
        conn = get_db()
        existing = conn.execute("""
            SELECT c.*, o.name as organization_name
            FROM contacts c LEFT JOIN organizations o ON c.organization_id = o.id
            WHERE c.id = ?
        """, (contact_id,)).fetchone()
        if not existing:
            conn.close()
            return self.send_error_json("Contact not found", 404)

        _sync_contact_to_fundraising_state(conn, row_to_dict(existing), actor_user_id=user['user_id'], remove=True)
        # Soft-delete (guardrail #3 — never hard-delete): mark deleted_at and
        # cascade to the contact's opportunities and communications.
        _ts = now()
        conn.execute("UPDATE contacts SET deleted_at = ?, updated_at = ? WHERE id = ?", (_ts, _ts, contact_id))
        conn.execute("UPDATE opportunities SET deleted_at = ? WHERE contact_id = ? AND deleted_at IS NULL", (_ts, contact_id))
        conn.execute("UPDATE communications SET deleted_at = ? WHERE contact_id = ? AND deleted_at IS NULL", (_ts, contact_id))
        log_audit(conn, user['user_id'], 'contact', contact_id, 'delete')
        conn.commit()
        conn.close()
        return self.send_json({"message": "Contact deleted"})

    # ═══════════════════════════════════════════════════════════════════════════
    # ORGANIZATION HANDLERS
    # ═══════════════════════════════════════════════════════════════════════════

    def handle_list_organizations(self, user, params):
        conn = get_db()
        query = """
            SELECT o.*,
                   (SELECT COUNT(*) FROM contacts WHERE organization_id = o.id AND deleted_at IS NULL) as contact_count,
                   (SELECT COALESCE(SUM(commitment_amount), 0) FROM opportunities WHERE organization_id = o.id AND stage = 'commitment' AND deleted_at IS NULL) as total_funded
            FROM organizations o WHERE 1=1 AND o.deleted_at IS NULL
        """
        args = []
        if params.get('search'):
            search = f"%{params['search']}%"
            query += " AND (o.name LIKE ? OR o.industry LIKE ?)"
            args.extend([search, search])
        if params.get('type'):
            query += " AND o.type = ?"
            args.append(params['type'])

        query += " ORDER BY o.name ASC"
        limit = min(int(params.get('limit', 50)), 500)
        offset = int(params.get('offset', 0))
        total = conn.execute(f"SELECT COUNT(*) as total FROM ({query})", args).fetchone()['total']
        query += " LIMIT ? OFFSET ?"
        args.extend([limit, offset])

        orgs = rows_to_list(conn.execute(query, args).fetchall())
        conn.close()
        return self.send_json({"data": orgs, "total": total, "limit": limit, "offset": offset})

    def handle_get_organization(self, user, org_id):
        conn = get_db()
        org = conn.execute("SELECT * FROM organizations WHERE id = ? AND deleted_at IS NULL", (org_id,)).fetchone()
        if not org:
            conn.close()
            return self.send_error_json("Organization not found", 404)

        result = row_to_dict(org)
        result['contacts'] = rows_to_list(conn.execute(
            "SELECT * FROM contacts WHERE organization_id = ? AND deleted_at IS NULL ORDER BY last_name", (org_id,)
        ).fetchall())
        result['opportunities'] = rows_to_list(conn.execute(
            "SELECT * FROM opportunities WHERE organization_id = ? AND deleted_at IS NULL ORDER BY updated_at DESC", (org_id,)
        ).fetchall())
        conn.close()
        return self.send_json({"data": result})

    def handle_create_organization(self, user, body):
        if not body.get('name'):
            return self.send_error_json("name is required")

        org_id = generate_id()
        conn = get_db()
        tags = json.dumps(body.get('tags', []))
        conn.execute("""
            INSERT INTO organizations (id, name, type, industry, website, phone, email,
                address, city, state, country, description, tags, created_by)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
        """, (
            org_id, body['name'], body.get('type', 'other'), body.get('industry'),
            body.get('website'), body.get('phone'), body.get('email'),
            body.get('address'), body.get('city'), body.get('state'),
            body.get('country'), body.get('description'), tags, user['user_id']
        ))
        log_audit(conn, user['user_id'], 'organization', org_id, 'create')
        conn.commit()
        org = row_to_dict(conn.execute("SELECT * FROM organizations WHERE id = ?", (org_id,)).fetchone())
        conn.close()
        return self.send_json({"data": org}, 201)

    def handle_update_organization(self, user, org_id, body):
        conn = get_db()
        existing = conn.execute("SELECT id FROM organizations WHERE id = ?", (org_id,)).fetchone()
        if not existing:
            conn.close()
            return self.send_error_json("Organization not found", 404)

        updatable = ['name', 'type', 'industry', 'website', 'phone', 'email',
                     'address', 'city', 'state', 'country', 'description']
        sets = []
        args = []
        for field in updatable:
            if field in body:
                sets.append(f"{field} = ?")
                args.append(body[field])
        if 'tags' in body:
            sets.append("tags = ?")
            args.append(json.dumps(body['tags']))

        if sets:
            sets.append("updated_at = ?")
            args.append(now())
            args.append(org_id)
            conn.execute(f"UPDATE organizations SET {', '.join(sets)} WHERE id = ?", args)
            log_audit(conn, user['user_id'], 'organization', org_id, 'update', body)
            conn.commit()

        org = row_to_dict(conn.execute("SELECT * FROM organizations WHERE id = ?", (org_id,)).fetchone())
        conn.close()
        return self.send_json({"data": org})

    def handle_delete_organization(self, user, org_id):
        conn = get_db()
        existing = conn.execute("SELECT id FROM organizations WHERE id = ?", (org_id,)).fetchone()
        if not existing:
            conn.close()
            return self.send_error_json("Organization not found", 404)

        conn.execute("UPDATE contacts SET organization_id = NULL WHERE organization_id = ?", (org_id,))
        conn.execute("UPDATE organizations SET deleted_at = ?, updated_at = ? WHERE id = ?", (now(), now(), org_id))
        log_audit(conn, user['user_id'], 'organization', org_id, 'delete')
        conn.commit()
        conn.close()
        return self.send_json({"message": "Organization deleted"})

    # ═══════════════════════════════════════════════════════════════════════════
    # OPPORTUNITY (PIPELINE) HANDLERS
    # ═══════════════════════════════════════════════════════════════════════════

    def handle_list_opportunities(self, user, params):
        conn = get_db()
        query = """
            SELECT op.*, c.first_name, c.last_name, c.email as contact_email,
                   o.name as organization_name, u.full_name as owner_name,
                   fi.source_row_id as source_row_id,  -- the fundraising_investors row id (opportunities has no such column)
                   (SELECT MAX(communication_date) FROM communications
                     WHERE contact_id = op.contact_id AND deleted_at IS NULL) as last_contact_date
            FROM opportunities op
            LEFT JOIN contacts c ON op.contact_id = c.id
            LEFT JOIN organizations o ON op.organization_id = o.id
            LEFT JOIN users u ON op.owner_id = u.id
            LEFT JOIN fundraising_investors fi ON fi.id = op.fundraising_investor_id
            WHERE 1=1 AND op.deleted_at IS NULL
        """
        args = []
        if params.get('stage'):
            query += " AND op.stage = ?"
            args.append(params['stage'])
        if params.get('owner_id'):
            query += " AND op.owner_id = ?"
            args.append(params['owner_id'])
        if params.get('search'):
            search = f"%{params['search']}%"
            query += " AND (op.name LIKE ? OR c.first_name LIKE ? OR c.last_name LIKE ?)"
            args.extend([search, search, search])
        if params.get('priority'):
            query += " AND op.priority = ?"
            args.append(params['priority'])
        if params.get('fund_name'):
            query += " AND op.fund_name = ?"
            args.append(params['fund_name'])

        query += " ORDER BY op.updated_at DESC"
        limit = min(int(params.get('limit', 100)), 500)
        offset = int(params.get('offset', 0))
        total = conn.execute(f"SELECT COUNT(*) as total FROM ({query})", args).fetchone()['total']
        query += " LIMIT ? OFFSET ?"
        args.extend([limit, offset])

        opps = rows_to_list(conn.execute(query, args).fetchall())
        # Read-only existing-LP signal for the mobile Pipeline card earmark (8f): the deal's linked
        # contact rolls up to a grid investor with committed capital. Same source as the Contacts
        # list's `committed` and the detail sheet's "Existing LP" pill (contact_grid_signals →
        # committed > 0), so card earmark and detail pill agree. Derived on GET, never persisted;
        # opportunity writes use a field allowlist (handle_update_opportunity) so it can't round-trip.
        # `last_contact_date` (subselect above) drives the card recency line + the Staleness sort.
        # `source_row_id` (joined above via the durable fundraising_investor_id, not the contact path)
        # is the 8h "Open investor in Grid" deep-link target for the Pipeline detail; null for an opp
        # not linked to a grid investor (a legacy/manual deal).
        signals = contact_grid_signals(conn)
        for op in opps:
            sig = signals.get(str(op.get('contact_id') or ''))
            op['existing_investor'] = bool(sig and sig['committed'] > 0)
        conn.close()
        return self.send_json({"data": opps, "total": total, "limit": limit, "offset": offset})

    def handle_get_opportunity(self, user, opp_id):
        conn = get_db()
        opp = conn.execute("""
            SELECT op.*, c.first_name, c.last_name, c.email as contact_email,
                   o.name as organization_name, u.full_name as owner_name
            FROM opportunities op
            LEFT JOIN contacts c ON op.contact_id = c.id
            LEFT JOIN organizations o ON op.organization_id = o.id
            LEFT JOIN users u ON op.owner_id = u.id
            WHERE op.id = ? AND op.deleted_at IS NULL
        """, (opp_id,)).fetchone()

        if not opp:
            conn.close()
            return self.send_error_json("Opportunity not found", 404)

        result = row_to_dict(opp)
        result['communications'] = rows_to_list(conn.execute(
            """SELECT cm.*, u.full_name as created_by_name
               FROM communications cm LEFT JOIN users u ON cm.created_by = u.id
               WHERE cm.opportunity_id = ? AND cm.deleted_at IS NULL ORDER BY cm.communication_date DESC""",
            (opp_id,)
        ).fetchall())

        # Stage history from audit log
        result['stage_history'] = rows_to_list(conn.execute(
            """SELECT al.*, u.full_name as user_name FROM audit_log al
               LEFT JOIN users u ON al.user_id = u.id
               WHERE al.entity_type = 'opportunity' AND al.entity_id = ?
               AND al.action = 'stage_change'
               ORDER BY al.created_at DESC""",
            (opp_id,)
        ).fetchall())

        conn.close()
        return self.send_json({"data": result})

    def handle_create_opportunity(self, user, body):
        if not body.get('name') or not body.get('contact_id'):
            return self.send_error_json("name and contact_id are required")
        # Validate stage (mirrors handle_update_stage). Matters more since the 4-stage migration:
        # a stale cached client could otherwise write a legacy value that's invisible to the
        # report ORDER BY CASEs and unsettable from the UI.
        stage = body.get('stage', 'lead')
        if stage not in PIPELINE_STAGES:
            return self.send_error_json(f"Invalid stage. Must be one of: {', '.join(PIPELINE_STAGES)}")

        opp_id = generate_id()
        conn = get_db()

        # Verify contact exists
        contact = conn.execute("SELECT id, organization_id FROM contacts WHERE id = ?",
                               (body['contact_id'],)).fetchone()
        if not contact:
            conn.close()
            return self.send_error_json("Contact not found", 404)

        org_id = body.get('organization_id', contact['organization_id'])

        conn.execute("""
            INSERT INTO opportunities (id, name, contact_id, organization_id, stage,
                commitment_amount, expected_amount, probability, expected_close_date,
                fund_name, description, next_step, owner_id, priority)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
        """, (
            opp_id, body['name'], body['contact_id'], org_id,
            stage,
            body.get('commitment_amount', 0), body.get('expected_amount', 0),
            body.get('probability', 10), body.get('expected_close_date'),
            body.get('fund_name'), body.get('description'), body.get('next_step'),
            body.get('owner_id', user['user_id']), body.get('priority', 'medium')
        ))
        log_audit(conn, user['user_id'], 'opportunity', opp_id, 'create')
        conn.commit()

        opp = row_to_dict(conn.execute("""
            SELECT op.*, c.first_name, c.last_name, o.name as organization_name
            FROM opportunities op
            LEFT JOIN contacts c ON op.contact_id = c.id
            LEFT JOIN organizations o ON op.organization_id = o.id
            WHERE op.id = ?
        """, (opp_id,)).fetchone())
        conn.close()
        return self.send_json({"data": opp}, 201)

    def handle_update_opportunity(self, user, opp_id, body):
        conn = get_db()
        existing = conn.execute("SELECT * FROM opportunities WHERE id = ?", (opp_id,)).fetchone()
        if not existing:
            conn.close()
            return self.send_error_json("Opportunity not found", 404)

        if 'stage' in body and body['stage'] not in PIPELINE_STAGES:
            conn.close()
            return self.send_error_json(f"Invalid stage. Must be one of: {', '.join(PIPELINE_STAGES)}")

        updatable = ['name', 'contact_id', 'organization_id', 'stage', 'commitment_amount',
                     'expected_amount', 'probability', 'expected_close_date', 'fund_name',
                     'description', 'next_step', 'owner_id', 'priority', 'lost_reason']
        sets = []
        args = []

        old_stage = existing['stage']
        new_stage = body.get('stage', old_stage)

        for field in updatable:
            if field in body:
                sets.append(f"{field} = ?")
                args.append(body[field])

        if sets:
            sets.append("updated_at = ?")
            args.append(now())
            args.append(opp_id)
            conn.execute(f"UPDATE opportunities SET {', '.join(sets)} WHERE id = ?", args)

            if new_stage != old_stage:
                log_audit(conn, user['user_id'], 'opportunity', opp_id, 'stage_change',
                          {"from": old_stage, "to": new_stage})
            else:
                log_audit(conn, user['user_id'], 'opportunity', opp_id, 'update', body)
            conn.commit()

        opp = row_to_dict(conn.execute("""
            SELECT op.*, c.first_name, c.last_name, o.name as organization_name, u.full_name as owner_name
            FROM opportunities op
            LEFT JOIN contacts c ON op.contact_id = c.id
            LEFT JOIN organizations o ON op.organization_id = o.id
            LEFT JOIN users u ON op.owner_id = u.id
            WHERE op.id = ?
        """, (opp_id,)).fetchone())
        conn.close()
        return self.send_json({"data": opp})

    def handle_update_stage(self, user, opp_id, body):
        new_stage = body.get('stage', '')
        if new_stage not in PIPELINE_STAGES:
            return self.send_error_json(f"Invalid stage. Must be one of: {', '.join(PIPELINE_STAGES)}")

        conn = get_db()
        existing = conn.execute("SELECT stage FROM opportunities WHERE id = ?", (opp_id,)).fetchone()
        if not existing:
            conn.close()
            return self.send_error_json("Opportunity not found", 404)

        old_stage = existing['stage']
        conn.execute("UPDATE opportunities SET stage = ?, updated_at = ? WHERE id = ?",
                     (new_stage, now(), opp_id))
        log_audit(conn, user['user_id'], 'opportunity', opp_id, 'stage_change',
                  {"from": old_stage, "to": new_stage})
        conn.commit()

        opp = row_to_dict(conn.execute("""
            SELECT op.*, c.first_name, c.last_name, o.name as organization_name, u.full_name as owner_name
            FROM opportunities op
            LEFT JOIN contacts c ON op.contact_id = c.id
            LEFT JOIN organizations o ON op.organization_id = o.id
            LEFT JOIN users u ON op.owner_id = u.id
            WHERE op.id = ?
        """, (opp_id,)).fetchone())
        conn.close()
        return self.send_json({"data": opp})

    def handle_delete_opportunity(self, user, opp_id):
        conn = get_db()
        existing = conn.execute("SELECT id FROM opportunities WHERE id = ?", (opp_id,)).fetchone()
        if not existing:
            conn.close()
            return self.send_error_json("Opportunity not found", 404)

        conn.execute("UPDATE opportunities SET deleted_at = ?, updated_at = ? WHERE id = ?", (now(), now(), opp_id))
        log_audit(conn, user['user_id'], 'opportunity', opp_id, 'delete')
        conn.commit()
        conn.close()
        return self.send_json({"message": "Opportunity deleted"})

    # ═══════════════════════════════════════════════════════════════════════════
    # COMMUNICATION HANDLERS
    # ═══════════════════════════════════════════════════════════════════════════

    def handle_list_communications(self, user, params):
        conn = get_db()
        query = """
            SELECT cm.*, c.first_name, c.last_name, u.full_name as created_by_name
            FROM communications cm
            LEFT JOIN contacts c ON cm.contact_id = c.id
            LEFT JOIN users u ON cm.created_by = u.id
            WHERE 1=1 AND cm.deleted_at IS NULL
        """
        args = []
        if params.get('contact_id'):
            query += " AND cm.contact_id = ?"
            args.append(params['contact_id'])
        # source_row_id is an investor-scope filter; contact_id is a contact-scope filter. They are
        # never combined by any caller (the /contacts/{id}/communications subpath sets contact_id;
        # the mobile Grid detail sets source_row_id) — keep them mutually exclusive so a future
        # caller passing both can't silently get the empty intersection.
        elif params.get('source_row_id'):
            # Investor-level timeline for a fundraising-grid row (the mobile Grid detail): map the
            # grid JSON row id → its canonical contacts via the relational mirror, then return every
            # communication across those contacts. cm.deleted_at is still filtered above (soft-delete).
            query += """ AND cm.contact_id IN (
                SELECT fc.contact_id FROM fundraising_contacts fc
                JOIN fundraising_investors fi ON fc.investor_id = fi.id
                WHERE fi.source_row_id = ? AND fc.contact_id IS NOT NULL
            )"""
            args.append(params['source_row_id'])
        if params.get('type'):
            query += " AND cm.type = ?"
            args.append(params['type'])
        if params.get('search'):
            search = f"%{params['search']}%"
            query += " AND (cm.subject LIKE ? OR cm.body LIKE ?)"
            args.extend([search, search])

        query += " ORDER BY cm.communication_date DESC"
        limit = min(int(params.get('limit', 50)), 500)
        offset = int(params.get('offset', 0))
        total = conn.execute(f"SELECT COUNT(*) as total FROM ({query})", args).fetchone()['total']
        query += " LIMIT ? OFFSET ?"
        args.extend([limit, offset])

        comms = rows_to_list(conn.execute(query, args).fetchall())
        conn.close()
        return self.send_json({"data": comms, "total": total, "limit": limit, "offset": offset})

    def handle_list_contact_communications(self, user, contact_id, params):
        params['contact_id'] = contact_id
        return self.handle_list_communications(user, params)

    def handle_create_communication(self, user, body):
        if not body.get('contact_id'):
            return self.send_error_json("contact_id is required")

        comm_id = generate_id()
        conn = get_db()

        contact = conn.execute("SELECT id FROM contacts WHERE id = ?", (body['contact_id'],)).fetchone()
        if not contact:
            conn.close()
            return self.send_error_json("Contact not found", 404)

        attendees = json.dumps(body.get('attendees', []))
        conn.execute("""
            INSERT INTO communications (id, contact_id, opportunity_id, type, subject, body,
                communication_date, duration_minutes, outcome, next_action, next_action_date,
                attendees, created_by)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
        """, (
            comm_id, body['contact_id'], body.get('opportunity_id'),
            body.get('type', 'note'), body.get('subject'), body.get('body'),
            body.get('communication_date', now()),
            body.get('duration_minutes'), body.get('outcome'),
            body.get('next_action'), body.get('next_action_date'),
            attendees, user['user_id']
        ))

        # Update contact's updated_at
        conn.execute("UPDATE contacts SET updated_at = ? WHERE id = ?", (now(), body['contact_id']))

        log_audit(conn, user['user_id'], 'communication', comm_id, 'create')
        conn.commit()

        comm = row_to_dict(conn.execute("""
            SELECT cm.*, c.first_name, c.last_name, u.full_name as created_by_name
            FROM communications cm
            LEFT JOIN contacts c ON cm.contact_id = c.id
            LEFT JOIN users u ON cm.created_by = u.id
            WHERE cm.id = ?
        """, (comm_id,)).fetchone())
        conn.close()
        return self.send_json({"data": comm}, 201)

    def handle_log_fundraising_communication(self, user, body):
        row_id = str(body.get('row_id') or '').strip()
        investor_name_in = str(body.get('investor_name') or '').strip()
        contact_in = body.get('contact')
        append_note = bool(body.get('append_note', True))
        create_investor_if_missing = bool(body.get('create_investor_if_missing', False))
        # Optional initial Priority flag for the mobile add-investor sheet (8g). Honored ONLY on the
        # create branch below — an existing-row log-communication never touches its priority.
        create_priority = bool(body.get('priority', False))
        # Provenance: where this logged communication originated (grid UI vs the Matrix
        # intake bot). Default preserves prior behavior; callers may override.
        comm_source = (str(body.get('source') or 'fundraising_grid').strip() or 'fundraising_grid')[:64]

        if not row_id and not investor_name_in:
            return self.send_error_json("row_id or investor_name is required")

        conn = get_db()
        ensure_fundraising_state_row(conn)
        state = conn.execute("SELECT * FROM fundraising_state WHERE id = 'main'").fetchone()
        if not state:
            conn.close()
            return self.send_error_json("Fundraising state not found", 404)

        try:
            grid = json.loads(state['grid_json']) if state['grid_json'] else {}
        except Exception:
            grid = {}
        grid = sanitize_fundraising_grid(grid)
        rows = grid.get('rows', [])
        if not isinstance(rows, list):
            rows = []

        target_index = -1
        target_row = None
        for idx, row in enumerate(rows):
            if not isinstance(row, dict):
                continue
            if row_id and str(row.get('id') or '').strip() == row_id:
                target_index = idx
                target_row = row
                break
        if target_row is None and investor_name_in:
            wanted = _normalize_text(investor_name_in)
            for idx, row in enumerate(rows):
                if not isinstance(row, dict):
                    continue
                if _normalize_text(row.get('investor_name')) == wanted:
                    target_index = idx
                    target_row = row
                    break
        if target_row is None and investor_name_in and create_investor_if_missing:
            new_row = {
                "id": f"inv-{int(time.time() * 1000)}-{uuid.uuid4().hex[:6]}",
                "investor_name": investor_name_in,
                "contacts": [contact_in] if isinstance(contact_in, dict) else [],
                "notes": "",
                "notes_last_modified": "",
                "last_communication_date": "",
                "lead": "",
                "priority": create_priority,
                "follow_up": False,
                "graveyard": False
            }
            columns = grid.get('columns') if isinstance(grid.get('columns'), list) else []
            for col in columns:
                if not isinstance(col, dict):
                    continue
                col_id = str(col.get('id') or '').strip()
                if not col_id or col_id == 'longshot_followup' or col_id in new_row:
                    continue
                col_type = str(col.get('type') or 'text').strip().lower()
                if col_type == 'checkbox':
                    new_row[col_id] = False
                elif col_type in ('currency', 'number'):
                    new_row[col_id] = 0
                elif col_type == 'contacts':
                    new_row[col_id] = []
                else:
                    new_row[col_id] = ''
            rows.append(new_row)
            target_index = len(rows) - 1
            target_row = new_row
        if target_row is None:
            conn.close()
            return self.send_error_json("Fundraising row not found", 404)

        investor_name = str(target_row.get('investor_name') or investor_name_in or '').strip()
        contacts = target_row.get('contacts')
        if not isinstance(contact_in, dict):
            contact_in = None
        if contact_in is None and isinstance(contacts, list) and contacts:
            first = contacts[0]
            contact_in = first if isinstance(first, dict) else None
        if not isinstance(contact_in, dict):
            conn.close()
            return self.send_error_json("contact is required")

        contact_id = _upsert_contact_from_fundraising(conn, investor_name, contact_in, actor_user_id=user['user_id'])
        if not contact_id:
            conn.close()
            return self.send_error_json("Could not resolve contact for communication")

        comm_id = generate_id()
        comm_type = str(body.get('type') or 'note').strip() or 'note'
        comm_subject = str(body.get('subject') or '').strip()
        comm_body = str(body.get('body') or '').strip()
        comm_outcome = str(body.get('outcome') or '').strip()
        next_action = str(body.get('next_action') or '').strip()
        next_action_date = str(body.get('next_action_date') or '').strip()
        comm_date = str(body.get('communication_date') or now()).strip() or now()
        conn.execute("""
            INSERT INTO communications (id, contact_id, opportunity_id, type, subject, body,
                communication_date, duration_minutes, outcome, next_action, next_action_date,
                attendees, created_by)
            VALUES (?, ?, NULL, ?, ?, ?, ?, NULL, ?, ?, ?, '[]', ?)
        """, (
            comm_id,
            contact_id,
            comm_type,
            comm_subject,
            comm_body,
            comm_date,
            comm_outcome or None,
            next_action or None,
            next_action_date or None,
            user['user_id']
        ))
        conn.execute("UPDATE contacts SET updated_at = ? WHERE id = ?", (now(), contact_id))
        log_audit(conn, user['user_id'], 'communication', comm_id, 'create', {"source": comm_source})

        iso_day = now()[:10]
        target_row['last_communication_date'] = iso_day
        if append_note:
            contact_name = str(contact_in.get('name') or '').strip()
            summary = comm_subject or comm_body[:120]
            type_tag = "" if comm_type == "note" else f"[{comm_type}] "
            note_line = f"{iso_day} {type_tag}{contact_name}: {summary}".strip()
            existing_notes = str(target_row.get('notes') or '')
            target_row['notes'] = f"{existing_notes}\n{note_line}".strip() if existing_notes.strip() else note_line
            target_row['notes_last_modified'] = iso_day
        rows[target_index] = target_row

        try:
            views = json.loads(state['views_json']) if state['views_json'] else []
        except Exception:
            views = []
        views = sanitize_grid_views(views)
        next_version = int(state['version'] or 1) + 1
        conn.execute("""
            UPDATE fundraising_state
            SET grid_json = ?, views_json = ?, version = ?, updated_by = ?, updated_at = ?
            WHERE id = 'main'
        """, (json.dumps(grid), json.dumps(views), next_version, user['user_id'], now()))
        sync_fundraising_relational(conn, grid, views, actor_user_id=user['user_id'])
        conn.commit()

        comm = row_to_dict(conn.execute("""
            SELECT cm.*, c.first_name, c.last_name, u.full_name as created_by_name
            FROM communications cm
            LEFT JOIN contacts c ON cm.contact_id = c.id
            LEFT JOIN users u ON cm.created_by = u.id
            WHERE cm.id = ?
        """, (comm_id,)).fetchone())
        conn.close()
        return self.send_json({"data": {"communication": comm, "row": target_row, "version": next_version}}, 201)

    def handle_update_fundraising_row(self, user, body):
        """Version-safe single-row edit of an investor's NAME and/or contact pills (P3b).

        The one-row twin of handle_log_fundraising_communication: the canonical grid blob is read
        FRESH server-side, ONLY the target row's investor_name / contacts are mutated, then the
        blob is written back with a version bump + relational re-sync. It never accepts a
        whole-grid payload, so a stale mobile client can't clobber concurrent edits to OTHER rows
        the way the grid PUT (handle_update_fundraising_state) would (BRIEF §3a). Same-row
        name/contacts is last-write-wins, exactly like the notes append in log-communication.

        Body: { row_id (required), investor_name?, contacts? } — at least one of investor_name /
        contacts must be present. `contacts` REPLACES the row's full pill list (the client owns
        add/edit/remove + dedup). Removing a pill drops it from the row + its fundraising_contacts
        row on re-sync, but never hard-deletes the classic contacts directory entry
        (soft-delete-only convention — _upsert_contact_from_fundraising only ever upserts)."""
        row_id = str(body.get('row_id') or '').strip()
        if not row_id:
            return self.send_error_json("row_id is required")

        has_name = 'investor_name' in body
        has_contacts = 'contacts' in body
        if not has_name and not has_contacts:
            return self.send_error_json("investor_name or contacts is required")

        new_name = None
        if has_name:
            new_name = str(body.get('investor_name') or '').strip()
            if not new_name:
                return self.send_error_json("investor_name cannot be blank")

        new_contacts = None
        if has_contacts:
            if not isinstance(body.get('contacts'), list):
                return self.send_error_json("contacts must be an array")
            new_contacts = _sanitize_fundraising_contacts(body.get('contacts'))

        conn = get_db()
        ensure_fundraising_state_row(conn)
        state = conn.execute("SELECT * FROM fundraising_state WHERE id = 'main'").fetchone()
        if not state:
            conn.close()
            return self.send_error_json("Fundraising state not found", 404)

        try:
            grid = json.loads(state['grid_json']) if state['grid_json'] else {}
        except Exception:
            grid = {}
        grid = sanitize_fundraising_grid(grid)
        rows = grid.get('rows', [])
        if not isinstance(rows, list):
            rows = []

        target_index = -1
        target_row = None
        for idx, row in enumerate(rows):
            if isinstance(row, dict) and str(row.get('id') or '').strip() == row_id:
                target_index = idx
                target_row = row
                break
        if target_row is None:
            conn.close()
            return self.send_error_json("Fundraising row not found", 404)

        if new_name is not None:
            target_row['investor_name'] = new_name
        if new_contacts is not None:
            target_row['contacts'] = new_contacts
        rows[target_index] = target_row

        try:
            views = json.loads(state['views_json']) if state['views_json'] else []
        except Exception:
            views = []
        views = sanitize_grid_views(views)
        next_version = int(state['version'] or 1) + 1
        conn.execute("""
            UPDATE fundraising_state
            SET grid_json = ?, views_json = ?, version = ?, updated_by = ?, updated_at = ?
            WHERE id = 'main'
        """, (json.dumps(grid), json.dumps(views), next_version, user['user_id'], now()))
        sync_fundraising_relational(conn, grid, views, actor_user_id=user['user_id'])
        log_audit(conn, user['user_id'], 'fundraising_state', 'main', 'update_row',
                  {"version": next_version, "row_id": row_id})
        conn.commit()
        conn.close()

        # The returned row lacks the read-only computed columns (pipeline/existing_investor/…),
        # which are injected only on the GET path — the mobile client reloads after a save.
        return self.send_json({"data": {"row": target_row, "version": next_version}}, 200)

    def _fetch_opportunity_row(self, conn, opp_id):
        return row_to_dict(conn.execute("""
            SELECT op.*, c.first_name, c.last_name, c.email as contact_email,
                   o.name as organization_name, u.full_name as owner_name
            FROM opportunities op
            LEFT JOIN contacts c ON op.contact_id = c.id
            LEFT JOIN organizations o ON op.organization_id = o.id
            LEFT JOIN users u ON op.owner_id = u.id
            WHERE op.id = ?
        """, (opp_id,)).fetchone())

    def _resolve_grid_primary_contact(self, conn, investor_id, contact_index, actor_user_id):
        """Resolve the classic contacts.id for a grid investor's chosen contact pill,
        reusing the link sync already records in fundraising_contacts.contact_id (no
        bare POST /api/contacts side-door). contact_index matches the pill order
        (fundraising_contacts.sort_order)."""
        try:
            idx = int(contact_index)
        except (TypeError, ValueError):
            idx = 0
        if idx < 0:
            idx = 0
        fc = conn.execute(
            "SELECT contact_id, full_name, email FROM fundraising_contacts "
            "WHERE investor_id = ? ORDER BY sort_order, rowid LIMIT 1 OFFSET ?",
            (investor_id, idx)
        ).fetchone()
        if not fc:
            # requested index out of range — fall back to the first pill
            fc = conn.execute(
                "SELECT contact_id, full_name, email FROM fundraising_contacts "
                "WHERE investor_id = ? ORDER BY sort_order, rowid LIMIT 1",
                (investor_id,)
            ).fetchone()
        if not fc:
            return None
        if fc['contact_id']:
            return fc['contact_id']
        # Rows predating the contact_id backfill: resolve via the same grid→classic
        # upsert the relational sync uses, not a fresh side-door create.
        inv = conn.execute("SELECT investor_name FROM fundraising_investors WHERE id = ?", (investor_id,)).fetchone()
        investor_name = str(inv['investor_name'] if inv else '') or ''
        return _upsert_contact_from_fundraising(
            conn, investor_name, {"name": fc['full_name'], "email": fc['email']}, actor_user_id=actor_user_id
        )

    def handle_pipeline_link(self, user, body):
        """Create (or return the existing) Pipeline opportunity for a fundraising-grid
        investor row and link it durably via opportunities.fundraising_investor_id.
        Idempotent: one live opp per investor — a re-link returns the existing opp
        without reseeding its Pipeline-owned funnel fields."""
        source_row_id = str(body.get('source_row_id') or '').strip()
        if not source_row_id:
            return self.send_error_json("source_row_id is required")

        conn = get_db()
        investor = conn.execute(
            "SELECT id, investor_name, lead FROM fundraising_investors WHERE source_row_id = ?",
            (source_row_id,)
        ).fetchone()
        if not investor:
            conn.close()
            return self.send_error_json("Investor not found for that grid row — save the grid first", 404)
        investor_id = investor['id']
        investor_name = str(investor['investor_name'] or '').strip() or 'Untitled Investor'

        existing = conn.execute(
            "SELECT id FROM opportunities WHERE fundraising_investor_id = ? AND deleted_at IS NULL "
            "ORDER BY created_at LIMIT 1",
            (investor_id,)
        ).fetchone()
        if existing:
            opp = self._fetch_opportunity_row(conn, existing['id'])
            conn.close()
            return self.send_json({"data": opp, "already_linked": True})

        contact_id = self._resolve_grid_primary_contact(
            conn, investor_id, body.get('contact_index'), actor_user_id=user['user_id']
        )
        if not contact_id:
            conn.close()
            return self.send_error_json("Add at least one contact to the investor row before adding it to the pipeline")

        contact = conn.execute("SELECT organization_id FROM contacts WHERE id = ?", (contact_id,)).fetchone()
        org_id = contact['organization_id'] if contact else None

        stage = str(body.get('stage') or 'lead').strip() or 'lead'
        if stage not in PIPELINE_STAGES:
            conn.close()
            return self.send_error_json(f"Invalid stage. Must be one of: {', '.join(PIPELINE_STAGES)}")
        try:
            expected_amount = float(body.get('expected_amount') or 0)
        except (TypeError, ValueError):
            expected_amount = 0.0
        try:
            probability = int(body.get('probability'))
        except (TypeError, ValueError):
            probability = 35
        probability = max(0, min(100, probability))
        fund_name = str(body.get('fund_name') or '').strip()
        name = str(body.get('name') or '').strip() or f"{investor_name} — Pipeline"
        owner_id = _resolve_owner_from_lead(conn, investor['lead'], user['user_id'])

        opp_id = generate_id()
        conn.execute("""
            INSERT INTO opportunities (id, name, contact_id, organization_id, stage,
                commitment_amount, expected_amount, probability, fund_name,
                owner_id, priority, fundraising_investor_id)
            VALUES (?, ?, ?, ?, ?, 0, ?, ?, ?, ?, 'medium', ?)
        """, (opp_id, name, contact_id, org_id, stage, expected_amount, probability,
              fund_name or None, owner_id, investor_id))
        log_audit(conn, user['user_id'], 'opportunity', opp_id, 'create', {"source": "grid_pipeline_link"})
        conn.commit()

        opp = self._fetch_opportunity_row(conn, opp_id)
        conn.close()
        return self.send_json({"data": opp, "already_linked": False}, 201)

    def handle_pipeline_unlink(self, user, body):
        """Remove a grid investor from the Pipeline: soft-delete its linked opportunity.
        The fundraising-grid row (investor, contacts, commitments, notes) is left fully
        intact — only the opportunity is archived (recoverable via the board)."""
        source_row_id = str(body.get('source_row_id') or '').strip()
        if not source_row_id:
            return self.send_error_json("source_row_id is required")
        conn = get_db()
        investor = conn.execute(
            "SELECT id FROM fundraising_investors WHERE source_row_id = ?", (source_row_id,)
        ).fetchone()
        if not investor:
            conn.close()
            return self.send_error_json("Investor not found for that grid row", 404)
        opp_rows = conn.execute(
            "SELECT id FROM opportunities WHERE fundraising_investor_id = ? AND deleted_at IS NULL",
            (investor['id'],)
        ).fetchall()
        archived = 0
        for opp in opp_rows:
            conn.execute("UPDATE opportunities SET deleted_at = ?, updated_at = ? WHERE id = ?",
                         (now(), now(), opp['id']))
            log_audit(conn, user['user_id'], 'opportunity', opp['id'], 'delete', {"source": "grid_pipeline_unlink"})
            archived += 1
        conn.commit()
        conn.close()
        return self.send_json({"data": {"archived": archived}})

    # ═══════════════════════════════════════════════════════════════════════════
    # REMINDERS / FOLLOW-UPS
    # ═══════════════════════════════════════════════════════════════════════════
    # First-class tickler tied to the fundraising grid (logical FK to
    # fundraising_investors.id, denormalized name). Supersedes the binary follow_up
    # checkbox + per-comm next_action_date as the place follow-ups live. All reads filter
    # deleted_at IS NULL (standing soft-delete rule). Pure local CRM data — no LLM path.

    _REMINDER_STATUSES = ('open', 'done', 'snoozed', 'cancelled')

    def handle_list_reminders(self, user, params):
        params = params or {}
        where = ["r.deleted_at IS NULL"]
        args = []
        # `overdue` owns the status constraint (open + strictly past due) so it can never
        # conflict with a stray status= param — two contradictory r.status predicates would
        # silently return zero rows. So overdue wins and an accompanying status= is ignored.
        overdue = str(params.get('overdue') or '').strip().lower() in ('1', 'true', 'yes')
        status = str(params.get('status') or '').strip()
        if overdue:
            where.append("r.status = 'open'")
            where.append("r.due_date IS NOT NULL AND r.due_date != '' AND r.due_date < ?")
            args.append(now()[:10])
        elif status == 'active':          # open + snoozed (the actionable set)
            where.append("r.status IN ('open','snoozed')")
        elif status in self._REMINDER_STATUSES:
            where.append("r.status = ?")
            args.append(status)
        assignee = str(params.get('assignee') or '').strip()
        if assignee == 'me':
            where.append("r.assignee_id = ?")
            args.append(user['user_id'])
        elif assignee:
            where.append("r.assignee_id = ?")
            args.append(assignee)
        investor_id = str(params.get('investor_id') or '').strip()
        if investor_id:
            where.append("r.investor_id = ?")
            args.append(investor_id)
        # The grid only knows the source_row_id, not the backend investor id (mirrors the
        # pipeline-link endpoint) — resolve it via a subquery so the grid stays decoupled.
        source_row_id = str(params.get('source_row_id') or '').strip()
        if source_row_id:
            where.append("r.investor_id IN (SELECT id FROM fundraising_investors WHERE source_row_id = ?)")
            args.append(source_row_id)
        due_before = str(params.get('due_before') or '').strip()
        if due_before:
            where.append("r.due_date IS NOT NULL AND r.due_date != '' AND r.due_date <= ?")
            args.append(due_before)

        conn = get_db()
        try:
            rows = conn.execute(
                # fi.source_row_id (joined via the reminder's investor_id) is the 8h "Open investor
                # in Grid" deep-link target for the Reminders edit sheet; null for an unlinked reminder.
                "SELECT r.*, ua.username AS assignee_name, uc.username AS creator_name, "
                "fi.source_row_id AS source_row_id "
                "FROM reminders r "
                "LEFT JOIN users ua ON ua.id = r.assignee_id "
                "LEFT JOIN users uc ON uc.id = r.created_by "
                "LEFT JOIN fundraising_investors fi ON fi.id = r.investor_id "
                "WHERE " + " AND ".join(where) +
                # dated reminders first (soonest due), then undated by recency
                " ORDER BY (r.due_date IS NULL OR r.due_date = ''), r.due_date ASC, r.created_at ASC",
                args
            ).fetchall()
            last_by_inv = last_activity_by_investor(conn)
            data = []
            for row in rows:
                d = row_to_dict(row)
                d['last_activity_at'] = last_by_inv.get(d.get('investor_id'))
                data.append(d)
            return self.send_json({"data": data, "total": len(data)})
        finally:
            conn.close()

    def handle_create_reminder(self, user, body):
        body = body or {}
        title = str(body.get('title') or '').strip()
        if not title:
            return self.send_error_json("title is required")
        investor_id = str(body.get('investor_id') or '').strip() or None
        investor_name = str(body.get('investor_name') or '').strip()
        contact_id = str(body.get('contact_id') or '').strip() or None
        source_row_id = str(body.get('source_row_id') or '').strip()
        due_date = str(body.get('due_date') or '').strip() or None
        assignee_id = str(body.get('assignee_id') or '').strip() or None
        details = str(body.get('details') or '').strip() or None
        source = str(body.get('source') or 'human').strip() or 'human'

        conn = get_db()
        try:
            # The grid hands us a source_row_id, not the backend investor id (mirrors the
            # pipeline-link endpoint) — resolve it to the stable investor id + name.
            if not investor_id and source_row_id:
                inv = conn.execute(
                    "SELECT id, investor_name FROM fundraising_investors WHERE source_row_id = ?",
                    (source_row_id,)
                ).fetchone()
                if not inv:
                    conn.close()
                    return self.send_error_json("Unknown source_row_id", 404)
                investor_id = inv['id']
                investor_name = investor_name or str(inv['investor_name'] or '').strip()
            # Refresh the denormalized name from the grid when we have a live investor id;
            # reject an id that doesn't resolve unless the caller supplied a name to keep.
            elif investor_id:
                inv = conn.execute(
                    "SELECT investor_name FROM fundraising_investors WHERE id = ?", (investor_id,)
                ).fetchone()
                if inv:
                    investor_name = str(inv['investor_name'] or '').strip() or investor_name
                elif not investor_name:
                    conn.close()
                    return self.send_error_json("Unknown investor_id", 404)
            rid = generate_id()
            conn.execute(
                "INSERT INTO reminders (id, investor_id, investor_name, contact_id, title, details, "
                "due_date, status, assignee_id, created_by, source, created_at, updated_at) "
                "VALUES (?, ?, ?, ?, ?, ?, ?, 'open', ?, ?, ?, ?, ?)",
                (rid, investor_id, investor_name or None, contact_id, title, details,
                 due_date, assignee_id, user['user_id'], source, now(), now())
            )
            log_audit(conn, user['user_id'], 'reminder', rid, 'create',
                      {"title": title, "investor_id": investor_id, "due_date": due_date})
            conn.commit()
            row = conn.execute("SELECT * FROM reminders WHERE id = ?", (rid,)).fetchone()
            return self.send_json({"data": row_to_dict(row)}, 201)
        finally:
            conn.close()

    def handle_update_reminder(self, user, reminder_id, body):
        body = body or {}
        conn = get_db()
        try:
            existing = conn.execute(
                "SELECT id FROM reminders WHERE id = ? AND deleted_at IS NULL", (reminder_id,)
            ).fetchone()
            if not existing:
                conn.close()
                return self.send_error_json("Reminder not found", 404)
            sets, args = [], []
            if 'title' in body:
                t = str(body.get('title') or '').strip()
                if not t:
                    conn.close()
                    return self.send_error_json("title cannot be empty")
                sets.append("title = ?")
                args.append(t)
            for field in ('details', 'due_date', 'assignee_id', 'contact_id', 'snoozed_until'):
                if field in body:
                    sets.append(f"{field} = ?")
                    args.append(str(body.get(field) or '').strip() or None)
            if 'status' in body:
                st = str(body.get('status') or '').strip()
                if st not in self._REMINDER_STATUSES:
                    conn.close()
                    return self.send_error_json(
                        f"Invalid status. Must be one of: {', '.join(self._REMINDER_STATUSES)}")
                sets.append("status = ?")
                args.append(st)
                # Stamp completion time on done; clear it if the reminder is reopened.
                sets.append("completed_at = ?")
                args.append(now() if st == 'done' else None)
            if not sets:
                conn.close()
                return self.send_error_json("No updatable fields provided")
            sets.append("updated_at = ?")
            args.append(now())
            args.append(reminder_id)
            conn.execute(f"UPDATE reminders SET {', '.join(sets)} WHERE id = ?", args)
            log_audit(conn, user['user_id'], 'reminder', reminder_id, 'update', body)
            conn.commit()
            row = conn.execute("SELECT * FROM reminders WHERE id = ?", (reminder_id,)).fetchone()
            return self.send_json({"data": row_to_dict(row)})
        finally:
            conn.close()

    def handle_delete_reminder(self, user, reminder_id):
        conn = get_db()
        try:
            existing = conn.execute(
                "SELECT id FROM reminders WHERE id = ? AND deleted_at IS NULL", (reminder_id,)
            ).fetchone()
            if not existing:
                conn.close()
                return self.send_error_json("Reminder not found", 404)
            conn.execute("UPDATE reminders SET deleted_at = ?, updated_at = ? WHERE id = ?",
                         (now(), now(), reminder_id))
            log_audit(conn, user['user_id'], 'reminder', reminder_id, 'delete', None)
            conn.commit()
            return self.send_json({"data": {"deleted": True}})
        finally:
            conn.close()

    # ── Natural-language query (W2) — read-only, the safe nl_query core ──
    def handle_nl_query(self, user, body):
        """Answer a plain-English question about the fundraising database, read-only.

        Bot-or-admin: a query can surface the whole investor book, so it stays off the member
        tier; the audit row written below (entity_type='nl_query') makes a query made through a
        leaked or automated credential detectable. Accepts either {question} (mapped to an
        intent+slots on the LOCAL model — nothing leaves the box) or {intent, slots} (run a
        curated query directly, e.g. a UI re-run). BOTH go through the same validator and the
        same fixed parameterized SQL in nl_query; result rows never reach any model.

        Status: a local-model outage -> 503; an unexpected SQL fault -> 500; everything else,
        including a soft 'no question matched', returns 200 with the structured result, because
        the UI always wants the interpreted query + summary back rather than a bare HTTP code."""
        if not require_bot_or_admin(user):
            return self.send_error_json("Bot or admin required", 403)
        body = body or {}
        question = str(body.get('question') or '').strip()
        intent = str(body.get('intent') or '').strip()
        if not question and not intent:
            return self.send_error_json("question or intent is required")
        source = (str(body.get('source') or 'api').strip()[:20]) or 'api'

        import nl_query  # pure-stdlib at import; the local-model leg is lazy inside translate
        conn = get_db()
        try:
            def _audit(p):
                log_audit(conn, user['user_id'], 'nl_query', p.get('intent') or '-', 'query',
                          {"source": p.get('source'), "slots": p.get('slots'),
                           "row_count": p.get('row_count'), "error": p.get('error'),
                           "question": question or None})
            if question:
                result = nl_query.answer(conn, question, audit_fn=_audit,
                                         actor=user['user_id'], source=source)
            else:
                result = nl_query.run_query(conn, intent, body.get('slots') or {},
                                            audit_fn=_audit, actor=user['user_id'], source=source)
            conn.commit()  # persist the audit row(s)
        finally:
            conn.close()

        err = result.get('error')
        if err == 'model_unavailable':
            return self.send_json({"data": result}, 503)
        if err == 'query_failed':
            return self.send_json({"data": result}, 500)
        return self.send_json({"data": result})

    def handle_nl_query_catalog(self, user):
        """The askable surface: every intent's key, summary, slot specs and an example
        question — so the UI can show what can be asked. Same gate as the query endpoint."""
        if not require_bot_or_admin(user):
            return self.send_error_json("Bot or admin required", 403)
        import nl_query
        return self.send_json({"data": nl_query.catalog()})

    def handle_intake_match(self, user, params):
        """Read-only: does an investor matching this intake already exist? Used by the
        Matrix intake bot to label its in-thread proposal new-vs-existing. Returns the
        grid row id so an approved note lands on exactly that investor.

        `match` is the confident exact match (auto-attached by the bot). When there is no
        exact match, `candidates` carries ranked fuzzy near-matches so the bot can surface
        a disambiguation shortlist in-thread (the human picks one or creates new) — closing
        the duplicate-investor hole the exact-only matcher leaves open."""
        q = str(params.get('q') or '').strip()
        email = str(params.get('email') or '').strip()
        if not q and not email:
            return self.send_error_json("q or email is required")
        conn = get_db()
        try:
            match = find_intake_match(conn, q, email)
            candidates = find_intake_candidates(conn, q, email) if match is None else []
        finally:
            conn.close()
        return self.send_json({"data": {"match": match, "candidates": candidates}})

    def handle_intake_card(self, user, body):
        """In-app business-card intake (#7): a mobile photo -> vision-transcribe (local VL via
        Spark Control) -> the SAME text-parse + fuzzy match the Matrix card flow runs (M3),
        minus Matrix, surfaced as a mobile sheet. READ-ONLY: nothing is written here. The
        proposal it returns is edited + approved by a human, and that approve write reuses
        POST /api/fundraising/log-communication tagged source='app_card'. Authenticated member+
        (a human approves every write — same draft->approve gate as the bot, not bot-/admin-only).

        Body: {image_b64 (raw base64; a data: URI is also tolerated), mime?}. On success returns
        status 200 {ok:true, transcription, proposal, match, candidates}; soft-fails also return
        200 (or 502) with {ok:false, reason}: 'unreadable' (model saw no card / <5 chars),
        'vision_unavailable' (502 — Spark/model down, or the intake core failed to import on a
        dev box without it). See docs/handoffs/in-app-card-intake-plan.md + the matrix-intake guide."""
        image_b64 = str(body.get('image_b64') or '').strip()
        mime = (str(body.get('mime') or 'image/jpeg').strip() or 'image/jpeg')[:64]
        # Tolerate a full data-URI (clients often send the whole `data:...;base64,XXXX`):
        # llm.chat_vision builds the `data:` wrapper itself, so strip any prefix to raw base64.
        if image_b64.startswith('data:'):
            comma = image_b64.find(',')
            if comma != -1:
                image_b64 = image_b64[comma + 1:].strip()
        if not image_b64:
            return self.send_error_json("image_b64 is required")
        # Size guard: the client downscales to ~2 MP before base64, so a legit card is well under
        # this; reject larger up front so a huge upload can't tie up a worker thread. (base64
        # inflates ~4/3; 12 MB b64 ~= 9 MB image — generous for a downscaled card.)
        if len(image_b64) > 12_000_000:
            return self.send_error_json("Image too large; retake a smaller photo", 413)
        try:
            base64.b64decode(image_b64, validate=True)
        except Exception:
            return self.send_error_json("image_b64 is not valid base64")

        # Lazily import the nio-free intake core (parse + spark) reused from the Matrix bot,
        # guarded like _summarize_email_gist's `import llm`: a dev box without the modules or with
        # Spark unreachable still boots and this endpoint just 502s. Import ONLY parse + spark
        # (both nio-free) — never the bot's crm_client/settings/bot. (matrix-intake guide: the
        # bot imports by bare name; parse/spark don't collide with ingest's modules.)
        try:
            _intake_dir = os.path.join(BASE_DIR, "matrix_intake")
            if _intake_dir not in sys.path:
                sys.path.insert(0, _intake_dir)
            import parse as _intake_parse  # noqa: E402
            import spark as _intake_spark  # noqa: E402
        except Exception:
            return self.send_json({"data": {"ok": False, "reason": "vision_unavailable"}}, 502)

        # The one added step vs the text path: vision-transcribe. The model only TRANSCRIBES; the
        # text parser then extracts — so the email/phone/LinkedIn integrity rule (a value is kept
        # only if it literally appears in the transcription, never minted) protects card intake too.
        try:
            transcription = _intake_spark.transcribe_card(image_b64, mime)
        except Exception:
            return self.send_json({"data": {"ok": False, "reason": "vision_unavailable"}}, 502)
        if len((transcription or "").strip()) < 5:
            return self.send_json({"data": {"ok": False, "reason": "unreadable"}})

        # Frame as a new-investor note so the extractor reads it that way (same framing as M3).
        framed = "New investor — from a business card:\n" + transcription
        try:
            proposal = _intake_parse.parse_message(framed, roster=None)
        except Exception:
            return self.send_json({"data": {"ok": False, "reason": "vision_unavailable"}}, 502)

        # Same new-vs-existing resolution the bot uses (crm_client.match): q = firm-or-person.
        q = str(proposal.get("investor_name") or proposal.get("contact_name") or "").strip()
        email = str(proposal.get("contact_email") or "").strip()
        match = None
        candidates = []
        if q or email:
            conn = get_db()
            try:
                match = find_intake_match(conn, q, email)
                candidates = find_intake_candidates(conn, q, email) if match is None else []
            finally:
                conn.close()

        # Drop internal control keys (e.g. _source_text) before handing the proposal to the client.
        proposal_out = {k: v for k, v in proposal.items() if not k.startswith("_")}
        return self.send_json({"data": {
            "ok": True,
            "transcription": transcription,
            "proposal": proposal_out,
            "match": match,
            "candidates": candidates,
        }})

    def handle_update_communication(self, user, comm_id, body):
        conn = get_db()
        existing = conn.execute("SELECT id FROM communications WHERE id = ?", (comm_id,)).fetchone()
        if not existing:
            conn.close()
            return self.send_error_json("Communication not found", 404)

        updatable = ['type', 'subject', 'body', 'communication_date', 'duration_minutes',
                     'outcome', 'next_action', 'next_action_date']
        sets = []
        args = []
        for field in updatable:
            if field in body:
                sets.append(f"{field} = ?")
                args.append(body[field])
        if 'attendees' in body:
            sets.append("attendees = ?")
            args.append(json.dumps(body['attendees']))

        if sets:
            sets.append("updated_at = ?")
            args.append(now())
            args.append(comm_id)
            conn.execute(f"UPDATE communications SET {', '.join(sets)} WHERE id = ?", args)
            log_audit(conn, user['user_id'], 'communication', comm_id, 'update')
            conn.commit()

        comm = row_to_dict(conn.execute("SELECT * FROM communications WHERE id = ?", (comm_id,)).fetchone())
        conn.close()
        return self.send_json({"data": comm})

    def handle_delete_communication(self, user, comm_id):
        conn = get_db()
        existing = conn.execute("SELECT id FROM communications WHERE id = ?", (comm_id,)).fetchone()
        if not existing:
            conn.close()
            return self.send_error_json("Communication not found", 404)

        conn.execute("UPDATE communications SET deleted_at = ?, updated_at = ? WHERE id = ?", (now(), now(), comm_id))
        log_audit(conn, user['user_id'], 'communication', comm_id, 'delete')
        conn.commit()
        conn.close()
        return self.send_json({"message": "Communication deleted"})

    # ═══════════════════════════════════════════════════════════════════════════
    # REPORT HANDLERS
    # ═══════════════════════════════════════════════════════════════════════════

    def handle_dashboard_report(self, user):
        conn = get_db()

        # Key metrics. "Total LPs" / "Prospects" are derived from the canonical grid
        # (investor entities), not the retired contacts.contact_type: an LP is a grid
        # investor with committed capital (total_invested > 0); a prospect is a live
        # grid row with nothing committed yet. Graveyarded rows are excluded (same basis
        # as Total Committed below); blank grid rows (synced as the 'Untitled Investor'
        # placeholder, see sync_fundraising_relational) are excluded from the prospect
        # count so empties don't inflate it.
        total_lps = conn.execute(
            "SELECT COUNT(*) as c FROM fundraising_investors WHERE total_invested > 0 AND graveyard = 0"
        ).fetchone()['c']
        total_prospects = conn.execute(
            "SELECT COUNT(*) as c FROM fundraising_investors "
            "WHERE COALESCE(total_invested, 0) = 0 AND graveyard = 0 AND investor_name != 'Untitled Investor'"
        ).fetchone()['c']
        total_contacts = conn.execute("SELECT COUNT(*) as c FROM contacts WHERE deleted_at IS NULL").fetchone()['c']

        # Committed capital comes from the canonical fundraising grid (per-investor
        # rollup of per-fund commitments). Graveyarded (written-off) investors are
        # excluded so the headline reflects live committed capital — a deliberate
        # divergence from /api/fundraising/relational-summary, which sums all rows.
        # The legacy lp_profiles table is retired; the grid tracks commitments, not a
        # separate "funded" amount, so total_funded is no longer reported.
        total_committed = conn.execute(
            "SELECT COALESCE(SUM(total_invested), 0) as total FROM fundraising_investors WHERE graveyard = 0"
        ).fetchone()['total']

        pipeline_value = conn.execute(
            "SELECT COALESCE(SUM(expected_amount), 0) as total FROM opportunities WHERE stage != 'commitment' AND deleted_at IS NULL"
        ).fetchone()['total']

        active_opportunities = conn.execute(
            "SELECT COUNT(*) as c FROM opportunities WHERE stage != 'commitment' AND deleted_at IS NULL"
        ).fetchone()['c']

        # Pipeline by stage
        pipeline_stages = rows_to_list(conn.execute("""
            SELECT stage, COUNT(*) as count, COALESCE(SUM(expected_amount), 0) as total_value,
                   COALESCE(SUM(commitment_amount), 0) as committed_value
            FROM opportunities
            WHERE deleted_at IS NULL
            GROUP BY stage
            ORDER BY CASE stage
                WHEN 'lead' THEN 1 WHEN 'engaged' THEN 2
                WHEN 'diligence' THEN 3 WHEN 'commitment' THEN 4
            END
        """).fetchall())

        # Recent communications
        recent_comms = rows_to_list(conn.execute("""
            SELECT cm.*, c.first_name, c.last_name, u.full_name as created_by_name
            FROM communications cm
            LEFT JOIN contacts c ON cm.contact_id = c.id
            LEFT JOIN users u ON cm.created_by = u.id
            ORDER BY cm.communication_date DESC LIMIT 10
        """).fetchall())

        # Upcoming actions
        upcoming = rows_to_list(conn.execute("""
            SELECT cm.*, c.first_name, c.last_name
            FROM communications cm
            LEFT JOIN contacts c ON cm.contact_id = c.id
            WHERE cm.next_action IS NOT NULL AND cm.next_action != ''
            AND cm.next_action_date >= date('now')
            ORDER BY cm.next_action_date ASC LIMIT 10
        """).fetchall())

        # Recent stage changes
        recent_stage_changes = rows_to_list(conn.execute("""
            SELECT al.*, u.full_name as user_name, op.name as opportunity_name
            FROM audit_log al
            LEFT JOIN users u ON al.user_id = u.id
            LEFT JOIN opportunities op ON al.entity_id = op.id
            WHERE al.entity_type = 'opportunity' AND al.action = 'stage_change'
            ORDER BY al.created_at DESC LIMIT 10
        """).fetchall())

        # Comms this month
        comms_this_month = conn.execute("""
            SELECT COUNT(*) as c FROM communications
            WHERE communication_date >= date('now', 'start of month')
        """).fetchone()['c']

        # Meetings this month
        meetings_this_month = conn.execute("""
            SELECT COUNT(*) as c FROM communications
            WHERE type = 'meeting' AND communication_date >= date('now', 'start of month')
        """).fetchone()['c']

        conn.close()
        return self.send_json({
            "data": {
                "metrics": {
                    "total_lps": total_lps,
                    "total_prospects": total_prospects,
                    "total_contacts": total_contacts,
                    "total_committed": total_committed,
                    "pipeline_value": pipeline_value,
                    "active_opportunities": active_opportunities,
                    "comms_this_month": comms_this_month,
                    "meetings_this_month": meetings_this_month
                },
                "pipeline_stages": pipeline_stages,
                "recent_communications": recent_comms,
                "upcoming_actions": upcoming,
                "recent_stage_changes": recent_stage_changes
            }
        })

    def handle_pipeline_report(self, user):
        conn = get_db()
        stages = rows_to_list(conn.execute("""
            SELECT stage, COUNT(*) as count,
                   COALESCE(SUM(expected_amount), 0) as total_expected,
                   COALESCE(SUM(commitment_amount), 0) as total_committed,
                   COALESCE(AVG(probability), 0) as avg_probability
            FROM opportunities
            WHERE deleted_at IS NULL
            GROUP BY stage
            ORDER BY CASE stage
                WHEN 'lead' THEN 1 WHEN 'engaged' THEN 2
                WHEN 'diligence' THEN 3 WHEN 'commitment' THEN 4
            END
        """).fetchall())

        by_owner = rows_to_list(conn.execute("""
            SELECT u.full_name as owner, op.stage, COUNT(*) as count,
                   COALESCE(SUM(op.expected_amount), 0) as total_expected
            FROM opportunities op
            LEFT JOIN users u ON op.owner_id = u.id
            WHERE op.deleted_at IS NULL
            GROUP BY op.owner_id, op.stage
            ORDER BY u.full_name, op.stage
        """).fetchall())

        by_priority = rows_to_list(conn.execute("""
            SELECT priority, COUNT(*) as count,
                   COALESCE(SUM(expected_amount), 0) as total_expected
            FROM opportunities
            WHERE stage != 'commitment' AND deleted_at IS NULL
            GROUP BY priority
        """).fetchall())

        conn.close()
        return self.send_json({
            "data": {
                "by_stage": stages,
                "by_owner": by_owner,
                "by_priority": by_priority
            }
        })

    def handle_activity_report(self, user, params):
        conn = get_db()
        days = int(params.get('days', 30))

        by_user = rows_to_list(conn.execute("""
            SELECT u.full_name, cm.type, COUNT(*) as count
            FROM communications cm
            LEFT JOIN users u ON cm.created_by = u.id
            WHERE cm.communication_date >= date('now', ?)
            GROUP BY cm.created_by, cm.type
            ORDER BY u.full_name, cm.type
        """, (f'-{days} days',)).fetchall())

        by_day = rows_to_list(conn.execute("""
            SELECT date(communication_date) as date, type, COUNT(*) as count
            FROM communications
            WHERE communication_date >= date('now', ?)
            GROUP BY date(communication_date), type
            ORDER BY date DESC
        """, (f'-{days} days',)).fetchall())

        conn.close()
        return self.send_json({
            "data": {
                "by_user": by_user,
                "by_day": by_day,
                "period_days": days
            }
        })

    # ═══════════════════════════════════════════════════════════════════════════
    # IMPORT / EXPORT HANDLERS
    # ═══════════════════════════════════════════════════════════════════════════

    def handle_import_csv(self, user, body):
        """Import contacts from CSV data (sent as JSON array or raw CSV text)."""
        csv_data = body.get('data', [])
        entity_type = body.get('entity_type', 'contacts')
        mapping = body.get('mapping', {})
        dry_run = body.get('dry_run', False)
        update_existing = bool(body.get('update_existing', True))
        action_overrides_raw = body.get('action_overrides', {}) or {}

        if not csv_data:
            return self.send_error_json("No data provided. Send 'data' as array of objects.")

        conn = get_db()
        results = {"created": 0, "updated": 0, "skipped": 0, "errors": [], "matches": []}
        # Keep in-memory email matches so dry-run mirrors real behavior for
        # duplicate emails appearing multiple times in the same CSV batch.
        batch_email_matches = {}

        try:
            for i, row in enumerate(csv_data):
                try:
                    # Apply field mapping
                    mapped = {}
                    for csv_col, crm_field in mapping.items():
                        if csv_col in row:
                            mapped[crm_field] = row[csv_col]

                    # Use mapped data, fall back to raw row
                    data = mapped if mapping else row

                    if entity_type == 'contacts':
                        first_name = data.get('first_name', '').strip()
                        last_name = data.get('last_name', '').strip()

                        # Try splitting a 'name' field
                        if not first_name and not last_name and data.get('name'):
                            parts = data['name'].strip().split(' ', 1)
                            first_name = parts[0]
                            last_name = parts[1] if len(parts) > 1 else ''

                        if not first_name:
                            results['errors'].append(f"Row {i+1}: Missing first_name")
                            results['skipped'] += 1
                            continue

                        email = data.get('email', '').strip()
                        email_key = email.lower()
                        linkedin_url = data.get('linkedin_url', data.get('linkedin', '')).strip()
                        city = data.get('city', '').strip()
                        state = data.get('state', '').strip()
                        country = data.get('country', '').strip()
                        location_query = data.get('location_query', '').strip()
                        raw_location = data.get('location', data.get('city_location', data.get('city/location', ''))).strip()
                        if raw_location:
                            p_city, p_state, p_country, p_query = _parse_location_text(raw_location)
                            city = city or p_city
                            state = state or p_state
                            country = country or p_country
                            location_query = location_query or p_query

                        # Check for existing contact by email
                        existing = None
                        existing_summary = None
                        if email:
                            if email_key in batch_email_matches:
                                existing_summary = batch_email_matches[email_key]
                                existing = {"id": existing_summary.get('id')}
                            else:
                                existing = conn.execute("""
                                    SELECT c.id, c.first_name, c.last_name, c.email, o.name as organization_name
                                    FROM contacts c
                                    LEFT JOIN organizations o ON c.organization_id = o.id
                                    WHERE lower(c.email) = lower(?)
                                    ORDER BY c.updated_at DESC
                                    LIMIT 1
                                """, (email,)).fetchone()
                                if existing:
                                    existing_summary = {
                                        "id": existing['id'],
                                        "name": f"{str(existing['first_name'] or '').strip()} {str(existing['last_name'] or '').strip()}".strip(),
                                        "email": str(existing['email'] or ''),
                                        "organization": str(existing['organization_name'] or '')
                                    }
                                    batch_email_matches[email_key] = existing_summary

                        # Handle organization
                        org_id = None
                        org_name = data.get('organization', data.get('company', '')).strip()
                        if org_name:
                            org = conn.execute("SELECT id FROM organizations WHERE name = ?", (org_name,)).fetchone()
                            if org:
                                org_id = org['id']
                            elif not dry_run:
                                org_id = generate_id()
                                conn.execute(
                                    "INSERT INTO organizations (id, name, created_by) VALUES (?, ?, ?)",
                                    (org_id, org_name, user['user_id'])
                                )

                        action_override = None
                        if isinstance(action_overrides_raw, dict):
                            action_override = action_overrides_raw.get(str(i + 1)) or action_overrides_raw.get(i + 1)
                        default_action = 'update' if update_existing else 'skip'
                        action = action_override if action_override in ('update', 'skip', 'create_duplicate') else default_action
                        if existing:
                            incoming_name = f"{first_name} {last_name}".strip()
                            results['matches'].append({
                                "row": i + 1,
                                "incoming_name": incoming_name,
                                "incoming_email": email,
                                "incoming_organization": org_name,
                                "existing_id": existing_summary.get('id') if isinstance(existing_summary, dict) else existing['id'],
                                "existing_name": existing_summary.get('name') if isinstance(existing_summary, dict) else '',
                                "existing_email": existing_summary.get('email') if isinstance(existing_summary, dict) else email,
                                "existing_organization": existing_summary.get('organization') if isinstance(existing_summary, dict) else '',
                                "default_action": default_action,
                                "action": action
                            })

                        if not dry_run:
                            if existing:
                                if action == 'update':
                                    conn.execute("""
                                        UPDATE contacts SET first_name=?, last_name=?, phone=?, title=?,
                                        organization_id=COALESCE(?, organization_id),
                                        linkedin_url=COALESCE(?, linkedin_url),
                                        city=COALESCE(?, city),
                                        state=COALESCE(?, state),
                                        country=COALESCE(?, country),
                                        location_query=COALESCE(?, location_query),
                                        updated_at=?
                                        WHERE id=?
                                    """, (first_name, last_name, data.get('phone'),
                                          data.get('title'), org_id,
                                          linkedin_url if linkedin_url else None,
                                          city if city else None,
                                          state if state else None,
                                          country if country else None,
                                          location_query if location_query else None,
                                      now(), existing['id']))
                                    if email:
                                        batch_email_matches[email_key] = {
                                            "id": existing['id'],
                                            "name": f"{first_name} {last_name}".strip(),
                                            "email": email,
                                            "organization": org_name
                                        }
                                    updated_contact = row_to_dict(conn.execute("""
                                        SELECT c.*, o.name as organization_name
                                        FROM contacts c LEFT JOIN organizations o ON c.organization_id = o.id
                                        WHERE c.id = ?
                                    """, (existing['id'],)).fetchone())
                                    _sync_contact_to_fundraising_state(conn, updated_contact, actor_user_id=user['user_id'], remove=False)
                                    results['updated'] += 1
                                elif action == 'create_duplicate':
                                    contact_id = generate_id()
                                    conn.execute("""
                                        INSERT INTO contacts (id, first_name, last_name, email, phone,
                                            title, organization_id, status, source,
                                            linkedin_url, city, state, country, location_query, created_by)
                                        VALUES (?, ?, ?, ?, ?, ?, ?, 'active', 'import', ?, ?, ?, ?, ?, ?)
                                    """, (contact_id, first_name, last_name, email,
                                          data.get('phone'), data.get('title'), org_id,
                                          linkedin_url,
                                          city, state, country, location_query, user['user_id']))
                                    if email:
                                        batch_email_matches[email_key] = {
                                            "id": contact_id,
                                            "name": f"{first_name} {last_name}".strip(),
                                            "email": email,
                                            "organization": org_name
                                        }
                                    created_contact = row_to_dict(conn.execute("""
                                        SELECT c.*, o.name as organization_name
                                        FROM contacts c LEFT JOIN organizations o ON c.organization_id = o.id
                                        WHERE c.id = ?
                                    """, (contact_id,)).fetchone())
                                    _sync_contact_to_fundraising_state(conn, created_contact, actor_user_id=user['user_id'], remove=False)
                                    results['created'] += 1
                                else:
                                    results['skipped'] += 1
                                    results['errors'].append(f"Row {i+1}: Existing contact matched by email; skipped")
                            else:
                                contact_id = generate_id()
                                conn.execute("""
                                    INSERT INTO contacts (id, first_name, last_name, email, phone,
                                        title, organization_id, status, source,
                                        linkedin_url, city, state, country, location_query, created_by)
                                    VALUES (?, ?, ?, ?, ?, ?, ?, 'active', 'import', ?, ?, ?, ?, ?, ?)
                                """, (contact_id, first_name, last_name, email,
                                      data.get('phone'), data.get('title'), org_id,
                                      linkedin_url,
                                      city, state, country, location_query, user['user_id']))
                                if email:
                                    batch_email_matches[email_key] = {
                                        "id": contact_id,
                                        "name": f"{first_name} {last_name}".strip(),
                                        "email": email,
                                        "organization": org_name
                                    }
                                created_contact = row_to_dict(conn.execute("""
                                    SELECT c.*, o.name as organization_name
                                    FROM contacts c LEFT JOIN organizations o ON c.organization_id = o.id
                                    WHERE c.id = ?
                                """, (contact_id,)).fetchone())
                                _sync_contact_to_fundraising_state(conn, created_contact, actor_user_id=user['user_id'], remove=False)
                                results['created'] += 1
                        else:
                            if existing:
                                if action == 'update':
                                    results['updated'] += 1
                                    if email:
                                        batch_email_matches[email_key] = {
                                            "id": existing['id'],
                                            "name": f"{first_name} {last_name}".strip(),
                                            "email": email,
                                            "organization": org_name
                                        }
                                elif action == 'create_duplicate':
                                    results['created'] += 1
                                    if email:
                                        batch_email_matches[email_key] = {
                                            "id": f"dryrun-{i+1}",
                                            "name": f"{first_name} {last_name}".strip(),
                                            "email": email,
                                            "organization": org_name
                                        }
                                else:
                                    results['skipped'] += 1
                                    results['errors'].append(f"Row {i+1}: Existing contact matched by email; would be skipped")
                            else:
                                results['created'] += 1
                                if email:
                                    # Simulate that the row now exists for subsequent duplicate-email rows.
                                    batch_email_matches[email_key] = {
                                        "id": f"dryrun-{i+1}",
                                        "name": f"{first_name} {last_name}".strip(),
                                        "email": email,
                                        "organization": org_name
                                    }

                except Exception as e:
                    results['errors'].append(f"Row {i+1}: {str(e)}")
                    results['skipped'] += 1

            if not dry_run:
                conn.commit()

        except Exception as e:
            conn.rollback()
            conn.close()
            return self.send_error_json(f"Import failed: {str(e)}", 500)

        conn.close()
        return self.send_json({
            "data": results,
            "dry_run": dry_run
        })

    def handle_export_contacts(self, user, params):
        conn = get_db()
        contacts = rows_to_list(conn.execute("""
            SELECT c.*, o.name as organization_name
            FROM contacts c
            LEFT JOIN organizations o ON c.organization_id = o.id
            ORDER BY c.last_name, c.first_name
        """).fetchall())
        conn.close()
        return self.send_json({"data": contacts})

    # ═══════════════════════════════════════════════════════════════════════════
    # TAGS / USERS / AUDIT
    # ═══════════════════════════════════════════════════════════════════════════

    def handle_list_tags(self, user):
        conn = get_db()
        tags = rows_to_list(conn.execute("SELECT * FROM tags ORDER BY name").fetchall())
        conn.close()
        return self.send_json({"data": tags})

    def handle_create_tag(self, user, body):
        if not body.get('name'):
            return self.send_error_json("name is required")
        conn = get_db()
        tag_id = generate_id()
        try:
            conn.execute("INSERT INTO tags (id, name, color) VALUES (?, ?, ?)",
                         (tag_id, body['name'], body.get('color', '#6366f1')))
            conn.commit()
        except sqlite3.IntegrityError:
            conn.close()
            return self.send_error_json("Tag already exists")
        tag = row_to_dict(conn.execute("SELECT * FROM tags WHERE id = ?", (tag_id,)).fetchone())
        conn.close()
        return self.send_json({"data": tag}, 201)

    def handle_delete_tag(self, user, tag_id):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)
        conn = get_db()
        existing = conn.execute("SELECT id FROM tags WHERE id = ?", (tag_id,)).fetchone()
        if not existing:
            conn.close()
            return self.send_error_json("Tag not found", 404)
        conn.execute("DELETE FROM tags WHERE id = ?", (tag_id,))
        log_audit(conn, user['user_id'], 'tag', tag_id, 'delete')
        conn.commit()
        conn.close()
        return self.send_json({"message": "Tag deleted"})

    def handle_system_status(self, user):
        """System / search-index health for the in-app status view (DB-derived)."""
        conn = get_db()
        out = {}
        try:
            live = "deleted_at IS NULL"
            out['canonical_entities'] = {
                'investor': conn.execute(f"SELECT COUNT(*) FROM canonical_entities WHERE entity_kind='investor' AND {live}").fetchone()[0],
                'person': conn.execute(f"SELECT COUNT(*) FROM canonical_entities WHERE entity_kind='person' AND {live}").fetchone()[0],
            }
            out['entity_links'] = conn.execute("SELECT COUNT(*) FROM entity_links").fetchone()[0]
        except Exception:
            out['canonical_entities'] = None
        try:
            r = conn.execute("SELECT ts, payload FROM interaction_log WHERE action='ingest.sync' ORDER BY ts DESC LIMIT 1").fetchone()
            out['last_index_sync'] = ({'ts': r['ts'], **json.loads(r['payload'] or '{}')} if r else None)
        except Exception:
            out['last_index_sync'] = None
        try:
            out['thesis'] = {
                'lines': conn.execute("SELECT COUNT(*) FROM thesis_lines WHERE deleted_at IS NULL").fetchone()[0],
                'canonical_versions': conn.execute("SELECT COUNT(*) FROM thesis_versions WHERE status='canonical'").fetchone()[0],
                'in_review': conn.execute("SELECT COUNT(*) FROM thesis_versions WHERE status='in_review'").fetchone()[0],
            }
        except Exception:
            out['thesis'] = None
        try:
            out['recent_activity'] = [dict(r) for r in conn.execute(
                "SELECT ts, actor_type, actor_id, action FROM interaction_log ORDER BY ts DESC LIMIT 12")]
        except Exception:
            out['recent_activity'] = []
        try:
            # Count only candidates whose both sides are still live (mirror the
            # review queue in entity_merge.list_candidates) — self-healed twins
            # whose duplicate was soft-deleted no longer count as pending work.
            out['pending_merge_candidates'] = conn.execute(
                """SELECT COUNT(*) FROM entity_merge_candidates mc
                     JOIN canonical_entities a ON a.id=mc.entity_a AND a.deleted_at IS NULL
                     JOIN canonical_entities b ON b.id=mc.entity_b AND b.deleted_at IS NULL
                    WHERE mc.status='pending'""").fetchone()[0]
        except Exception:
            out['pending_merge_candidates'] = None
        out['index_job'] = entity_jobs.get_status() if entity_jobs else None
        # Raw source-record counts, so the resolved canonical numbers can be
        # sanity-checked against what's actually in the CRM.
        try:
            out['source_counts'] = {
                'contacts': conn.execute("SELECT COUNT(*) FROM contacts WHERE deleted_at IS NULL").fetchone()[0],
                'organizations': conn.execute("SELECT COUNT(*) FROM organizations WHERE deleted_at IS NULL").fetchone()[0],
                'fundraising_investors': conn.execute("SELECT COUNT(*) FROM fundraising_investors").fetchone()[0],
                'fundraising_contacts': conn.execute("SELECT COUNT(*) FROM fundraising_contacts").fetchone()[0],
            }
        except Exception:
            out['source_counts'] = None
        # Storage usage — DB file(s), email attachments, backups, and disk headroom,
        # so growth can be watched over time. Best-effort; never fails the status call.
        try:
            import shutil

            def _fsize(p):
                try:
                    return os.path.getsize(p)
                except OSError:
                    return 0

            def _dirsize(d):
                total = 0
                for root, _dirs, files in os.walk(d):
                    for f in files:
                        try:
                            total += os.path.getsize(os.path.join(root, f))
                        except OSError:
                            pass
                return total

            du = shutil.disk_usage(DATA_DIR)
            out['storage'] = {
                'database_bytes': sum(_fsize(DB_PATH + s) for s in ("", "-wal", "-shm")),
                'attachments_bytes': _dirsize(os.path.join(DATA_DIR, "email_attachments")),
                'backups_bytes': _dirsize(os.path.join(DATA_DIR, "backups")),
                'disk_total_bytes': du.total,
                'disk_used_bytes': du.used,
                'disk_free_bytes': du.free,
            }
        except Exception:
            out['storage'] = None
        conn.close()
        self.send_json({"data": out})

    def handle_list_activity_proposals(self, user):
        if not require_admin(user):
            return self.send_error_json("Admin required", 403)
        conn = get_db()
        try:
            return self.send_json({"proposals": list_email_activity_proposals(conn, status="pending")})
        finally:
            conn.close()

    # ─── Soft-deleted purge (admin maintenance) ──────────────────────────────────
    # Lists soft-deleted rows and HARD-deletes them — a deliberate, admin-only, type-to-confirm
    # exception to the never-hard-delete rule, for clearing out dummy/test data. A purge can only
    # ever touch a soft-deleted row, and refuses any contact/org whose delete would CASCADE or
    # SET NULL onto LIVE data, so it can never remove or mutate a live record.
    _PURGE_TABLES = ('contacts', 'organizations', 'opportunities', 'communications')

    def handle_list_soft_deleted(self, user):
        if not require_admin(user):
            return self.send_error_json("Admin required", 403)
        conn = get_db()
        try:
            groups = {}
            groups['contacts'] = [
                {"id": r["id"],
                 "label": (f"{r['first_name'] or ''} {r['last_name'] or ''}".strip() or r["email"] or r["id"]),
                 "deleted_at": r["deleted_at"]}
                for r in conn.execute(
                    "SELECT id, first_name, last_name, email, deleted_at FROM contacts "
                    "WHERE deleted_at IS NOT NULL ORDER BY deleted_at DESC").fetchall()]
            groups['organizations'] = [
                {"id": r["id"], "label": (r["name"] or r["id"]), "deleted_at": r["deleted_at"]}
                for r in conn.execute(
                    "SELECT id, name, deleted_at FROM organizations "
                    "WHERE deleted_at IS NOT NULL ORDER BY deleted_at DESC").fetchall()]
            groups['opportunities'] = [
                {"id": r["id"], "label": (r["name"] or r["id"]), "deleted_at": r["deleted_at"]}
                for r in conn.execute(
                    "SELECT id, name, deleted_at FROM opportunities "
                    "WHERE deleted_at IS NOT NULL ORDER BY deleted_at DESC").fetchall()]
            groups['communications'] = [
                {"id": r["id"],
                 "label": ((r["subject"] or (r["type"] or "note"))
                           + (f" · {(r['communication_date'] or '')[:10]}" if r["communication_date"] else "")),
                 "deleted_at": r["deleted_at"]}
                for r in conn.execute(
                    "SELECT id, type, subject, communication_date, deleted_at FROM communications "
                    "WHERE deleted_at IS NOT NULL ORDER BY deleted_at DESC").fetchall()]
            return self.send_json({"groups": groups, "total": sum(len(v) for v in groups.values())})
        finally:
            conn.close()

    def handle_purge_soft_deleted(self, user, body):
        if not require_admin(user):
            return self.send_error_json("Admin required", 403)
        body = body or {}
        table = str(body.get('table') or '').strip()
        row_id = str(body.get('id') or '').strip()
        if table not in self._PURGE_TABLES:   # validated -> safe to interpolate below
            return self.send_error_json("Unknown table", 400)
        if not row_id:
            return self.send_error_json("id is required", 400)
        conn = get_db()
        try:
            row = conn.execute(f"SELECT id, deleted_at FROM {table} WHERE id = ?", (row_id,)).fetchone()
            if not row:
                return self.send_error_json("Not found", 404)
            if not row["deleted_at"]:
                return self.send_error_json("Only soft-deleted rows can be purged", 400)
            # A contacts/organizations delete cascades / SET NULLs onto children. Refuse if any LIVE
            # child exists so a purge can never touch live data (a soft-deleted parent's children were
            # soft-deleted with it, so normally there are none).
            if table == 'contacts':
                live = conn.execute(
                    "SELECT (SELECT COUNT(*) FROM opportunities WHERE contact_id=? AND deleted_at IS NULL) + "
                    "(SELECT COUNT(*) FROM communications WHERE contact_id=? AND deleted_at IS NULL) AS n",
                    (row_id, row_id)).fetchone()["n"]
                if live:
                    return self.send_error_json("This contact still has live communications or opportunities — cannot purge", 409)
                # Drop the optional logical FKs that have no ON DELETE (so a purged contact leaves no
                # dangling reference): the derived grid link (fundraising_contacts.contact_id, migration
                # 0004) and any reminder's contact_id (migration 0006 — the reminder's real link is
                # investor_id, which is unaffected). Both are bare TEXT columns, not declared FKs.
                conn.execute("UPDATE fundraising_contacts SET contact_id = NULL WHERE contact_id = ?", (row_id,))
                conn.execute("UPDATE reminders SET contact_id = NULL WHERE contact_id = ?", (row_id,))
            elif table == 'organizations':
                live = conn.execute(
                    "SELECT (SELECT COUNT(*) FROM contacts WHERE organization_id=? AND deleted_at IS NULL) + "
                    "(SELECT COUNT(*) FROM opportunities WHERE organization_id=? AND deleted_at IS NULL) AS n",
                    (row_id, row_id)).fetchone()["n"]
                if live:
                    return self.send_error_json("This organization is still linked to live contacts or opportunities — cannot purge", 409)
            # CASCADE removes the (soft-deleted) children for contacts; the rest are leaves.
            conn.execute(f"DELETE FROM {table} WHERE id = ?", (row_id,))
            log_audit(conn, user['user_id'], table, row_id, 'purge', {"table": table})
            conn.commit()
            return self.send_json({"data": {"purged": True, "table": table, "id": row_id}})
        finally:
            conn.close()

    # ─── TEMPORARY (v0.1.0:105) — DELETE AFTER the contacts<->fundraising_contacts census ──
    # A throwaway admin diagnostic for the deferred consolidation (ROADMAP backlog): mirrors
    # backend/scripts/contacts_census.sql so the A/B/C populations can be read from the box
    # without shell access. Counts only — no names/PII. Remove this handler + its route + the
    # Settings "Contacts census" button once the numbers are captured.
    def handle_contacts_census(self, user):
        if not require_admin(user):
            return self.send_error_json("Admin required", 403)
        conn = get_db()
        try:
            def n(sql):
                return conn.execute(sql).fetchone()[0]
            no_pill = ("NOT EXISTS (SELECT 1 FROM fundraising_contacts fc WHERE fc.contact_id = c.id)")
            return self.send_json({"data": {
                "total_live_contacts": n("SELECT COUNT(*) FROM contacts WHERE deleted_at IS NULL"),
                "A_linked": n("SELECT COUNT(*) FROM contacts c WHERE c.deleted_at IS NULL "
                              "AND EXISTS (SELECT 1 FROM fundraising_contacts fc WHERE fc.contact_id = c.id)"),
                "B_contacts_only": n(f"SELECT COUNT(*) FROM contacts c WHERE c.deleted_at IS NULL AND {no_pill}"),
                "B_investor": n(f"SELECT COUNT(*) FROM contacts c WHERE c.deleted_at IS NULL AND c.contact_type='investor' AND {no_pill}"),
                "B_prospect": n(f"SELECT COUNT(*) FROM contacts c WHERE c.deleted_at IS NULL AND c.contact_type='prospect' AND {no_pill}"),
                "B_with_live_communication": n(f"SELECT COUNT(*) FROM contacts c WHERE c.deleted_at IS NULL AND {no_pill} "
                                               "AND EXISTS (SELECT 1 FROM communications cm WHERE cm.contact_id=c.id AND cm.deleted_at IS NULL)"),
                "B_with_live_opportunity": n(f"SELECT COUNT(*) FROM contacts c WHERE c.deleted_at IS NULL AND {no_pill} "
                                             "AND EXISTS (SELECT 1 FROM opportunities o WHERE o.contact_id=c.id AND o.deleted_at IS NULL)"),
                "C_pill_only": n("SELECT COUNT(*) FROM fundraising_contacts WHERE contact_id IS NULL"),
                "dangling_pills": n("SELECT COUNT(*) FROM fundraising_contacts fc WHERE fc.contact_id IS NOT NULL "
                                    "AND NOT EXISTS (SELECT 1 FROM contacts c WHERE c.id=fc.contact_id AND c.deleted_at IS NULL)"),
                "total_grid_pills": n("SELECT COUNT(*) FROM fundraising_contacts"),
                "total_grid_rows": n("SELECT COUNT(*) FROM fundraising_investors"),
            }})
        finally:
            conn.close()

    def handle_decide_activity_proposal(self, user, proposal_id, decision, body):
        if not require_admin(user):
            return self.send_error_json("Admin required", 403)
        conn = get_db()
        try:
            res = decide_email_activity_proposal(conn, proposal_id, decision,
                                                 user['user_id'], (body or {}).get('note'))
        finally:
            conn.close()
        if res.get("error"):
            code = {"not_found": 404, "already_decided": 409}.get(res["error"], 400)
            return self.send_error_json(res["error"], code)
        return self.send_json({"data": res})

    # ─── Matrix review-bot bridge for email-activity proposals (bot-or-admin) ───
    def handle_list_bot_email_proposals(self, user):
        """The bot's poll endpoint: {to_post, open, to_close}. Bot-or-admin (the proposals
        carry LP email content, so this stays off the member tier)."""
        if not require_bot_or_admin(user):
            return self.send_error_json("Bot or admin required", 403)
        conn = get_db()
        try:
            return self.send_json({"data": list_bot_email_proposals(conn)})
        finally:
            conn.close()

    def handle_mark_email_proposal_matrix(self, user, proposal_id, body):
        """Record Matrix thread state: {event_id} marks the card posted; {closed:true} marks
        the thread resolved after the bot announced a web-side decision."""
        if not require_bot_or_admin(user):
            return self.send_error_json("Bot or admin required", 403)
        body = body or {}
        conn = get_db()
        try:
            if body.get("closed"):
                res = mark_proposal_matrix_closed(conn, proposal_id)
            else:
                event_id = str(body.get("event_id") or "").strip()
                if not event_id:
                    return self.send_error_json("event_id or closed is required")
                res = mark_proposal_matrix_posted(conn, proposal_id, event_id)
        finally:
            conn.close()
        if res.get("error"):
            return self.send_error_json(res["error"], 404 if res["error"] == "not_found" else 400)
        return self.send_json({"data": res})

    def handle_decide_email_proposal_matrix(self, user, proposal_id, body):
        """In-thread Matrix decision relayed by the bot: approve/dismiss (+ optional edited note),
        tagged source='matrix' and closing the thread in the same transaction."""
        if not require_bot_or_admin(user):
            return self.send_error_json("Bot or admin required", 403)
        body = body or {}
        decision = str(body.get("decision") or "").strip()
        if decision not in ("approve", "dismiss"):
            return self.send_error_json("decision must be approve or dismiss")
        conn = get_db()
        try:
            res = decide_email_activity_proposal(conn, proposal_id, decision, user['user_id'],
                                                 body.get('note'), source="matrix", close_matrix=True)
        finally:
            conn.close()
        if res.get("error"):
            code = {"not_found": 404, "already_decided": 409}.get(res["error"], 400)
            return self.send_error_json(res["error"], code)
        return self.send_json({"data": res})

    # ─── UI-triggered index jobs + entity-merge review (Phase 1) ───
    def handle_index_job(self, user, kind):
        if not require_admin(user):
            return self.send_error_json("Admin required", 403)
        if entity_jobs is None:
            return self.send_error_json("Jobs unavailable", 503)
        res = entity_jobs.start(kind, DB_PATH)
        if res.get('error'):
            return self.send_error_json(res['error'], 409)
        return self.send_json({"data": res})

    def handle_list_merge_candidates(self, user, params):
        if not require_admin(user):
            return self.send_error_json("Admin required", 403)
        if entity_merge is None:
            return self.send_error_json("Unavailable", 503)
        return self.send_json(entity_merge.list_candidates(DB_PATH, params.get('status', 'pending')))

    def handle_decide_merge_candidate(self, user, candidate_id, body):
        if not require_admin(user):
            return self.send_error_json("Admin required", 403)
        if entity_merge is None:
            return self.send_error_json("Unavailable", 503)
        res = entity_merge.decide(DB_PATH, candidate_id, (body or {}).get('decision'), user['user_id'])
        if res.get('error'):
            return self.send_error_json(res['error'], 400)
        return self.send_json({"data": res})

    # ─── Architect thesis authoring (Phase 1) ───
    def handle_get_thesis_tree(self, user, line_key):
        if _architect_tools is None:
            return self.send_error_json("Unavailable", 503)
        return self.send_json(_architect_tools.get_thesis(line_key, DB_PATH))

    def handle_create_thesis_line(self, user, body):
        if not require_admin(user):
            return self.send_error_json("Admin required", 403)
        if _architect_tools is None:
            return self.send_error_json("Unavailable", 503)
        body = body or {}
        if not body.get('line_key') or not body.get('name'):
            return self.send_error_json("line_key and name required", 400)
        return self.send_json({"data": _architect_tools.create_thesis_line(
            body['line_key'], body['name'], segment_key=body.get('segment_key'),
            is_core=bool(body.get('is_core')), description=body.get('description'), db=DB_PATH)})

    def handle_add_thesis_node(self, user, line_key, body):
        if not require_admin(user):
            return self.send_error_json("Admin required", 403)
        if _architect_tools is None:
            return self.send_error_json("Unavailable", 503)
        conn = get_db()
        row = conn.execute("SELECT id FROM thesis_lines WHERE line_key=? AND deleted_at IS NULL", (line_key,)).fetchone()
        conn.close()
        if not row:
            return self.send_error_json("Line not found", 404)
        body = body or {}
        return self.send_json({"data": _architect_tools.upsert_thesis_node(
            row['id'], body.get('node_type', 'claim'), body.get('body', ''), title=body.get('title'),
            parent_id=body.get('parent_id'), node_id=body.get('node_id'),
            variant_group=body.get('variant_group'), change_reason=body.get('change_reason'), db=DB_PATH)})

    # ─── Architect agent (Phase 1, runs on Claude) ───
    def handle_architect_status(self, user):
        if _architect_agent is None:
            return self.send_error_json("Unavailable", 503)
        return self.send_json({"data": _architect_agent.status()})

    def handle_get_node_variants(self, user, node_id):
        if _architect_tools is None:
            return self.send_error_json("Unavailable", 503)
        return self.send_json(_architect_tools.get_node_variants(node_id, DB_PATH))

    def _architect_line_key(self, node_id):
        conn = get_db()
        row = conn.execute("SELECT l.line_key FROM thesis_nodes n JOIN thesis_lines l ON l.id=n.line_id WHERE n.id=?",
                           (node_id,)).fetchone()
        conn.close()
        return row['line_key'] if row else None

    def handle_edit_thesis_node(self, user, node_id, body):
        """Manual human edit of a node's title/body (no Architect)."""
        if not require_admin(user):
            return self.send_error_json("Admin required", 403)
        if _architect_tools is None:
            return self.send_error_json("Unavailable", 503)
        node = _architect_tools.get_node(node_id, db=DB_PATH)
        if node.get('error'):
            return self.send_error_json("Node not found", 404)
        body = body or {}
        res = _architect_tools.upsert_thesis_node(
            node['line_id'], node['node_type'], body.get('body', node.get('body') or ''),
            title=body.get('title', node.get('title')), node_id=node_id,
            change_reason='manual edit', actor_id=user['user_id'], actor_type='human', db=DB_PATH)
        return self.send_json({"data": res})

    def handle_retire_thesis_node(self, user, node_id):
        """Soft-delete a node + its subtree."""
        if not require_admin(user):
            return self.send_error_json("Admin required", 403)
        if _architect_tools is None:
            return self.send_error_json("Unavailable", 503)
        res = _architect_tools.retire_node(node_id, actor_id=user['user_id'], db=DB_PATH)
        if res.get('error'):
            return self.send_error_json("Node not found", 404)
        return self.send_json({"data": res})

    def handle_choose_variant(self, user, node_id):
        """'Use this option' — keep this variant, retire its siblings."""
        if not require_admin(user):
            return self.send_error_json("Admin required", 403)
        if _architect_tools is None:
            return self.send_error_json("Unavailable", 503)
        res = _architect_tools.choose_variant(node_id, actor_id=user['user_id'], db=DB_PATH)
        if res.get('error'):
            return self.send_error_json("Node not found", 404)
        return self.send_json({"data": res})

    def handle_approve_line(self, user, line_key):
        """One-click 'approve as current': record this admin's approval on the line's
        in-review version (creating + submitting one from the live tree if none exists);
        promotes to canonical once the required distinct approvals are reached."""
        if not require_admin(user):
            return self.send_error_json("Admin required", 403)
        if _architect_tools is None or thesis_review is None:
            return self.send_error_json("Unavailable", 503)
        conn = get_db()
        line = conn.execute("SELECT id FROM thesis_lines WHERE line_key=? AND deleted_at IS NULL", (line_key,)).fetchone()
        v = None
        if line:
            v = conn.execute("SELECT id FROM thesis_versions WHERE line_id=? AND status='in_review' ORDER BY version_no DESC LIMIT 1",
                             (line['id'],)).fetchone()
        conn.close()
        if not line:
            return self.send_error_json("Line not found", 404)
        if v:
            version_id = v['id']
        else:
            ver = _architect_tools.create_thesis_version(line_key, rationale="Approved as current (Workshop)",
                                                         created_by=user['user_id'], db=DB_PATH)
            if ver.get('error'):
                return self.send_error_json(ver['error'], 400)
            _architect_tools.submit_version_for_review(ver['id'], db=DB_PATH)
            version_id = ver['id']
        res = thesis_review.record_review(DB_PATH, version_id, user['user_id'], 'approve')
        return self.send_json({"data": res})

    def handle_generate_options(self, user, node_id, body):
        if not require_admin(user):
            return self.send_error_json("Admin required", 403)
        if _architect_agent is None:
            return self.send_error_json("Unavailable", 503)
        lk = self._architect_line_key(node_id)
        if not lk:
            return self.send_error_json("Node not found", 404)
        body = body or {}
        try:
            res = _architect_agent.generate_options(lk, node_id, int(body.get('n', 3) or 3),
                                                    body.get('guidance', '') or '', DB_PATH)
        except Exception as exc:
            return self.send_error_json(str(exc), 502)
        if res.get('error'):
            return self.send_error_json(res.get('raw') or res['error'], 502)
        return self.send_json({"data": res})

    def handle_node_feedback(self, user, node_id, body):
        if not require_admin(user):
            return self.send_error_json("Admin required", 403)
        if _architect_agent is None:
            return self.send_error_json("Unavailable", 503)
        lk = self._architect_line_key(node_id)
        if not lk:
            return self.send_error_json("Node not found", 404)
        body = body or {}
        try:
            res = _architect_agent.revise(lk, node_id, body.get('feedback', '') or '',
                                          int(body.get('n', 2) or 2), DB_PATH)
        except Exception as exc:
            return self.send_error_json(str(exc), 502)
        if res.get('error'):
            return self.send_error_json(res.get('raw') or res['error'], 502)
        return self.send_json({"data": res})

    def _ground_feedback_corpus(self, conn, limit=60):
        """Raw LP-feedback prose for grounding, newest-first, balanced across sources:
        matched email bodies (the richest objection signal), logged communications, and
        fundraising grid notes. Sensitive Tier-2-heavy text; ONLY ever passed into the
        redaction boundary, never to Claude directly."""
        # Email bodies are capped per item (long threads/quote-chains) to keep the local
        # minimize tractable; only `matched` emails (tied to a known investor/contact) are
        # pulled. Sources are round-robin merged so email is always represented even when
        # communications/notes are plentiful, rather than crowded out by a flat LIMIT.
        sources = (
            "SELECT SUBSTR(body_text,1,4000) FROM emails WHERE match_status='matched' "
            "AND body_text IS NOT NULL AND TRIM(body_text)<>'' ORDER BY sent_at DESC LIMIT ?",
            "SELECT body FROM communications WHERE body IS NOT NULL AND TRIM(body)<>'' "
            "ORDER BY communication_date DESC LIMIT ?",
            "SELECT notes FROM fundraising_investors WHERE notes IS NOT NULL AND TRIM(notes)<>'' LIMIT ?",
        )
        buckets = []
        for q in sources:
            try:
                buckets.append([r[0] for r in conn.execute(q, (limit,))])
            except Exception:
                buckets.append([])   # table absent (e.g. email integration not migrated) -> skip
        items, i = [], 0
        while len(items) < limit and any(i < len(b) for b in buckets):
            for b in buckets:
                if i < len(b):
                    items.append(b[i])
                    if len(items) >= limit:
                        break
            i += 1
        return items

    def handle_architect_ground(self, user, body):
        """Ground an objection register in real LP feedback THROUGH the redaction boundary
        (Workstream D). Retrieval + minimization + scrub stay local; only the de-identified
        register reaches Claude; the re-hydrated draft is for human review (guardrail #4)."""
        if not require_admin(user):
            return self.send_error_json("Admin required", 403)
        if _architect_grounding is None:
            return self.send_error_json("Unavailable", 503)
        body = body or {}
        segment_key = body.get('segment_key')
        feedback = body.get('feedback_items')
        conn = get_db()
        try:
            if not feedback:
                feedback = self._ground_feedback_corpus(conn)
            if not feedback:
                return self.send_error_json("No LP feedback found to ground against", 404)
            res = _architect_grounding.ground_objections(feedback, segment_key=segment_key,
                                                         db_path=DB_PATH, conn=conn)
        except Exception as exc:
            return self.send_error_json(str(exc), 502)
        finally:
            conn.close()
        return self.send_json({"data": res})

    def handle_list_outreach_investors(self, user):
        conn = get_db()
        try:
            rows = conn.execute("SELECT id, investor_name FROM fundraising_investors "
                                "ORDER BY investor_name LIMIT 2000").fetchall()
            return self.send_json({"investors": [{"id": r["id"], "name": r["investor_name"]} for r in rows]})
        finally:
            conn.close()

    def handle_outreach_radar(self, user):
        """Deterministic 'who needs attention' scan (reasons are checkable, not LLM guesses)."""
        if _outreach_agent is None:
            return self.send_error_json("Outreach agent unavailable", 503)
        conn = get_db()
        try:
            try:
                own = [r[0] for r in conn.execute("SELECT email_address FROM email_accounts")]
            except Exception:
                own = []
            items = _outreach_agent.follow_up_radar(conn, own, now())
            return self.send_json({"items": items})
        finally:
            conn.close()

    def handle_outreach_draft(self, user, body):
        """Draft tailored LP outreach through the redaction boundary (draft-only —
        a human reviews/edits/sends; guardrails #4, #6)."""
        if _outreach_agent is None:
            return self.send_error_json("Outreach agent unavailable", 503)
        body = body or {}
        inv = body.get('investor_id')
        if not inv:
            return self.send_error_json("investor_id required", 400)
        conn = get_db()
        try:
            sender_email = None
            try:
                r = conn.execute("SELECT email FROM users WHERE id=?", (user.get('user_id'),)).fetchone()
                sender_email = r[0] if r else None
            except Exception:
                pass
            res = _outreach_agent.draft_outreach(conn, inv, body.get('outreach_type', 'follow_up'),
                                                 body.get('guidance', '') or '', DB_PATH, sender_email=sender_email)
            try:
                conn.execute(
                    "INSERT INTO interaction_log (id, ts, actor_type, actor_id, action, target_type, target_id, payload, source, created_at) "
                    "VALUES (?,?,?,?,?,?,?,?,?,?)",
                    (generate_id(), now(), "human", user.get('user_id'), "outreach.drafted",
                     "fundraising_investor", inv,
                     json.dumps({"type": body.get('outreach_type'), "status": res.get('status')}), "crm_ui", now()))
                conn.commit()
            except Exception:
                pass
        except Exception as exc:
            return self.send_error_json(str(exc), 502)
        finally:
            conn.close()
        return self.send_json({"data": res})

    def handle_outreach_gmail_draft(self, user, body):
        """Create a Gmail DRAFT from an approved outreach draft (in-thread reply when there
        is an active thread). Never sends — the human sends from Gmail (guardrails #4, #6)."""
        body = body or {}
        inv = body.get('investor_id')
        text = body.get('draft') or ''
        if not inv or not text.strip():
            return self.send_error_json("investor_id and draft required", 400)
        try:
            from email_integration import compose as _compose
        except Exception:
            return self.send_error_json("Gmail compose unavailable", 503)
        conn = get_db()
        try:
            sender_email = None
            try:
                r = conn.execute("SELECT email FROM users WHERE id=?", (user.get('user_id'),)).fetchone()
                sender_email = r[0] if r else None
            except Exception:
                pass
            res = _compose.create_outreach_draft(conn, sender_email, inv, text)
            try:
                conn.execute(
                    "INSERT INTO interaction_log (id, ts, actor_type, actor_id, action, target_type, target_id, payload, source, created_at) "
                    "VALUES (?,?,?,?,?,?,?,?,?,?)",
                    (generate_id(), now(), "human", user.get('user_id'), "outreach.gmail_draft_created",
                     "fundraising_investor", inv, json.dumps({"status": res.get('status')}), "crm_ui", now()))
                conn.commit()
            except Exception:
                pass
        finally:
            conn.close()
        return self.send_json({"data": res})

    # ─── Architect thesis (Phase 1) ───
    def handle_list_thesis_lines(self, user):
        if thesis_review is None:
            return self.send_error_json("Thesis module unavailable", 503)
        return self.send_json(thesis_review.list_lines(DB_PATH))

    def handle_list_thesis_review_queue(self, user):
        if thesis_review is None:
            return self.send_error_json("Thesis module unavailable", 503)
        return self.send_json(thesis_review.list_versions_for_review(DB_PATH))

    def handle_get_thesis_version(self, user, version_id):
        if thesis_review is None:
            return self.send_error_json("Thesis module unavailable", 503)
        return self.send_json(thesis_review.get_version(DB_PATH, version_id))

    def handle_get_canonical_thesis(self, user, line_key):
        if thesis_review is None:
            return self.send_error_json("Thesis module unavailable", 503)
        return self.send_json(thesis_review.get_canonical(DB_PATH, line_key))

    def handle_thesis_review(self, user, version_id, body):
        # Promotion to canonical is a human partner action (guardrail #4).
        if not require_admin(user):
            return self.send_error_json("Admin required", 403)
        if thesis_review is None:
            return self.send_error_json("Thesis module unavailable", 503)
        body = body or {}
        res = thesis_review.record_review(DB_PATH, version_id, user['user_id'],
                                          body.get('decision'), body.get('feedback'),
                                          body.get('target_node_id'))
        if res.get('error'):
            return self.send_error_json(res['error'], 400)
        return self.send_json({"data": res})

    def handle_list_users(self, user):
        # The full user directory (names, emails, roles) is admin-only — it is only
        # consumed by the admin section of Settings. The nav already hides it from
        # members; this enforces the same boundary server-side.
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)
        conn = get_db()
        users = rows_to_list(conn.execute(
            "SELECT id, username, email, full_name, role, is_active, created_at FROM users ORDER BY full_name"
        ).fetchall())
        conn.close()
        return self.send_json({"data": users})

    def handle_admin_create_user(self, user, body):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)

        required = ['username', 'password', 'email', 'full_name']
        for field in required:
            if not str(body.get(field, '')).strip():
                return self.send_error_json(f"{field} is required")
        if len(str(body.get('password') or '').strip()) < 8:
            return self.send_error_json("password must be at least 8 characters")

        role = body.get('role', 'member')
        # 'bot' is a deliberately-provisioned agent service account (kept out of the invite UI's
        # member/admin dropdown) — authenticated but never an admin. See require_bot_or_admin.
        if role not in ('admin', 'member', 'bot'):
            return self.send_error_json("role must be admin, member, or bot")

        conn = get_db()
        existing = conn.execute(
            "SELECT id FROM users WHERE username = ? OR email = ?",
            (body['username'].strip(), body['email'].strip())
        ).fetchone()
        if existing:
            conn.close()
            return self.send_error_json("Username or email already exists")

        user_id = generate_id()
        conn.execute(
            "INSERT INTO users (id, username, email, password_hash, full_name, role) VALUES (?, ?, ?, ?, ?, ?)",
            (
                user_id,
                body['username'].strip(),
                body['email'].strip(),
                hash_password(body['password']),
                body['full_name'].strip(),
                role
            )
        )
        log_audit(conn, user['user_id'], 'user', user_id, 'create', {"username": body['username'].strip(), "role": role})
        conn.commit()
        created = row_to_dict(conn.execute(
            "SELECT id, username, email, full_name, role, is_active, created_at FROM users WHERE id = ?",
            (user_id,)
        ).fetchone())
        conn.close()
        return self.send_json({"data": created}, 201)

    def handle_admin_update_user(self, user, target_user_id, body):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)

        conn = get_db()
        existing = conn.execute(
            "SELECT id, username, full_name, role, is_active FROM users WHERE id = ?",
            (target_user_id,)
        ).fetchone()
        if not existing:
            conn.close()
            return self.send_error_json("User not found", 404)

        sets = []
        args = []

        if 'is_active' in body:
            next_active = 1 if bool(body.get('is_active')) else 0
            # prevent locking out the currently authenticated admin accidentally
            if existing['id'] == user.get('user_id') and next_active == 0:
                conn.close()
                return self.send_error_json("You cannot deactivate your own account", 400)
            sets.append("is_active = ?")
            args.append(next_active)

        if 'role' in body:
            role = str(body.get('role'))
            if role not in ('admin', 'member', 'bot'):
                conn.close()
                return self.send_error_json("role must be admin, member, or bot")
            sets.append("role = ?")
            args.append(role)

        if 'password' in body:
            password = str(body.get('password') or '')
            if len(password.strip()) < 8:
                conn.close()
                return self.send_error_json("password must be at least 8 characters")
            sets.append("password_hash = ?")
            args.append(hash_password(password))

        if not sets:
            conn.close()
            return self.send_error_json("No fields to update")

        sets.append("updated_at = ?")
        args.append(now())
        args.append(target_user_id)
        conn.execute(f"UPDATE users SET {', '.join(sets)} WHERE id = ?", args)
        audit_payload = dict(body)
        if 'password' in audit_payload:
            audit_payload['password'] = '[REDACTED]'
        log_audit(conn, user['user_id'], 'user', target_user_id, 'update', audit_payload)
        conn.commit()
        updated = row_to_dict(conn.execute(
            "SELECT id, username, email, full_name, role, is_active, created_at FROM users WHERE id = ?",
            (target_user_id,)
        ).fetchone())
        conn.close()
        return self.send_json({"data": updated})

    def handle_admin_reset_all_data(self, user, body):
        if not require_admin(user):
            return self.send_error_json("Admin only", 403)

        confirm_phrase = str(body.get('confirm_phrase') or '').strip()
        if confirm_phrase != 'RESET ALL DATA':
            return self.send_error_json("Confirmation phrase must be exactly: RESET ALL DATA", 400)

        conn = get_db()
        try:
            ensure_fundraising_state_row(conn)
            state = conn.execute("SELECT * FROM fundraising_state WHERE id = 'main'").fetchone()
            pre_backup = create_fundraising_backup_file(state, kind="pre_restore") if state else None

            conn.execute("DELETE FROM communications")
            conn.execute("DELETE FROM opportunities")
            conn.execute("DELETE FROM custom_field_values")
            conn.execute("DELETE FROM custom_fields")
            conn.execute("DELETE FROM contacts")
            conn.execute("DELETE FROM organizations")

            default_grid = {
                "columns": deep_copy_json(DEFAULT_FUNDRAISING_COLUMNS),
                "rows": deep_copy_json(DEFAULT_FUNDRAISING_ROWS)
            }
            default_views = sanitize_grid_views(deep_copy_json(DEFAULT_GRID_VIEWS))
            conn.execute("""
                UPDATE fundraising_state
                SET grid_json = ?, views_json = ?, version = COALESCE(version, 1) + 1, updated_by = ?, updated_at = ?
                WHERE id = 'main'
            """, (json.dumps(default_grid), json.dumps(default_views), user['user_id'], now()))
            sync_fundraising_relational(conn, default_grid, default_views, actor_user_id=user['user_id'])

            log_audit(conn, user['user_id'], 'system', 'all-data', 'reset', {
                "pre_backup": pre_backup['filename'] if pre_backup else None
            })
            conn.commit()
        except Exception as exc:
            conn.rollback()
            conn.close()
            return self.send_error_json(f"Failed to reset data: {str(exc)}", 500)

        conn.close()
        return self.send_json({
            "data": {
                "status": "ok",
                "pre_backup": pre_backup
            }
        })

    def handle_admin_send_test_email(self, user, body):
        """Send a test email through the active transport (Gmail DWD preferred,
        SMTP fallback) to prove the outbound pipe before the daily digest (Phase B)."""
        if not require_admin(user):
            return self.send_error_json("Admin only", 403)

        import digest_mailer

        conn = get_db()
        try:
            # Recipients are restricted to the active-admin set (the real digest
            # audience). An explicit `to` may NARROW to specific admins but can never
            # introduce an outside address — this endpoint is not an open mail relay.
            rows = conn.execute(
                "SELECT email FROM users WHERE role = 'admin' AND is_active = 1 "
                "AND email IS NOT NULL AND TRIM(email) != ''"
            ).fetchall()
            admin_emails = [str(r['email']).strip() for r in rows if str(r['email']).strip()]
            admin_lower = {e.lower() for e in admin_emails}

            to = body.get('to')
            if to:
                requested = [to] if isinstance(to, str) else list(to)
                requested = [str(e).strip() for e in requested if str(e).strip()]
                outside = [e for e in requested if e.lower() not in admin_lower]
                if outside:
                    return self.send_error_json(
                        "Test email may only go to an active admin address; "
                        f"not allowed: {', '.join(outside)}", 400)
                recipients = requested
            else:
                recipients = admin_emails
            if not recipients:
                return self.send_error_json(
                    "No recipient: give an active admin an email address first.", 400)

            subject = "Ten31 CRM — test digest email"
            email_body = (
                "This is a test message from the Ten31 CRM daily-digest mailer.\n\n"
                f"Triggered by {user.get('full_name') or user.get('user_id')} at {now()}.\n\n"
                "If you received this, outbound email works and the daily digest can "
                "be delivered to this address."
            )
            result = digest_mailer.send_digest(conn, recipients, subject, email_body)
        except digest_mailer.NoTransport as exc:
            return self.send_error_json(str(exc), 400)
        except Exception as exc:
            # Never echo the exception to the client — an auth error can carry a
            # credential or token. Log it server-side instead.
            print(f"[digest] test send failed: {type(exc).__name__}: {exc}", file=sys.stderr)
            return self.send_error_json("Send failed — see server logs for details.", 502)
        finally:
            conn.close()

        return self.send_json({"data": {"status": "sent", **result}})

    def handle_admin_digest_preview(self, user, body):
        """Build the activity digest over a chosen window and return it WITHOUT
        sending — the admin-panel preview. Window defaults to the last 24h, or
        {hours: N} / {since: 'YYYY-MM-DD'} (a local date -> that day's midnight).
        Runs the REAL Spark summarization, so widening the window is how you verify
        the summarizer on a quiet day. Never touches the daily cursor."""
        if not require_admin(user):
            return self.send_error_json("Admin only", 403)
        body = body or {}
        import digest_builder
        try:
            since_iso, until_iso = digest_builder.resolve_digest_window(
                hours=body.get('hours'), since=body.get('since'))
        except ValueError as exc:
            return self.send_error_json(str(exc), 400)
        conn = get_db()
        try:
            digest = digest_builder.build_digest(conn, since_iso, until_iso)
        except Exception as exc:
            print(f"[digest] preview failed: {type(exc).__name__}: {exc}", file=sys.stderr)
            return self.send_error_json("Preview failed — see server logs for details.", 502)
        finally:
            conn.close()
        return self.send_json({"data": {**digest, "window": [since_iso, until_iso]}})

    def handle_admin_send_digest_now(self, user, body):
        """Build the REAL activity digest over a chosen window and send it to the
        active-admin set now. Window defaults to the last 24h, or {hours: N} /
        {since: 'YYYY-MM-DD'} — same resolution as the preview. Does NOT touch the
        daily schedule's cursor, so it never suppresses the scheduled send. Content
        is summarized on Spark (local), never Claude."""
        if not require_admin(user):
            return self.send_error_json("Admin only", 403)
        body = body or {}
        import digest_mailer
        import digest_builder
        try:
            since_iso, until_iso = digest_builder.resolve_digest_window(
                hours=body.get('hours'), since=body.get('since'))
        except ValueError as exc:
            return self.send_error_json(str(exc), 400)
        try:
            from email_integration.digest_scheduler import send_digest_window
            result = send_digest_window(since_iso=since_iso, until_iso=until_iso)
        except digest_mailer.NoTransport as exc:
            return self.send_error_json(str(exc), 400)
        except Exception as exc:
            # Never echo the exception — an auth error can carry a token/credential.
            print(f"[digest] send-now failed: {type(exc).__name__}: {exc}", file=sys.stderr)
            return self.send_error_json("Send failed — see server logs for details.", 502)

        return self.send_json({"data": result})

    def handle_get_digest_policy(self, user):
        """Return the live daily-digest policy (enabled + send hour). DB-backed
        (app_settings), set from this same panel — see digest_builder.load_digest_policy."""
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)
        import digest_builder
        conn = get_db()
        try:
            return self.send_json({"data": digest_builder.load_digest_policy(conn)})
        finally:
            conn.close()

    def handle_update_digest_policy(self, user, body):
        """Update the daily-digest policy. Takes effect on the scheduler's next
        cycle (no restart). Recipients stay the active-admin set; sender/transport
        are env/StartOS config, not toggled here."""
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)
        import digest_builder
        conn = get_db()
        try:
            policy = digest_builder.load_digest_policy(conn)
            if 'enabled' in body:
                policy['enabled'] = bool(body.get('enabled'))
            if 'send_hour' in body:
                try:
                    policy['send_hour'] = max(0, min(23, int(body.get('send_hour'))))
                except (ValueError, TypeError):
                    return self.send_error_json("send_hour must be an integer from 0 to 23")
            normalized = {"enabled": bool(policy['enabled']), "send_hour": int(policy['send_hour'])}
            set_app_setting(conn, digest_builder.DIGEST_POLICY_KEY, normalized)
            conn.commit()
            return self.send_json({"data": normalized})
        finally:
            conn.close()

    def handle_list_audit_log(self, user, params):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)
        conn = get_db()
        query = """
            SELECT al.*, u.full_name as user_name
            FROM audit_log al LEFT JOIN users u ON al.user_id = u.id
            WHERE 1=1
        """
        args = []
        if params.get('entity_type'):
            query += " AND al.entity_type = ?"
            args.append(params['entity_type'])
        if params.get('entity_id'):
            query += " AND al.entity_id = ?"
            args.append(params['entity_id'])
        if params.get('action'):
            query += " AND al.action = ?"
            args.append(params['action'])
        if params.get('user_id'):
            query += " AND al.user_id = ?"
            args.append(params['user_id'])
        if params.get('date_from'):
            query += " AND al.created_at >= ?"
            args.append(params['date_from'])
        if params.get('date_to'):
            query += " AND al.created_at <= ?"
            args.append(params['date_to'])
        if params.get('search'):
            search = f"%{str(params.get('search')).strip()}%"
            query += " AND (al.entity_id LIKE ? OR al.entity_type LIKE ? OR al.action LIKE ? OR al.changes LIKE ?)"
            args.extend([search, search, search, search])

        try:
            limit = max(1, min(500, int(params.get('limit') or 100)))
        except Exception:
            limit = 100
        query += " ORDER BY al.created_at DESC LIMIT ?"
        args.append(limit)
        logs = rows_to_list(conn.execute(query, args).fetchall())
        conn.close()
        return self.send_json({"data": logs})

    def handle_get_backup_policy(self, user):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)
        conn = get_db()
        policy = load_backup_policy(conn)
        conn.close()
        last_run_dt = parse_iso_utc(policy.get('last_run_at'))
        next_run_at = None
        if policy.get('enabled'):
            if last_run_dt:
                next_run_at = (last_run_dt.replace(tzinfo=None) + timedelta(hours=int(policy.get('interval_hours') or 24))).isoformat() + "Z"
            else:
                next_run_at = now()
        return self.send_json({"data": {**policy, "next_run_at": next_run_at}})

    def handle_update_backup_policy(self, user, body):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)
        conn = get_db()
        policy = load_backup_policy(conn)
        if 'enabled' in body:
            policy['enabled'] = bool(body.get('enabled'))
        if 'interval_hours' in body:
            try:
                policy['interval_hours'] = max(1, min(168, int(body.get('interval_hours'))))
            except Exception:
                conn.close()
                return self.send_error_json("interval_hours must be an integer from 1 to 168")
        if 'retention_days' in body:
            try:
                policy['retention_days'] = max(1, min(365, int(body.get('retention_days'))))
            except Exception:
                conn.close()
                return self.send_error_json("retention_days must be an integer from 1 to 365")
        if 'max_backups' in body:
            try:
                policy['max_backups'] = max(1, min(1000, int(body.get('max_backups'))))
            except Exception:
                conn.close()
                return self.send_error_json("max_backups must be an integer from 1 to 1000")
        updated = save_backup_policy(conn, policy)
        apply_backup_retention(updated)
        conn.commit()
        conn.close()
        return self.send_json({"data": updated})

    def handle_get_fundraising_relational_summary(self, user):
        conn = get_db()
        counts = {
            "investors": conn.execute("SELECT COUNT(*) AS c FROM fundraising_investors").fetchone()['c'],
            "contacts": conn.execute("SELECT COUNT(*) AS c FROM fundraising_contacts").fetchone()['c'],
            "funds": conn.execute("SELECT COUNT(*) AS c FROM fundraising_funds WHERE active = 1").fetchone()['c'],
            "commitments": conn.execute("SELECT COUNT(*) AS c FROM fundraising_commitments").fetchone()['c'],
            "views": conn.execute("SELECT COUNT(*) AS c FROM fundraising_views").fetchone()['c']
        }
        totals = conn.execute("""
            SELECT
                COALESCE(SUM(total_invested), 0) AS total_invested,
                SUM(CASE WHEN graveyard = 1 THEN 1 ELSE 0 END) AS graveyard_count,
                SUM(CASE WHEN follow_up = 1 THEN 1 ELSE 0 END) AS follow_up_count
            FROM fundraising_investors
        """).fetchone()
        top_funds = rows_to_list(conn.execute("""
            SELECT f.fund_name, f.column_id, COALESCE(SUM(c.amount), 0) AS total_commitment
            FROM fundraising_funds f
            LEFT JOIN fundraising_commitments c ON c.fund_id = f.id
            WHERE f.active = 1
            GROUP BY f.id
            ORDER BY total_commitment DESC
            LIMIT 10
        """).fetchall())
        conn.close()
        return self.send_json({
            "data": {
                "counts": counts,
                "totals": {
                    "total_invested": totals['total_invested'] if totals else 0,
                    "graveyard_count": totals['graveyard_count'] if totals and totals['graveyard_count'] is not None else 0,
                    "follow_up_count": totals['follow_up_count'] if totals and totals['follow_up_count'] is not None else 0
                },
                "top_funds": top_funds
            }
        })

    def handle_list_fundraising_automations(self, user):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)
        conn = get_db()
        ensure_default_automation_rules(conn)
        rules = rows_to_list(conn.execute("""
            SELECT * FROM fundraising_automation_rules
            ORDER BY name ASC
        """).fetchall())
        conn.commit()
        conn.close()
        for r in rules:
            try:
                r['condition_json'] = json.loads(r.get('condition_json') or '{}')
            except Exception:
                r['condition_json'] = {}
            try:
                r['action_json'] = json.loads(r.get('action_json') or '{}')
            except Exception:
                r['action_json'] = {}
        return self.send_json({"data": rules, "total": len(rules)})

    def handle_update_fundraising_automation_rule(self, user, rule_id, body):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)
        conn = get_db()
        existing = conn.execute("SELECT * FROM fundraising_automation_rules WHERE id = ?", (rule_id,)).fetchone()
        if not existing:
            conn.close()
            return self.send_error_json("Automation rule not found", 404)
        sets = []
        args = []
        if 'enabled' in body:
            sets.append("enabled = ?")
            args.append(1 if _to_bool(body.get('enabled')) else 0)
        if 'name' in body:
            sets.append("name = ?")
            args.append(str(body.get('name') or '').strip() or existing['name'])
        if 'condition_json' in body:
            sets.append("condition_json = ?")
            args.append(json.dumps(body.get('condition_json') if isinstance(body.get('condition_json'), dict) else {}))
        if 'action_json' in body:
            sets.append("action_json = ?")
            args.append(json.dumps(body.get('action_json') if isinstance(body.get('action_json'), dict) else {}))
        if not sets:
            conn.close()
            return self.send_error_json("No fields to update")
        sets.append("updated_at = ?")
        args.append(now())
        args.append(rule_id)
        conn.execute(f"UPDATE fundraising_automation_rules SET {', '.join(sets)} WHERE id = ?", args)
        run_fundraising_automations(conn)
        log_audit(conn, user['user_id'], 'fundraising_automation_rule', rule_id, 'update', body)
        conn.commit()
        updated = row_to_dict(conn.execute("SELECT * FROM fundraising_automation_rules WHERE id = ?", (rule_id,)).fetchone())
        conn.close()
        try:
            updated['condition_json'] = json.loads(updated.get('condition_json') or '{}')
        except Exception:
            updated['condition_json'] = {}
        try:
            updated['action_json'] = json.loads(updated.get('action_json') or '{}')
        except Exception:
            updated['action_json'] = {}
        return self.send_json({"data": updated})

    def handle_list_fundraising_automation_runs(self, user, params):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)
        limit = 100
        try:
            limit = max(1, min(500, int(params.get('limit') or 100)))
        except Exception:
            limit = 100
        conn = get_db()
        rows = rows_to_list(conn.execute("""
            SELECT r.*, i.investor_name, ar.name as rule_name
            FROM fundraising_automation_runs r
            LEFT JOIN fundraising_investors i ON r.investor_id = i.id
            LEFT JOIN fundraising_automation_rules ar ON r.rule_id = ar.id
            ORDER BY r.created_at DESC
            LIMIT ?
        """, (limit,)).fetchall())
        conn.close()
        for row in rows:
            try:
                row['result_json'] = json.loads(row.get('result_json') or '{}')
            except Exception:
                row['result_json'] = {}
        return self.send_json({"data": rows, "total": len(rows)})

    def handle_verify_fundraising_backups(self, user):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)
        checked = 0
        valid = 0
        invalid = []
        for b in list_backups():
            checked += 1
            try:
                with open(b['path'], 'r', encoding='utf-8') as f:
                    payload = json.load(f)
                if not isinstance(payload, dict):
                    raise ValueError("payload is not an object")
                if not isinstance(payload.get('grid'), dict):
                    raise ValueError("missing grid")
                if not isinstance(payload.get('views'), list):
                    raise ValueError("missing views")
                if not isinstance(payload['grid'].get('columns'), list) or not isinstance(payload['grid'].get('rows'), list):
                    raise ValueError("grid missing columns/rows")
                valid += 1
            except Exception as exc:
                invalid.append({"filename": b['filename'], "error": str(exc)})
        return self.send_json({
            "data": {
                "checked": checked,
                "valid": valid,
                "invalid": invalid,
                "invalid_count": len(invalid)
            }
        })

    def handle_get_fundraising_activity(self, user, params):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)
        limit = 120
        try:
            limit = max(20, min(500, int(params.get('limit') or 120)))
        except Exception:
            limit = 120
        conn = get_db()
        audit_rows = rows_to_list(conn.execute("""
            SELECT al.id, al.created_at, 'audit' AS source, al.entity_type, al.entity_id, al.action, u.full_name as actor_name
            FROM audit_log al
            LEFT JOIN users u ON al.user_id = u.id
            ORDER BY al.created_at DESC
            LIMIT ?
        """, (limit,)).fetchall())
        auto_rows = rows_to_list(conn.execute("""
            SELECT r.id, r.created_at, 'automation' AS source, 'fundraising_automation' AS entity_type, COALESCE(i.source_row_id, r.investor_id) AS entity_id,
                   r.status AS action, ar.name AS actor_name
            FROM fundraising_automation_runs r
            LEFT JOIN fundraising_automation_rules ar ON r.rule_id = ar.id
            LEFT JOIN fundraising_investors i ON r.investor_id = i.id
            ORDER BY r.created_at DESC
            LIMIT ?
        """, (limit,)).fetchall())
        conn.close()
        backup_rows = []
        for b in list_backups()[:limit]:
            backup_rows.append({
                "id": f"backup:{b['filename']}",
                "created_at": b['modified_at'],
                "source": "backup",
                "entity_type": "fundraising_backup",
                "entity_id": b['filename'],
                "action": b['kind'],
                "actor_name": "system"
            })
        combined = audit_rows + auto_rows + backup_rows
        combined.sort(key=lambda x: str(x.get('created_at') or ''), reverse=True)
        return self.send_json({"data": combined[:limit], "total": len(combined[:limit])})

    def handle_security_status(self, user):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)
        has_custom_secret = bool(os.environ.get("CRM_SECRET_KEY"))
        return self.send_json({
            "data": {
                "env": ENV,
                "auth_required": True,
                "cors_origin": CORS_ORIGIN,
                "has_custom_secret": has_custom_secret,
                "warnings": [] if has_custom_secret else ["CRM_SECRET_KEY is not explicitly set"],
                "rate_limits": {
                    "login_per_min": LOGIN_RATE_LIMIT_PER_MIN,
                    "write_per_min": WRITE_RATE_LIMIT_PER_MIN
                }
            }
        })

    # ═══════════════════════════════════════════════════════════════════════════
    # FUNDRAISING STATE (AIRTABLE-LIKE GRID)
    # ═══════════════════════════════════════════════════════════════════════════

    def _ensure_fundraising_state_row(self, conn):
        ensure_fundraising_state_row(conn)

    def handle_get_fundraising_state(self, user):
        conn = get_db()
        self._ensure_fundraising_state_row(conn)
        row = conn.execute("SELECT * FROM fundraising_state WHERE id = 'main'").fetchone()
        stage_by_row = pipeline_stage_by_source_row(conn)
        opp_id_by_row = opportunity_id_by_source_row(conn)
        reminder_by_row = reminder_status_by_source_row(conn)
        existing_by_row = existing_investor_by_source_row(conn)
        recency_by_row = staleness_by_source_row(conn)
        emails_by_row = fundraising_contact_emails_by_row(conn)
        conn.close()

        try:
            grid = json.loads(row['grid_json']) if row and row['grid_json'] else {}
        except json.JSONDecodeError:
            grid = {}
        try:
            views = json.loads(row['views_json']) if row and row['views_json'] else []
        except json.JSONDecodeError:
            views = []
        views = sanitize_grid_views(views)

        grid = sanitize_fundraising_grid(grid)
        if not isinstance(views, list):
            views = []
        columns = grid.get('columns', [])
        rows = grid.get('rows', [])

        # Inject the read-only pipeline columns, derived from the live linked opportunity
        # (the opportunities table is canonical — never stored in the grid blob, so it
        # can't go stale). The frontend renders these read-only and strips them on save.
        for r in rows:
            if not isinstance(r, dict):
                continue
            stage = stage_by_row.get(str(r.get('id') or ''))
            r['pipeline'] = bool(stage)
            r['pipeline_stage'] = stage or ''
            # Live opportunity id for a linked row (read-only) — lets the mobile detail PATCH the
            # stage on the opportunities endpoint; '' when the row isn't in the pipeline.
            r['opportunity_id'] = opp_id_by_row.get(str(r.get('id') or ''), '')
            # Read-only reminder status, derived live from the reminders table (never stored
            # in the blob). '' = no open reminder; a saved view can filter on this column to
            # supersede the binary follow_up checkbox.
            r['reminder_status'] = reminder_by_row.get(str(r.get('id') or ''), '')
            # Auto-derived "Existing Investor" flag (total_invested > 0) + last-contact recency
            # and its staleness ramp ('' / 'aging' / 'stale'). All read-only, computed fresh on
            # read like the columns above (stripped on write), so the desktop grid and the mobile
            # card render the star + the grey/amber/red recency off one server signal.
            srid = str(r.get('id') or '')
            r['existing_investor'] = srid in existing_by_row
            last_activity, staleness = recency_by_row.get(srid, (None, ''))
            r['last_activity_at'] = last_activity
            r['staleness'] = staleness
            # Heal blank pill emails from the relational mirror (fill-only — never overwrite a value
            # already in the blob). Unlike the read-only columns above, email is a REAL blob field,
            # so this is a backfill, not a derived signal: it needs NO strip point, and the next
            # one-row save legitimately persists it. Match by pill order, then by name.
            heal = emails_by_row.get(srid)
            pills = r.get('contacts')
            if heal and isinstance(pills, list):
                for i, c in enumerate(pills):
                    if not isinstance(c, dict) or str(c.get('email') or '').strip():
                        continue
                    found = heal['order'].get(i) or heal['name'].get(_normalize_text(c.get('name')))
                    if found:
                        c['email'] = found

        return self.send_json({
            "data": {
                "grid": {"columns": columns, "rows": rows},
                "views": views if views else deep_copy_json(DEFAULT_GRID_VIEWS),
                "version": row['version'] if row else 1,
                "updated_at": row['updated_at'] if row else None
            }
        })

    def _cleanup_fundraising_collab(self, conn):
        now_epoch = int(time.time())
        conn.execute("DELETE FROM fundraising_presence WHERE expires_at_epoch <= ?", (now_epoch,))
        conn.execute("DELETE FROM fundraising_cell_locks WHERE expires_at_epoch <= ?", (now_epoch,))

    def _list_fundraising_collab_state(self, conn):
        presence_rows = rows_to_list(conn.execute("""
            SELECT user_id, username, full_name, active_view, row_id, col_id, is_editing, cell_key, last_seen_at
            FROM fundraising_presence
            ORDER BY last_seen_at DESC
        """).fetchall())
        lock_rows = rows_to_list(conn.execute("""
            SELECT cell_key, row_id, col_id, locked_by_user_id, locked_by_username, locked_by_full_name, last_seen_at
            FROM fundraising_cell_locks
            ORDER BY last_seen_at DESC
        """).fetchall())
        for row in presence_rows:
            row['is_editing'] = bool(row.get('is_editing'))
        return {"presence": presence_rows, "locks": lock_rows}

    def handle_get_fundraising_collab_state(self, user):
        conn = get_db()
        self._cleanup_fundraising_collab(conn)
        snapshot = self._list_fundraising_collab_state(conn)
        conn.commit()
        conn.close()
        return self.send_json({"data": snapshot})

    def handle_fundraising_collab_heartbeat(self, user, body):
        active_view = str(body.get('active_view') or '').strip()
        selected = body.get('selected') if isinstance(body.get('selected'), dict) else {}
        editing = body.get('editing') if isinstance(body.get('editing'), dict) else {}
        selected_row_id = str(selected.get('row_id') or '').strip()
        selected_col_id = str(selected.get('col_id') or '').strip()
        editing_row_id = str(editing.get('row_id') or '').strip()
        editing_col_id = str(editing.get('col_id') or '').strip()
        is_editing = bool(editing_row_id and editing_col_id)
        ttl_seconds = int(body.get('ttl_seconds') or 25)
        ttl_seconds = max(10, min(120, ttl_seconds))
        now_epoch = int(time.time())
        expires_at_epoch = now_epoch + ttl_seconds
        seen_at = now()
        lock_conflict = None

        conn = get_db()
        self._cleanup_fundraising_collab(conn)

        user_row = conn.execute("SELECT username, full_name FROM users WHERE id = ?", (user['user_id'],)).fetchone()
        username = str(user_row['username']) if user_row and user_row['username'] else str(user.get('username') or '')
        full_name = str(user_row['full_name']) if user_row and user_row['full_name'] else ''
        editing_cell_key = f"{editing_row_id}:{editing_col_id}" if is_editing else None

        if is_editing and editing_cell_key:
            existing_lock = conn.execute("""
                SELECT cell_key, row_id, col_id, locked_by_user_id, locked_by_username, locked_by_full_name, last_seen_at
                FROM fundraising_cell_locks
                WHERE cell_key = ? AND locked_by_user_id != ? AND expires_at_epoch > ?
                LIMIT 1
            """, (editing_cell_key, user['user_id'], now_epoch)).fetchone()
            if existing_lock:
                lock_conflict = row_to_dict(existing_lock)
                is_editing = False
                editing_cell_key = None
            else:
                conn.execute("""
                    INSERT INTO fundraising_cell_locks (
                        cell_key, row_id, col_id, locked_by_user_id, locked_by_username, locked_by_full_name, last_seen_at, expires_at_epoch
                    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
                    ON CONFLICT(cell_key) DO UPDATE SET
                        row_id = excluded.row_id,
                        col_id = excluded.col_id,
                        locked_by_user_id = excluded.locked_by_user_id,
                        locked_by_username = excluded.locked_by_username,
                        locked_by_full_name = excluded.locked_by_full_name,
                        last_seen_at = excluded.last_seen_at,
                        expires_at_epoch = excluded.expires_at_epoch
                """, (editing_cell_key, editing_row_id, editing_col_id, user['user_id'], username, full_name, seen_at, expires_at_epoch))
                conn.execute("""
                    DELETE FROM fundraising_cell_locks
                    WHERE locked_by_user_id = ? AND cell_key != ?
                """, (user['user_id'], editing_cell_key))
        else:
            conn.execute("DELETE FROM fundraising_cell_locks WHERE locked_by_user_id = ?", (user['user_id'],))

        conn.execute("""
            INSERT INTO fundraising_presence (
                user_id, username, full_name, active_view, row_id, col_id, is_editing, cell_key, last_seen_at, expires_at_epoch
            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
            ON CONFLICT(user_id) DO UPDATE SET
                username = excluded.username,
                full_name = excluded.full_name,
                active_view = excluded.active_view,
                row_id = excluded.row_id,
                col_id = excluded.col_id,
                is_editing = excluded.is_editing,
                cell_key = excluded.cell_key,
                last_seen_at = excluded.last_seen_at,
                expires_at_epoch = excluded.expires_at_epoch
        """, (
            user['user_id'],
            username,
            full_name,
            active_view,
            selected_row_id or editing_row_id,
            selected_col_id or editing_col_id,
            1 if is_editing else 0,
            editing_cell_key,
            seen_at,
            expires_at_epoch
        ))

        snapshot = self._list_fundraising_collab_state(conn)
        conn.commit()
        conn.close()

        return self.send_json({
            "data": {
                **snapshot,
                "lock_conflict": lock_conflict
            }
        })

    def handle_update_fundraising_state(self, user, body):
        grid = body.get('grid', {})
        views = body.get('views')
        expected_version = body.get('expected_version')

        if not isinstance(grid, dict):
            return self.send_error_json("grid must be an object")
        if 'columns' not in grid or 'rows' not in grid:
            return self.send_error_json("grid must include columns and rows")
        if not isinstance(grid.get('columns'), list) or not isinstance(grid.get('rows'), list):
            return self.send_error_json("grid.columns and grid.rows must be arrays")
        grid = sanitize_fundraising_grid(grid)
        if views is not None and not isinstance(views, list):
            return self.send_error_json("views must be an array when provided")

        conn = get_db()
        self._ensure_fundraising_state_row(conn)

        current = conn.execute("SELECT version FROM fundraising_state WHERE id = 'main'").fetchone()
        current_version = int(current['version']) if current else 1
        if expected_version is not None and int(expected_version) != current_version:
            snapshot = conn.execute("SELECT version, updated_at, updated_by FROM fundraising_state WHERE id = 'main'").fetchone()
            conn.close()
            return self.send_json({
                "error": "Version conflict",
                "current_version": current_version,
                "current_updated_at": snapshot['updated_at'] if snapshot else None,
                "current_updated_by": snapshot['updated_by'] if snapshot else None
            }, status=409)

        row = conn.execute("SELECT views_json FROM fundraising_state WHERE id = 'main'").fetchone()
        existing_views = []
        if row and row['views_json']:
            try:
                existing_views = json.loads(row['views_json'])
            except json.JSONDecodeError:
                existing_views = deep_copy_json(DEFAULT_GRID_VIEWS)
        existing_views = sanitize_grid_views(existing_views)

        next_views = sanitize_grid_views(views if views is not None else existing_views)
        next_version = current_version + 1
        conn.execute("""
            UPDATE fundraising_state
            SET grid_json = ?, views_json = ?, version = ?, updated_by = ?, updated_at = ?
            WHERE id = 'main'
        """, (json.dumps(grid), json.dumps(next_views), next_version, user['user_id'], now()))
        sync_fundraising_relational(conn, grid, next_views, actor_user_id=user['user_id'])
        # Archive pipeline opps orphaned by an investor deleted from the grid (one-way
        # cleanup; never creates or reseeds — see reconcile_grid_pipeline_links).
        reconcile_grid_pipeline_links(conn)
        # Cancel reminders orphaned by the same investor deletion (one-way cleanup twin).
        reconcile_grid_reminders(conn)
        log_audit(conn, user['user_id'], 'fundraising_state', 'main', 'update', {"version": next_version})
        conn.commit()
        conn.close()

        return self.send_json({
            "data": {
                "version": next_version,
                "updated_at": now()
            }
        })

    def handle_export_fundraising_state(self, user):
        conn = get_db()
        self._ensure_fundraising_state_row(conn)
        row = conn.execute("SELECT * FROM fundraising_state WHERE id = 'main'").fetchone()
        conn.close()

        if not row:
            return self.send_error_json("Fundraising state not found", 404)

        try:
            grid = json.loads(row['grid_json']) if row['grid_json'] else {}
        except json.JSONDecodeError:
            grid = {}
        try:
            views = json.loads(row['views_json']) if row['views_json'] else []
        except json.JSONDecodeError:
            views = []

        payload = {
            "exported_at": now(),
            "version": row['version'],
            "updated_at": row['updated_at'],
            "grid": grid,
            "views": views
        }
        return self.send_json({"data": payload})

    def handle_list_fundraising_backups(self, user):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)
        entries = list_backups()
        return self.send_json({"data": entries, "total": len(entries)})

    def _extract_fundraising_restore_payload(self, body):
        payload = body.get('payload')
        if payload is None:
            payload = body.get('data')

        if payload is None and body.get('filename'):
            filename = os.path.basename(str(body.get('filename')))
            if not filename.endswith('.json'):
                return None, "filename must point to a .json backup file"
            backup_dir = os.path.join(DATA_DIR, "backups")
            candidate = os.path.join(backup_dir, filename)
            if not os.path.isfile(candidate):
                return None, "backup file not found"
            try:
                with open(candidate, 'r', encoding='utf-8') as f:
                    payload = json.load(f)
            except Exception:
                return None, "backup file is not valid JSON"

        if payload is None:
            payload = body

        if not isinstance(payload, dict):
            return None, "payload must be a JSON object"
        return payload, None

    def _normalize_fundraising_payload(self, payload):
        grid = payload.get('grid')
        views = payload.get('views')

        if not isinstance(grid, dict):
            return None, None, "payload.grid is required"
        if not isinstance(grid.get('columns'), list) or not isinstance(grid.get('rows'), list):
            return None, None, "payload.grid.columns and payload.grid.rows must be arrays"
        if views is None:
            views = deep_copy_json(DEFAULT_GRID_VIEWS)
        if not isinstance(views, list):
            return None, None, "payload.views must be an array"
        return sanitize_fundraising_grid(grid), sanitize_grid_views(views), None

    def handle_preview_fundraising_restore(self, user, body):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)

        payload, err = self._extract_fundraising_restore_payload(body)
        if err:
            return self.send_error_json(err)
        grid, views, err = self._normalize_fundraising_payload(payload)
        if err:
            return self.send_error_json(err)

        conn = get_db()
        self._ensure_fundraising_state_row(conn)
        current = conn.execute("SELECT * FROM fundraising_state WHERE id = 'main'").fetchone()
        conn.close()
        try:
            current_grid = json.loads(current['grid_json']) if current and current['grid_json'] else {}
        except Exception:
            current_grid = {}
        try:
            current_views = json.loads(current['views_json']) if current and current['views_json'] else []
        except Exception:
            current_views = []
        diff = compute_restore_diff(current_grid, current_views, grid, views)

        preview = {
            "columns_count": len(grid.get('columns', [])),
            "rows_count": len(grid.get('rows', [])),
            "views_count": len(views),
            "source": "filename" if body.get('filename') else "payload",
            "diff": diff
        }
        return self.send_json({"data": preview})

    def handle_backup_fundraising_state(self, user):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)

        conn = get_db()
        self._ensure_fundraising_state_row(conn)
        row = conn.execute("SELECT * FROM fundraising_state WHERE id = 'main'").fetchone()
        conn.close()
        if not row:
            return self.send_error_json("Fundraising state not found", 404)

        info = create_fundraising_backup_file(row, kind="backup")
        conn = get_db()
        policy = load_backup_policy(conn)
        apply_backup_retention(policy)
        conn.close()
        return self.send_json({"data": info}, 201)

    def handle_restore_fundraising_state(self, user, body):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)

        payload, err = self._extract_fundraising_restore_payload(body)
        if err:
            return self.send_error_json(err)
        grid, views, err = self._normalize_fundraising_payload(payload)
        if err:
            return self.send_error_json(err)

        conn = get_db()
        self._ensure_fundraising_state_row(conn)
        current = conn.execute("SELECT * FROM fundraising_state WHERE id = 'main'").fetchone()
        if not current:
            conn.close()
            return self.send_error_json("Fundraising state not found", 404)

        # Always create a rollback snapshot before restore
        pre_backup = create_fundraising_backup_file(current, kind="pre_restore")

        next_version = int(current['version']) + 1
        conn.execute("""
            UPDATE fundraising_state
            SET grid_json = ?, views_json = ?, version = ?, updated_by = ?, updated_at = ?
            WHERE id = 'main'
        """, (json.dumps(grid), json.dumps(views), next_version, user['user_id'], now()))
        sync_fundraising_relational(conn, grid, views, actor_user_id=user['user_id'])
        log_audit(conn, user['user_id'], 'fundraising_state', 'main', 'restore', {"version": next_version, "pre_backup": pre_backup['filename']})
        conn.commit()
        conn.close()

        return self.send_json({
            "data": {
                "version": next_version,
                "pre_restore_backup": {
                    "filename": pre_backup['filename'],
                    "path": pre_backup['path']
                }
            }
        }, 201)


# ─── Seed Data ────────────────────────────────────────────────────────────────

def seed_demo_data():
    """Create demo data for testing."""
    conn = get_db()

    # Check if already seeded
    if conn.execute("SELECT COUNT(*) as c FROM users").fetchone()['c'] > 0:
        conn.close()
        return

    print("Seeding demo data...")

    # Create admin user (password: admin123)
    admin_id = generate_id()
    conn.execute("""
        INSERT INTO users (id, username, email, password_hash, full_name, role)
        VALUES (?, 'admin', 'admin@fund.com', ?, 'Fund Admin', 'admin')
    """, (admin_id, hash_password('admin123')))

    # Create a second user
    user2_id = generate_id()
    conn.execute("""
        INSERT INTO users (id, username, email, password_hash, full_name, role)
        VALUES (?, 'grant', 'grant@ten31.xyz', ?, 'Grant', 'admin')
    """, (user2_id, hash_password('password')))

    # Create organizations
    orgs = [
        (generate_id(), "Sovereign Wealth Holdings", "institutional", "Sovereign Wealth", "https://example.com"),
        (generate_id(), "Pacific Capital Partners", "family_office", "Family Office", "https://example.com"),
        (generate_id(), "Northeast Pension Fund", "pension", "Pension Fund", "https://example.com"),
        (generate_id(), "Redwood Endowment", "endowment", "Endowment", "https://example.com"),
        (generate_id(), "Atlas Family Office", "family_office", "Family Office", "https://example.com"),
        (generate_id(), "Summit Insurance Group", "institutional", "Insurance", "https://example.com"),
        (generate_id(), "Cascade Wealth Management", "wealth_management", "Wealth Management", "https://example.com"),
        (generate_id(), "Blue Harbor Foundation", "foundation", "Foundation", "https://example.com"),
    ]
    for org in orgs:
        conn.execute("""
            INSERT INTO organizations (id, name, type, industry, website, created_by)
            VALUES (?, ?, ?, ?, ?, ?)
        """, (*org, admin_id))

    # Create contacts (mix of investors and prospects)
    contacts = [
        # Investors
        (generate_id(), "James", "Chen", "jchen@sovereign.com", "Managing Director", orgs[0][0], "investor"),
        (generate_id(), "Sarah", "Williams", "swilliams@pacificcap.com", "CIO", orgs[1][0], "investor"),
        (generate_id(), "Michael", "Davis", "mdavis@nepension.org", "Investment Director", orgs[2][0], "investor"),
        (generate_id(), "Elizabeth", "Thompson", "ethompson@redwood.edu", "Endowment Manager", orgs[3][0], "investor"),
        (generate_id(), "Robert", "Kim", "rkim@atlas.family", "Principal", orgs[4][0], "investor"),
        (generate_id(), "Patricia", "Anderson", "panderson@summit.com", "VP Investments", orgs[5][0], "investor"),
        # Prospects
        (generate_id(), "David", "Martinez", "dmartinez@cascade.com", "Partner", orgs[6][0], "prospect"),
        (generate_id(), "Jennifer", "Taylor", "jtaylor@blueharbor.org", "Executive Director", orgs[7][0], "prospect"),
        (generate_id(), "William", "Johnson", "wjohnson@gmail.com", "Independent Investor", None, "prospect"),
        (generate_id(), "Maria", "Garcia", "mgarcia@example.com", "Family Office Principal", None, "prospect"),
        (generate_id(), "Thomas", "Brown", "tbrown@example.com", "Wealth Manager", None, "prospect"),
        (generate_id(), "Linda", "Wilson", "lwilson@example.com", "Portfolio Manager", None, "prospect"),
    ]
    for c in contacts:
        conn.execute("""
            INSERT INTO contacts (id, first_name, last_name, email, title, organization_id, contact_type, created_by)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?)
        """, (*c, admin_id))

    # Create opportunities
    opp_data = [
        (contacts[6][0], orgs[6][0], "Cascade Wealth - Fund II", "engaged", 10000000, 10000000, 40, user2_id),
        (contacts[7][0], orgs[7][0], "Blue Harbor - Fund II", "diligence", 5000000, 5000000, 60, user2_id),
        (contacts[8][0], None, "William Johnson - Direct", "lead", 0, 2000000, 20, admin_id),
        (contacts[9][0], None, "Garcia Family Office - Fund II", "lead", 0, 15000000, 10, admin_id),
        (contacts[10][0], None, "Thomas Brown - WM Referral", "engaged", 0, 3000000, 30, user2_id),
        (contacts[11][0], None, "Linda Wilson - PM Intro", "lead", 0, 5000000, 15, admin_id),
    ]
    for opp in opp_data:
        conn.execute("""
            INSERT INTO opportunities (id, contact_id, organization_id, name, stage,
                commitment_amount, expected_amount, probability, owner_id, fund_name)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'Fund II')
        """, (generate_id(), *opp))

    # Create communications
    comm_data = [
        (contacts[0][0], "meeting", "Q4 Review Meeting", "Discussed fund performance and upcoming distributions.", "2025-12-15", 60, "positive", admin_id),
        (contacts[1][0], "email", "Capital Call Notice", "Sent Q1 capital call documentation.", "2026-01-10", None, "neutral", admin_id),
        (contacts[2][0], "call", "Annual Check-in", "Reviewed portfolio allocation and discussed Fund II.", "2026-01-20", 30, "positive", user2_id),
        (contacts[6][0], "meeting", "Initial Presentation", "Presented Fund II thesis and track record.", "2026-02-01", 90, "positive", user2_id),
        (contacts[7][0], "email", "Due Diligence Materials", "Sent DDQ, audited financials, and PPM.", "2026-02-05", None, "neutral", user2_id),
        (contacts[8][0], "call", "Intro Call", "Initial outreach, discussed investment interests.", "2026-02-08", 20, "positive", admin_id),
        (contacts[9][0], "email", "Introduction Email", "Sent overview deck and one-pager.", "2026-02-10", None, "neutral", admin_id),
        (contacts[6][0], "call", "Follow-up on Presentation", "Cascade team asked about risk management approach.", "2026-02-10", 25, "positive", user2_id),
    ]
    for cm in comm_data:
        conn.execute("""
            INSERT INTO communications (id, contact_id, type, subject, body,
                communication_date, duration_minutes, outcome, created_by)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
        """, (generate_id(), *cm))

    # Create some tags
    tag_data = [
        ("High Priority", "#ef4444"),
        ("Fund I LP", "#3b82f6"),
        ("Fund II Prospect", "#8b5cf6"),
        ("Family Office", "#f59e0b"),
        ("Institutional", "#10b981"),
        ("Re-up Target", "#ec4899"),
    ]
    for tag in tag_data:
        conn.execute("INSERT INTO tags (id, name, color) VALUES (?, ?, ?)",
                     (generate_id(), *tag))

    conn.commit()
    conn.close()
    print("Demo data seeded successfully.")


# ─── Email-activity summary agent ────────────────────────────────────────────
# When a sent/received email is matched to an investor, summarize it to ONE dated,
# marked note on the LOCAL model (sovereign — nothing leaves Ten31, so no redaction
# boundary is needed) and append it to that investor's notes in the fundraising grid.
# Going-forward only: we never summarize email dated before the feature was switched
# on, so the historical backfill does not generate noise.

_ACTIVITY_SINCE_KEY = "email_activity_since"
_ACTIVITY_MARKER = "✉"   # marks an email-derived note (kept editable before approval)


def _fmt_activity_date(sent_at):
    """ISO-ish timestamp -> 'Jun 25, 2026' (falls back to the raw date part)."""
    from datetime import datetime
    datepart = str(sent_at or "")[:10]   # YYYY-MM-DD
    try:
        return datetime.strptime(datepart, "%Y-%m-%d").strftime("%b %-d, %Y")
    except Exception:
        return datepart


def _activity_investor(conn, email_id):
    """Resolve (grid_row_id, investor_name) for a matched email via its highest-
    confidence investor link. Either may be None."""
    link = conn.execute(
        "SELECT fundraising_investor_id, organization_id, contact_id FROM email_investor_links "
        "WHERE email_id=? ORDER BY match_confidence DESC LIMIT 1", (email_id,)).fetchone()
    if not link:
        return None, None
    inv_id = link["fundraising_investor_id"]
    name = None
    if inv_id:
        r = conn.execute("SELECT investor_name FROM fundraising_investors WHERE id=?", (inv_id,)).fetchone()
        name = r["investor_name"] if r else None
    if not name and link["organization_id"]:
        r = conn.execute("SELECT name FROM organizations WHERE id=?", (link["organization_id"],)).fetchone()
        name = r["name"] if r else None
    if not name and link["contact_id"]:
        r = conn.execute("SELECT first_name, last_name FROM contacts WHERE id=?", (link["contact_id"],)).fetchone()
        if r:
            name = f"{r['first_name'] or ''} {r['last_name'] or ''}".strip()
    return inv_id, name


def _summarize_email_gist(subject, body):
    """One short clause describing the email's substance, from the LOCAL model."""
    try:
        sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "ingest"))
        import llm  # noqa: E402
    except Exception:
        return None
    text = (body or "")[:4000]
    if not (subject or text).strip():
        return None
    out = llm.chat(
        f"Subject: {subject or '(none)'}\n\n{text}",
        system=("You summarize an email into a brief CRM note. Reply with ONE clause under 14 words "
                "describing what the email is about. No greeting, no names, no quotes, no trailing period."),
        max_tokens=40, temperature=0.0)
    gist = " ".join((out or "").split()).strip().rstrip(".")
    return gist or None


def _append_grid_note(conn, inv_id, inv_name, note, updated_by=None):
    """Append a note to the matched investor's notes cell in the live grid (newest at
    bottom), bump the grid version, and refresh the relational projection. Best-effort."""
    row = conn.execute("SELECT grid_json, views_json, version FROM fundraising_state WHERE id='main'").fetchone()
    if not row or not row["grid_json"]:
        return False
    try:
        grid = json.loads(row["grid_json"])
    except Exception:
        return False
    rows = grid.get("rows", []) if isinstance(grid, dict) else []
    target = None
    if inv_id:
        target = next((r for r in rows if isinstance(r, dict) and str(r.get("id")) == str(inv_id)), None)
    if target is None and inv_name:
        nn = _normalize_text(inv_name)
        target = next((r for r in rows if isinstance(r, dict)
                       and _normalize_text(str(r.get("investor_name") or "")) == nn), None)
    if target is None:
        return False
    existing = str(target.get("notes") or "").rstrip()
    target["notes"] = (existing + "\n" + note) if existing else note
    try:
        views = json.loads(row["views_json"]) if row["views_json"] else []
    except Exception:
        views = []
    # updated_by has a FK to users(id); stamp the approving user (None -> NULL is fine).
    conn.execute("UPDATE fundraising_state SET grid_json=?, version=?, updated_by=?, updated_at=? WHERE id='main'",
                 (json.dumps(grid), (row["version"] or 0) + 1, updated_by, now()))
    try:
        sync_fundraising_relational(conn, grid, views, actor_user_id=updated_by)
    except Exception:
        pass
    return True


# Public mailbox providers: a teammate's enrolled gmail must NOT make every gmail sender read
# as "ours" (that would flip inbound LP gmail to outbound). Only corporate domains imply us.
_PUBLIC_EMAIL_DOMAINS = {
    "gmail.com", "googlemail.com", "outlook.com", "hotmail.com", "live.com", "msn.com",
    "yahoo.com", "ymail.com", "icloud.com", "me.com", "mac.com", "aol.com",
    "proton.me", "protonmail.com", "pm.me",
}


def _our_email_domains(own_addresses):
    """Corporate domains we send from, derived from the enrolled mailboxes (public providers
    excluded). Lets an @ten31.xyz sender read as outbound even when that mailbox isn't enrolled —
    the misclassification behind a teammate's email showing as 'Received'."""
    out = set()
    for a in own_addresses:
        if "@" in a:
            dom = a.split("@", 1)[1]
            if dom and dom not in _PUBLIC_EMAIL_DOMAINS:
                out.add(dom)
    return out


def _sender_display(from_name, from_email):
    """A human name for the sender: their display name, else the email's local part, else a stub."""
    if from_name and from_name.strip():
        return from_name.strip()
    if from_email and "@" in from_email:
        return from_email.split("@", 1)[0]
    return (from_email or "").strip() or "Someone"


def _activity_note_text(sent_at, outbound, gist, sender, investor_name):
    """Name WHO emailed WHOM rather than a bare Sent/Received (which left 'who received?'
    ambiguous): outbound -> "{teammate} emailed {investor}", inbound -> "{sender} emailed the team"."""
    if outbound:
        rel = f"{sender} emailed {investor_name}" if investor_name else f"{sender} sent an email"
    else:
        rel = f"{sender} emailed the team"
    return f"{_ACTIVITY_MARKER} {_fmt_activity_date(sent_at)} — {rel}: {gist}"


def propose_email_activity_notes(limit=50):
    """Draft a PROPOSED grid note per newly-matched email and queue it for human
    review (status 'pending'). Does NOT touch the grid — approval does that. Idempotent
    (one proposal per email), going-forward only. Safe to call after each Gmail sync."""
    conn = get_db()
    try:
        try:
            conn.execute("SELECT 1 FROM email_activity_proposals LIMIT 1")
        except sqlite3.OperationalError:
            return {"proposed": 0, "skipped": "tables_absent"}
        since = get_app_setting(conn, _ACTIVITY_SINCE_KEY)
        if not since:
            since = now()
            set_app_setting(conn, _ACTIVITY_SINCE_KEY, since)
            conn.commit()
        try:
            rows = conn.execute(
                "SELECT id, subject, body_text, snippet, from_name, from_email, sent_at FROM emails "
                "WHERE is_matched=1 AND sent_at >= ? "
                "AND id NOT IN (SELECT email_id FROM email_activity_proposals) "
                "ORDER BY sent_at ASC LIMIT ?", (since, limit)).fetchall()
        except sqlite3.OperationalError:
            return {"proposed": 0, "skipped": "emails_absent"}
        if not rows:
            return {"proposed": 0}
        own = set()
        try:
            own = {(r[0] or "").lower() for r in conn.execute("SELECT email_address FROM email_accounts")}
        except Exception:
            pass
        our_domains = _our_email_domains(own)
        done = 0
        for r in rows:
            inv_id, inv_name = _activity_investor(conn, r["id"])
            from_email = (r["from_email"] or "").lower()
            # Outbound = sent by us: an enrolled mailbox, OR any address on our corporate domain
            # (covers teammates whose mailbox isn't enrolled — the screenshot's "Received" bug).
            outbound = from_email in own or (from_email.rsplit("@", 1)[-1] in our_domains if "@" in from_email else False)
            direction = "Sent" if outbound else "Received"
            gist = _summarize_email_gist(r["subject"], r["body_text"] or r["snippet"] or "")
            if not gist:
                continue  # leave unproposed; a later pass retries once the model answers
            note = _activity_note_text(r["sent_at"], outbound, gist, _sender_display(r["from_name"], r["from_email"]), inv_name)
            conn.execute(
                "INSERT OR IGNORE INTO email_activity_proposals "
                "(id,email_id,investor_id,investor_name,direction,summary,proposed_note,"
                " email_subject,email_date,status,created_at) "
                "VALUES (?,?,?,?,?,?,?,?,?,'pending',?)",
                (generate_id(), r["id"], inv_id, inv_name, direction.lower(), gist, note,
                 r["subject"], r["sent_at"], now()))
            conn.commit()
            done += 1
        return {"proposed": done}
    finally:
        conn.close()


def list_email_activity_proposals(conn, status="pending", limit=200):
    try:
        rows = conn.execute(
            "SELECT id, email_id, investor_id, investor_name, direction, summary, proposed_note, "
            "email_subject, email_date, status, created_at FROM email_activity_proposals "
            "WHERE status=? ORDER BY email_date ASC, created_at ASC LIMIT ?", (status, limit)).fetchall()
        return [dict(r) for r in rows]
    except sqlite3.OperationalError:
        return []


def decide_email_activity_proposal(conn, proposal_id, decision, user_id, edited_note=None,
                                   source="crm_ui", close_matrix=False):
    """Approve (optionally with an edited note -> append to grid) or dismiss a proposal.

    `source` records the channel in the audit log ('crm_ui' for the web panel, 'matrix' for an
    in-thread approval relayed by the review bot). `close_matrix` is set by the bot when the
    decision was made in-thread: it stamps the Matrix side row closed in the same transaction so
    the web->Matrix close path (list_bot_email_proposals.to_close) won't re-announce it."""
    p = conn.execute("SELECT * FROM email_activity_proposals WHERE id=?", (proposal_id,)).fetchone()
    if not p:
        return {"error": "not_found"}
    if p["status"] != "pending":
        return {"error": "already_decided", "status": p["status"]}
    if decision == "approve":
        note = (edited_note or "").strip() or p["proposed_note"]
        placed = _append_grid_note(conn, p["investor_id"], p["investor_name"], note, updated_by=user_id)
        conn.execute("UPDATE email_activity_proposals SET status='approved', final_note=?, decided_by=?, decided_at=? WHERE id=?",
                     (note, user_id, now(), proposal_id))
        action, result = "email.activity_approved", {"status": "approved", "placed_in_grid": placed}
    elif decision == "dismiss":
        conn.execute("UPDATE email_activity_proposals SET status='dismissed', decided_by=?, decided_at=? WHERE id=?",
                     (user_id, now(), proposal_id))
        action, result = "email.activity_dismissed", {"status": "dismissed"}
    else:
        return {"error": "bad_decision"}
    if close_matrix:
        _mark_proposal_matrix(conn, proposal_id, closed_at=now())
    conn.execute(
        "INSERT INTO interaction_log (id, ts, actor_type, actor_id, action, target_type, target_id, payload, source, created_at) "
        "VALUES (?,?,?,?,?,?,?,?,?,?)",
        (generate_id(), now(), "human", user_id, action, "fundraising_investor", p["investor_id"],
         json.dumps({"proposal_id": proposal_id}), source, now()))
    conn.commit()
    return result


# ─── Matrix review-bot bridge for email-activity proposals (Features 2/3) ──────
# The CRM (box, stdlib, no matrix-nio) can't post to Matrix, so the intake bot (Spark) PULLS
# pending proposals via list_bot_email_proposals, posts a review card to the dedicated review
# room, and writes the thread-root event_id back here. State lives CRM-side (email_proposal_matrix)
# so both surfaces stay in sync and it survives a bot restart. All queries degrade to empty when
# the email integration tables are absent (OperationalError), mirroring list_email_activity_proposals.
_BOT_PROPOSAL_COLS = (
    "SELECT p.id, p.investor_name, p.direction, p.summary, p.proposed_note, "
    "p.email_subject, p.email_date, e.from_name, e.from_email, e.snippet, "
    "m.event_id AS event_id, p.status AS status "
    "FROM email_activity_proposals p "
    "LEFT JOIN email_proposal_matrix m ON m.proposal_id = p.id "
    "LEFT JOIN emails e ON e.id = p.email_id ")


def list_bot_email_proposals(conn, limit=100):
    """The three work-lists the Matrix review bot polls:
      to_post  — pending, not yet posted to Matrix  -> bot posts a review card.
      open     — pending, posted, not closed         -> live threads; the bot rebuilds its
                 event_id->proposal routing map from these after a restart.
      to_close — decided on the WEB while a thread is open -> bot announces it in-thread, closes.
    Each item carries the card content (investor, direction, subject, date, from, snippet, note)."""
    try:
        to_post = [dict(r) for r in conn.execute(
            _BOT_PROPOSAL_COLS + "WHERE p.status='pending' AND (m.proposal_id IS NULL OR m.posted_at IS NULL) "
            "ORDER BY p.email_date ASC, p.created_at ASC LIMIT ?", (limit,))]
        open_threads = [dict(r) for r in conn.execute(
            _BOT_PROPOSAL_COLS + "WHERE p.status='pending' AND m.posted_at IS NOT NULL AND m.closed_at IS NULL "
            "ORDER BY p.email_date ASC, p.created_at ASC LIMIT ?", (limit,))]
        to_close = [dict(r) for r in conn.execute(
            _BOT_PROPOSAL_COLS + "WHERE p.status!='pending' AND m.posted_at IS NOT NULL AND m.closed_at IS NULL "
            "ORDER BY p.decided_at ASC LIMIT ?", (limit,))]
    except sqlite3.OperationalError:
        return {"to_post": [], "open": [], "to_close": []}
    return {"to_post": to_post, "open": open_threads, "to_close": to_close}


def _mark_proposal_matrix(conn, proposal_id, *, event_id=None, posted_at=None, closed_at=None):
    """Idempotent upsert of the 1:1 Matrix side row. Only the passed fields are written."""
    cols, vals, sets = ["proposal_id"], [proposal_id], []
    for name, val in (("event_id", event_id), ("posted_at", posted_at), ("closed_at", closed_at)):
        if val is not None:
            cols.append(name); vals.append(val); sets.append(f"{name}=excluded.{name}")
    placeholders = ",".join("?" for _ in cols)
    sql = f"INSERT INTO email_proposal_matrix ({','.join(cols)}) VALUES ({placeholders})"
    if sets:
        sql += " ON CONFLICT(proposal_id) DO UPDATE SET " + ",".join(sets)
    conn.execute(sql, vals)


def mark_proposal_matrix_posted(conn, proposal_id, event_id):
    """Record that the bot posted a review card (thread root = event_id)."""
    if not conn.execute("SELECT 1 FROM email_activity_proposals WHERE id=?", (proposal_id,)).fetchone():
        return {"error": "not_found"}
    _mark_proposal_matrix(conn, proposal_id, event_id=event_id, posted_at=now())
    conn.commit()
    return {"ok": True}


def mark_proposal_matrix_closed(conn, proposal_id):
    """Mark the Matrix review thread resolved (the bot announced a web-side decision)."""
    if not conn.execute("SELECT 1 FROM email_activity_proposals WHERE id=?", (proposal_id,)).fetchone():
        return {"error": "not_found"}
    _mark_proposal_matrix(conn, proposal_id, closed_at=now())
    conn.commit()
    return {"ok": True}


# ─── Main Entry Point ────────────────────────────────────────────────────────

def main():
    if ENV == "production" and not os.environ.get("CRM_SECRET_KEY"):
        print("ERROR: CRM_SECRET_KEY must be set in production mode (CRM_ENV=production).")
        sys.exit(1)
    init_db()
    if SEED_DEMO_DATA:
        seed_demo_data()
    else:
        print("Demo data seeding disabled (set CRM_SEED_DEMO_DATA=1 to enable).")
    start_backup_scheduler()

    # ─── Gmail sync scheduler (feature-flag-guarded) ─────────────────
    if os.environ.get("CRM_GMAIL_INTEGRATION_ENABLED", "").lower() in ("1", "true", "yes", "on"):
        try:
            from email_integration.scheduler import start_sync_scheduler
            # After each Gmail sync, draft proposed activity notes for human review.
            start_sync_scheduler(post_sync=lambda: propose_email_activity_notes())
            print("[email_integration] Gmail sync scheduler started")
        except Exception as _e:
            print(f"[email_integration] failed to start scheduler: {_e}")

    # ─── Daily activity digest scheduler ─────────────────────────────
    # Always started; it reads the digest policy (enabled + send hour) from the DB
    # each cycle, so the Settings → Admin toggle controls it live (no restart).
    try:
        from email_integration.digest_scheduler import start_digest_scheduler
        start_digest_scheduler()
        print("[digest] daily activity digest scheduler started (policy-controlled)")
    except Exception as _e:
        print(f"[digest] failed to start digest scheduler: {_e}")

    # ThreadingHTTPServer lets one slow request (or a wave of scanner probes)
    # not block legit users. SQLite is opened per-request via get_db(), and
    # WAL mode allows concurrent readers + a single writer, so this is safe.
    server = ThreadingHTTPServer((HOST, PORT), CRMHandler)
    server.daemon_threads = True
    print(f"\n{'='*60}")
    print(f"  Venture Fund CRM Server")
    print(f"  Running at http://{HOST}:{PORT}")
    print(f"  Database: {DB_PATH}")
    print(f"{'='*60}")
    if SEED_DEMO_DATA:
        print(f"\n  Demo login: admin / admin123")
        print(f"  Or: grant / password")
    print(f"\n  Press Ctrl+C to stop\n")

    try:
        server.serve_forever()
    except KeyboardInterrupt:
        print("\nShutting down server...")
        server.shutdown()

if __name__ == "__main__":
    main()
