#!/usr/bin/env python3
"""
Venture Fund CRM — Local Self-Hosted Server
Backend API server using Python stdlib + SQLite + bcrypt + PyJWT
"""

import json
import os
import sys
import sqlite3
import hashlib
import hmac
import time
import uuid
import csv
import io
import re
import base64
import threading
from datetime import datetime, timedelta
from http.server import HTTPServer, ThreadingHTTPServer, BaseHTTPRequestHandler
from urllib.parse import urlparse, parse_qs, unquote
from functools import wraps

# Available system packages
try:
    import bcrypt  # type: ignore
    BCRYPT_AVAILABLE = True
except Exception:
    bcrypt = None
    BCRYPT_AVAILABLE = False

try:
    import jwt  # type: ignore
    JWT_AVAILABLE = True
except Exception:
    jwt = None
    JWT_AVAILABLE = False

# Phase-1 Architect: human-gated thesis approval logic (pure stdlib; guarded).
try:
    import thesis_review  # type: ignore
except Exception:
    thesis_review = None

# Phase-1: entity-merge review + UI-triggered index jobs (guarded).
try:
    import entity_merge  # type: ignore
except Exception:
    entity_merge = None
try:
    import entity_jobs  # type: ignore
except Exception:
    entity_jobs = None

# Phase-1: the Architect agent (runs on Claude) + its tools live in backend/mcp.
# (Compute the path inline — this runs before BASE_DIR is defined below.)
try:
    sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "mcp"))
    import architect_tools as _architect_tools  # type: ignore
    import architect_agent as _architect_agent  # type: ignore
    import architect_grounding as _architect_grounding  # type: ignore
    import outreach_agent as _outreach_agent  # type: ignore
except Exception:
    _architect_tools = None
    _architect_agent = None
    _architect_grounding = None
    _outreach_agent = None

# ─── Configuration ────────────────────────────────────────────────────────────

BASE_DIR = os.path.dirname(os.path.abspath(__file__))
PROJECT_DIR = os.path.dirname(BASE_DIR)
DATA_DIR = os.environ.get("CRM_DATA_DIR", os.path.join(PROJECT_DIR, "data"))
FRONTEND_DIR = os.environ.get("CRM_FRONTEND_DIR", os.path.join(PROJECT_DIR, "frontend"))
FRONTEND_ROOT = os.path.realpath(FRONTEND_DIR)  # resolved once; the /assets/ containment boundary
DB_PATH = os.environ.get("CRM_DB_PATH", os.path.join(DATA_DIR, "crm.db"))
SECRET_KEY = os.environ.get("CRM_SECRET_KEY", "venture-crm-secret-change-in-production-" + str(uuid.uuid4()))
TOKEN_EXPIRY_HOURS = 24
HOST = os.environ.get("CRM_HOST", "0.0.0.0")
PORT = int(os.environ.get("CRM_PORT", "8080"))
CORS_ORIGIN = os.environ.get("CRM_CORS_ORIGIN", "*")
ENV = os.environ.get("CRM_ENV", "development")
LOGIN_RATE_LIMIT_PER_MIN = int(os.environ.get("CRM_LOGIN_RATE_LIMIT_PER_MIN", "20"))
WRITE_RATE_LIMIT_PER_MIN = int(os.environ.get("CRM_WRITE_RATE_LIMIT_PER_MIN", "300"))
GET_RATE_LIMIT_PER_MIN = int(os.environ.get("CRM_GET_RATE_LIMIT_PER_MIN", "600"))
# Auto-ban any IP that racks up too many 404s in a short window — almost always
# a vulnerability scanner blasting common paths (/.env, /.git/config, /swagger,
# /actuator/env, wp-json, etc.). Banned IPs get instant 429s with no DB or
# filesystem work, so they can't keep the single SQLite writer busy.
ABUSE_404_THRESHOLD = int(os.environ.get("CRM_ABUSE_404_THRESHOLD", "15"))
ABUSE_404_WINDOW_SEC = int(os.environ.get("CRM_ABUSE_404_WINDOW_SEC", "60"))
ABUSE_BAN_SEC = int(os.environ.get("CRM_ABUSE_BAN_SEC", "900"))  # 15 minutes
BACKUP_POLICY_SETTING_KEY = "fundraising_backup_policy"
DEFAULT_BACKUP_POLICY = {
    "enabled": True,
    "interval_hours": 24,
    "retention_days": 30,
    "max_backups": 60,
    "last_run_at": None
}
SEED_DEMO_DATA = os.environ.get("CRM_SEED_DEMO_DATA", "").strip().lower() in ("1", "true", "yes", "on")

os.makedirs(DATA_DIR, exist_ok=True)

# ─── Database Setup ───────────────────────────────────────────────────────────

def get_db():
    """Get a database connection with WAL mode and foreign keys enabled."""
    conn = sqlite3.connect(DB_PATH)
    conn.execute("PRAGMA journal_mode=WAL")
    conn.execute("PRAGMA foreign_keys=ON")
    conn.execute("PRAGMA busy_timeout=5000")
    conn.row_factory = sqlite3.Row
    return conn

def init_db():
    """Initialize all database tables."""
    conn = get_db()
    cursor = conn.cursor()

    cursor.executescript("""
        CREATE TABLE IF NOT EXISTS users (
            id TEXT PRIMARY KEY,
            username TEXT NOT NULL UNIQUE,
            email TEXT NOT NULL UNIQUE,
            password_hash TEXT NOT NULL,
            full_name TEXT NOT NULL,
            role TEXT NOT NULL DEFAULT 'member',
            is_active INTEGER DEFAULT 1,
            created_at TEXT DEFAULT (datetime('now')),
            updated_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS organizations (
            id TEXT PRIMARY KEY,
            name TEXT NOT NULL,
            type TEXT DEFAULT 'other',
            industry TEXT,
            website TEXT,
            phone TEXT,
            email TEXT,
            address TEXT,
            city TEXT,
            state TEXT,
            country TEXT,
            description TEXT,
            tags TEXT DEFAULT '[]',
            created_by TEXT REFERENCES users(id),
            created_at TEXT DEFAULT (datetime('now')),
            updated_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS contacts (
            id TEXT PRIMARY KEY,
            first_name TEXT NOT NULL,
            last_name TEXT NOT NULL,
            email TEXT,
            phone TEXT,
            mobile TEXT,
            title TEXT,
            organization_id TEXT REFERENCES organizations(id) ON DELETE SET NULL,
            contact_type TEXT NOT NULL DEFAULT 'prospect',
            status TEXT NOT NULL DEFAULT 'active',
            source TEXT,
            tags TEXT DEFAULT '[]',
            notes TEXT,
            linkedin_url TEXT,
            city TEXT,
            state TEXT,
            country TEXT,
            location_query TEXT,
            preferred_contact TEXT DEFAULT 'email',
            created_by TEXT REFERENCES users(id),
            created_at TEXT DEFAULT (datetime('now')),
            updated_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS opportunities (
            id TEXT PRIMARY KEY,
            name TEXT NOT NULL,
            contact_id TEXT NOT NULL REFERENCES contacts(id) ON DELETE CASCADE,
            organization_id TEXT REFERENCES organizations(id) ON DELETE SET NULL,
            stage TEXT NOT NULL DEFAULT 'lead',
            commitment_amount REAL DEFAULT 0,
            expected_amount REAL DEFAULT 0,
            probability INTEGER DEFAULT 10,
            expected_close_date TEXT,
            fund_name TEXT,
            description TEXT,
            next_step TEXT,
            owner_id TEXT NOT NULL REFERENCES users(id),
            priority TEXT DEFAULT 'medium',
            lost_reason TEXT,
            created_at TEXT DEFAULT (datetime('now')),
            updated_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS communications (
            id TEXT PRIMARY KEY,
            contact_id TEXT NOT NULL REFERENCES contacts(id) ON DELETE CASCADE,
            opportunity_id TEXT REFERENCES opportunities(id) ON DELETE SET NULL,
            type TEXT NOT NULL DEFAULT 'note',
            subject TEXT,
            body TEXT,
            communication_date TEXT NOT NULL,
            duration_minutes INTEGER,
            outcome TEXT,
            next_action TEXT,
            next_action_date TEXT,
            attendees TEXT DEFAULT '[]',
            created_by TEXT NOT NULL REFERENCES users(id),
            created_at TEXT DEFAULT (datetime('now')),
            updated_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS lp_profiles (
            id TEXT PRIMARY KEY,
            contact_id TEXT NOT NULL UNIQUE REFERENCES contacts(id) ON DELETE CASCADE,
            commitment_amount REAL DEFAULT 0,
            funded_amount REAL DEFAULT 0,
            commitment_date TEXT,
            fund_name TEXT,
            investor_type TEXT,
            accredited INTEGER DEFAULT 0,
            legal_docs_signed INTEGER DEFAULT 0,
            signed_date TEXT,
            wire_received INTEGER DEFAULT 0,
            wire_date TEXT,
            k1_sent INTEGER DEFAULT 0,
            preferred_communication TEXT DEFAULT 'email',
            notes TEXT,
            created_at TEXT DEFAULT (datetime('now')),
            updated_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS custom_fields (
            id TEXT PRIMARY KEY,
            name TEXT NOT NULL,
            entity_type TEXT NOT NULL,
            field_type TEXT NOT NULL DEFAULT 'text',
            options TEXT DEFAULT '[]',
            required INTEGER DEFAULT 0,
            display_order INTEGER DEFAULT 0,
            created_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS custom_field_values (
            id TEXT PRIMARY KEY,
            custom_field_id TEXT NOT NULL REFERENCES custom_fields(id) ON DELETE CASCADE,
            entity_id TEXT NOT NULL,
            entity_type TEXT NOT NULL,
            value TEXT,
            updated_at TEXT DEFAULT (datetime('now')),
            UNIQUE(custom_field_id, entity_id, entity_type)
        );

        CREATE TABLE IF NOT EXISTS audit_log (
            id TEXT PRIMARY KEY,
            user_id TEXT REFERENCES users(id),
            entity_type TEXT NOT NULL,
            entity_id TEXT NOT NULL,
            action TEXT NOT NULL,
            changes TEXT,
            created_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS tags (
            id TEXT PRIMARY KEY,
            name TEXT NOT NULL UNIQUE,
            color TEXT DEFAULT '#6366f1',
            created_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS feature_requests (
            id TEXT PRIMARY KEY,
            title TEXT NOT NULL,
            description TEXT,
            page TEXT,
            category TEXT DEFAULT 'general',
            priority TEXT DEFAULT 'medium',
            status TEXT DEFAULT 'new',
            requested_by TEXT,
            requested_by_user_id TEXT REFERENCES users(id),
            created_at TEXT DEFAULT (datetime('now')),
            updated_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS fundraising_state (
            id TEXT PRIMARY KEY,
            grid_json TEXT NOT NULL,
            views_json TEXT NOT NULL,
            version INTEGER NOT NULL DEFAULT 1,
            updated_by TEXT REFERENCES users(id),
            created_at TEXT DEFAULT (datetime('now')),
            updated_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS fundraising_investors (
            id TEXT PRIMARY KEY,
            investor_name TEXT NOT NULL,
            notes TEXT,
            lead TEXT,
            lead_source TEXT,
            priority INTEGER DEFAULT 0,
            follow_up INTEGER DEFAULT 0,
            graveyard INTEGER DEFAULT 0,
            longshot_followup INTEGER DEFAULT 0,
            source_row_id TEXT NOT NULL UNIQUE,
            total_invested REAL DEFAULT 0,
            updated_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS fundraising_contacts (
            id TEXT PRIMARY KEY,
            investor_id TEXT NOT NULL REFERENCES fundraising_investors(id) ON DELETE CASCADE,
            full_name TEXT NOT NULL,
            email TEXT,
            title TEXT,
            city TEXT,
            state TEXT,
            country TEXT,
            location_query TEXT,
            sort_order INTEGER DEFAULT 0,
            updated_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS fundraising_funds (
            id TEXT PRIMARY KEY,
            column_id TEXT NOT NULL UNIQUE,
            fund_name TEXT NOT NULL,
            display_order INTEGER DEFAULT 0,
            active INTEGER DEFAULT 1,
            updated_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS fundraising_commitments (
            id TEXT PRIMARY KEY,
            investor_id TEXT NOT NULL REFERENCES fundraising_investors(id) ON DELETE CASCADE,
            fund_id TEXT NOT NULL REFERENCES fundraising_funds(id) ON DELETE CASCADE,
            amount REAL DEFAULT 0,
            updated_at TEXT DEFAULT (datetime('now')),
            UNIQUE(investor_id, fund_id)
        );

        CREATE TABLE IF NOT EXISTS fundraising_views (
            id TEXT PRIMARY KEY,
            name TEXT NOT NULL,
            filters_json TEXT NOT NULL,
            quick_search TEXT,
            hidden_columns_json TEXT NOT NULL DEFAULT '[]',
            column_filters_json TEXT NOT NULL DEFAULT '[]',
            updated_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS fundraising_automation_rules (
            id TEXT PRIMARY KEY,
            name TEXT NOT NULL,
            trigger_type TEXT NOT NULL DEFAULT 'flag_change',
            condition_json TEXT NOT NULL,
            action_json TEXT NOT NULL,
            enabled INTEGER DEFAULT 1,
            updated_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS fundraising_automation_runs (
            id TEXT PRIMARY KEY,
            rule_id TEXT REFERENCES fundraising_automation_rules(id) ON DELETE SET NULL,
            investor_id TEXT REFERENCES fundraising_investors(id) ON DELETE SET NULL,
            status TEXT NOT NULL DEFAULT 'applied',
            result_json TEXT,
            created_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS fundraising_list_memberships (
            id TEXT PRIMARY KEY,
            investor_id TEXT NOT NULL REFERENCES fundraising_investors(id) ON DELETE CASCADE,
            list_key TEXT NOT NULL,
            source TEXT NOT NULL DEFAULT 'automation',
            updated_at TEXT DEFAULT (datetime('now')),
            UNIQUE(investor_id, list_key)
        );

        CREATE TABLE IF NOT EXISTS fundraising_presence (
            user_id TEXT PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE,
            username TEXT NOT NULL,
            full_name TEXT,
            active_view TEXT,
            row_id TEXT,
            col_id TEXT,
            is_editing INTEGER DEFAULT 0,
            cell_key TEXT,
            last_seen_at TEXT DEFAULT (datetime('now')),
            expires_at_epoch INTEGER NOT NULL
        );

        CREATE TABLE IF NOT EXISTS fundraising_cell_locks (
            cell_key TEXT PRIMARY KEY,
            row_id TEXT NOT NULL,
            col_id TEXT NOT NULL,
            locked_by_user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
            locked_by_username TEXT NOT NULL,
            locked_by_full_name TEXT,
            last_seen_at TEXT DEFAULT (datetime('now')),
            expires_at_epoch INTEGER NOT NULL
        );

        CREATE TABLE IF NOT EXISTS app_settings (
            key TEXT PRIMARY KEY,
            value_json TEXT NOT NULL,
            updated_at TEXT DEFAULT (datetime('now'))
        );

        -- Indexes for performance
        CREATE INDEX IF NOT EXISTS idx_contacts_type ON contacts(contact_type);
        CREATE INDEX IF NOT EXISTS idx_contacts_status ON contacts(status);
        CREATE INDEX IF NOT EXISTS idx_contacts_org ON contacts(organization_id);
        CREATE INDEX IF NOT EXISTS idx_opportunities_stage ON opportunities(stage);
        CREATE INDEX IF NOT EXISTS idx_opportunities_owner ON opportunities(owner_id);
        CREATE INDEX IF NOT EXISTS idx_opportunities_contact ON opportunities(contact_id);
        CREATE INDEX IF NOT EXISTS idx_communications_contact ON communications(contact_id);
        CREATE INDEX IF NOT EXISTS idx_communications_date ON communications(communication_date);
        CREATE INDEX IF NOT EXISTS idx_audit_entity ON audit_log(entity_type, entity_id);
        CREATE INDEX IF NOT EXISTS idx_lp_profiles_contact ON lp_profiles(contact_id);
        CREATE INDEX IF NOT EXISTS idx_feature_requests_status ON feature_requests(status);
        CREATE INDEX IF NOT EXISTS idx_feature_requests_created_at ON feature_requests(created_at);
        CREATE INDEX IF NOT EXISTS idx_fr_investor_name ON fundraising_investors(investor_name);
        CREATE INDEX IF NOT EXISTS idx_fr_investor_lead ON fundraising_investors(lead);
        CREATE INDEX IF NOT EXISTS idx_fr_contacts_investor ON fundraising_contacts(investor_id);
        CREATE INDEX IF NOT EXISTS idx_fr_commitments_investor ON fundraising_commitments(investor_id);
        CREATE INDEX IF NOT EXISTS idx_fr_commitments_fund ON fundraising_commitments(fund_id);
        CREATE INDEX IF NOT EXISTS idx_fr_automation_runs_created ON fundraising_automation_runs(created_at);
        CREATE INDEX IF NOT EXISTS idx_fr_memberships_list ON fundraising_list_memberships(list_key);
        CREATE INDEX IF NOT EXISTS idx_fr_presence_expires ON fundraising_presence(expires_at_epoch);
        CREATE INDEX IF NOT EXISTS idx_fr_locks_expires ON fundraising_cell_locks(expires_at_epoch);
    """)

    # Lightweight schema migrations for existing databases.
    for stmt in [
        "ALTER TABLE contacts ADD COLUMN city TEXT",
        "ALTER TABLE contacts ADD COLUMN state TEXT",
        "ALTER TABLE contacts ADD COLUMN country TEXT",
        "ALTER TABLE contacts ADD COLUMN location_query TEXT",
        "ALTER TABLE fundraising_investors ADD COLUMN lead_source TEXT",
    ]:
        try:
            conn.execute(stmt)
        except sqlite3.OperationalError:
            pass

    # ─── Gmail integration migrations (feature-flag-guarded import) ───
    try:
        from email_integration.db import apply_migrations as _email_apply_migrations
        _email_apply_migrations(cursor)
    except ImportError:
        pass
    except Exception as _e:
        print(f"[email_integration] migration warning: {_e}")

    conn.commit()

    # ─── Core schema migrations (Phase 0+; ordered .sql files w/ ledger) ───
    # Additive/reversible only; tracked in schema_migrations. See core_migrations.py.
    try:
        from core_migrations import apply_core_migrations as _apply_core_migrations
        _apply_core_migrations(conn)
    except Exception as _e:
        print(f"[migrations] core migration warning: {_e}")

    # One-time: populate the new fundraising_contacts.contact_id (migration 0004)
    # by re-running the grid→relational sync. No-op once every row is linked.
    try:
        _backfill_grid_contact_ids(conn)
    except Exception as _e:
        print(f"[backfill] grid contact_id backfill warning: {_e}")

    # One-time: seed the v5 thesis into the Architect's Workshop if it is empty.
    try:
        from thesis_seed import ensure_thesis_seed as _ensure_thesis_seed
        _ensure_thesis_seed(conn)
    except Exception as _e:
        print(f"[thesis] seed warning: {_e}")

    # One-time: add the 2026-06-05 Architect positioning framings as candidate options.
    try:
        from thesis_seed import ensure_positioning_framings as _ensure_positioning_framings
        _ensure_positioning_framings(conn)
    except Exception as _e:
        print(f"[thesis] positioning framings warning: {_e}")

    # One-time: stage the v2.0 reserve-asset spine (signal-engine workstream) as candidates.
    try:
        from thesis_seed import ensure_thesis_v2_candidate as _ensure_thesis_v2_candidate
        _ensure_thesis_v2_candidate(conn)
    except Exception as _e:
        print(f"[thesis] v2 candidate warning: {_e}")

    # One-time: promote the v2.0 spine to the WORKING (approved) thesis and soft-retire the old
    # settlement throughline + Pillar 1, so the live agents stop emitting the dead spine. Node-level
    # only; the canonical thesis_version freeze stays the partners' dual-approval action (guardrail #4).
    try:
        from thesis_seed import ensure_thesis_v2_promoted as _ensure_thesis_v2_promoted
        _ensure_thesis_v2_promoted(conn)
    except Exception as _e:
        print(f"[thesis] v2 promote warning: {_e}")

    conn.close()
    print(f"Database initialized at {DB_PATH}")

# ─── Auth Helpers ─────────────────────────────────────────────────────────────

def hash_password(password):
    if BCRYPT_AVAILABLE:
        return bcrypt.hashpw(password.encode('utf-8'), bcrypt.gensalt()).decode('utf-8')

    # Stdlib fallback (PBKDF2) when bcrypt is unavailable
    salt = os.urandom(16).hex()
    digest = hashlib.pbkdf2_hmac('sha256', password.encode('utf-8'), bytes.fromhex(salt), 200_000).hex()
    return f"pbkdf2_sha256${salt}${digest}"

def verify_password(password, hashed):
    if BCRYPT_AVAILABLE and not str(hashed).startswith("pbkdf2_sha256$"):
        return bcrypt.checkpw(password.encode('utf-8'), hashed.encode('utf-8'))

    try:
        scheme, salt_hex, digest_hex = str(hashed).split('$', 2)
        if scheme != 'pbkdf2_sha256':
            return False
        check = hashlib.pbkdf2_hmac('sha256', password.encode('utf-8'), bytes.fromhex(salt_hex), 200_000).hex()
        return hmac.compare_digest(check, digest_hex)
    except Exception:
        return False

def _b64url_encode(data: bytes) -> str:
    return base64.urlsafe_b64encode(data).decode('utf-8').rstrip('=')

def _b64url_decode(data: str) -> bytes:
    padding = '=' * (-len(data) % 4)
    return base64.urlsafe_b64decode((data + padding).encode('utf-8'))

def create_token(user_id, username, role):
    payload = {
        "user_id": user_id,
        "username": username,
        "role": role,
        "exp": int(time.time()) + TOKEN_EXPIRY_HOURS * 3600,
        "iat": int(time.time())
    }
    if JWT_AVAILABLE:
        return jwt.encode(payload, SECRET_KEY, algorithm="HS256")

    # Stdlib fallback token format: base64url(payload).hmac_sha256_signature
    payload_bytes = json.dumps(payload, separators=(',', ':'), sort_keys=True).encode('utf-8')
    payload_part = _b64url_encode(payload_bytes)
    signature = hmac.new(SECRET_KEY.encode('utf-8'), payload_part.encode('utf-8'), hashlib.sha256).digest()
    return f"{payload_part}.{_b64url_encode(signature)}"

def decode_token(token):
    if JWT_AVAILABLE:
        try:
            return jwt.decode(token, SECRET_KEY, algorithms=["HS256"])
        except jwt.ExpiredSignatureError:
            return None
        except jwt.InvalidTokenError:
            return None

    try:
        payload_part, sig_part = str(token).split('.', 1)
        expected_sig = hmac.new(SECRET_KEY.encode('utf-8'), payload_part.encode('utf-8'), hashlib.sha256).digest()
        actual_sig = _b64url_decode(sig_part)
        if not hmac.compare_digest(expected_sig, actual_sig):
            return None
        payload = json.loads(_b64url_decode(payload_part).decode('utf-8'))
        exp = int(payload.get('exp', 0))
        if exp and time.time() > exp:
            return None
        return payload
    except Exception:
        return None

# ─── Helper Functions ─────────────────────────────────────────────────────────

def row_to_dict(row):
    if row is None:
        return None
    d = dict(row)
    # Parse JSON fields
    for key in ['tags', 'attendees', 'options']:
        if key in d and isinstance(d[key], str):
            try:
                d[key] = json.loads(d[key])
            except (json.JSONDecodeError, TypeError):
                pass
    return d

def rows_to_list(rows):
    return [row_to_dict(r) for r in rows]

def generate_id():
    return str(uuid.uuid4())[:8]

def now():
    return datetime.utcnow().isoformat() + "Z"

def deep_copy_json(value):
    return json.loads(json.dumps(value))

def parse_iso_utc(ts):
    if not ts or not isinstance(ts, str):
        return None
    try:
        if ts.endswith('Z'):
            ts = ts[:-1] + '+00:00'
        return datetime.fromisoformat(ts)
    except Exception:
        return None

def require_admin(user):
    return bool(user and user.get('role') == 'admin')

def log_audit(conn, user_id, entity_type, entity_id, action, changes=None):
    conn.execute(
        "INSERT INTO audit_log (id, user_id, entity_type, entity_id, action, changes) VALUES (?, ?, ?, ?, ?, ?)",
        (generate_id(), user_id, entity_type, entity_id, action, json.dumps(changes) if changes else None)
    )

def get_app_setting(conn, key, default_value=None):
    row = conn.execute("SELECT value_json FROM app_settings WHERE key = ?", (key,)).fetchone()
    if not row:
        return deep_copy_json(default_value) if default_value is not None else None
    try:
        return json.loads(row['value_json'])
    except Exception:
        return deep_copy_json(default_value) if default_value is not None else None

def set_app_setting(conn, key, value):
    payload = json.dumps(value)
    conn.execute("""
        INSERT INTO app_settings (key, value_json, updated_at)
        VALUES (?, ?, ?)
        ON CONFLICT(key) DO UPDATE SET value_json = excluded.value_json, updated_at = excluded.updated_at
    """, (key, payload, now()))

def load_backup_policy(conn):
    raw = get_app_setting(conn, BACKUP_POLICY_SETTING_KEY, DEFAULT_BACKUP_POLICY)
    if not isinstance(raw, dict):
        raw = {}
    policy = deep_copy_json(DEFAULT_BACKUP_POLICY)
    policy.update(raw)
    policy['enabled'] = bool(policy.get('enabled'))
    policy['interval_hours'] = max(1, min(168, int(policy.get('interval_hours') or 24)))
    policy['retention_days'] = max(1, min(365, int(policy.get('retention_days') or 30)))
    policy['max_backups'] = max(1, min(1000, int(policy.get('max_backups') or 60)))
    policy['last_run_at'] = policy.get('last_run_at') if isinstance(policy.get('last_run_at'), str) else None
    return policy

def save_backup_policy(conn, policy):
    normalized = {
        "enabled": bool(policy.get('enabled')),
        "interval_hours": max(1, min(168, int(policy.get('interval_hours') or 24))),
        "retention_days": max(1, min(365, int(policy.get('retention_days') or 30))),
        "max_backups": max(1, min(1000, int(policy.get('max_backups') or 60))),
        "last_run_at": policy.get('last_run_at') if isinstance(policy.get('last_run_at'), str) else None
    }
    set_app_setting(conn, BACKUP_POLICY_SETTING_KEY, normalized)
    return normalized

def _to_bool(value):
    if isinstance(value, bool):
        return value
    if isinstance(value, (int, float)):
        return value != 0
    if isinstance(value, str):
        v = value.strip().lower()
        return v in ('1', 'true', 'yes', 'y', 'on')
    return False

def _to_number(value):
    if value is None:
        return 0.0
    if isinstance(value, (int, float)):
        return float(value)
    if isinstance(value, str):
        cleaned = value.replace(',', '').replace('$', '').strip()
        if cleaned == '':
            return 0.0
        try:
            return float(cleaned)
        except Exception:
            return 0.0
    return 0.0

def _split_full_name(full_name):
    parts = [p for p in str(full_name or '').strip().split() if p]
    if not parts:
        return '', ''
    if len(parts) == 1:
        return parts[0], ''
    return parts[0], ' '.join(parts[1:])

def _normalize_text(value):
    return str(value or '').strip().lower()

def _parse_location_text(text):
    raw = str(text or '').strip()
    if not raw:
        return '', '', '', ''
    parts = [p.strip() for p in raw.split(',') if p.strip()]
    city = parts[0] if len(parts) >= 1 else ''
    state = parts[1] if len(parts) >= 2 else ''
    country = parts[2] if len(parts) >= 3 else ''
    return city, state, country, raw

def ensure_default_automation_rules(conn):
    defaults = [
        {
            "id": "auto-graveyard-route",
            "name": "Route Graveyard Investors",
            "trigger_type": "flag_change",
            "condition_json": {"field": "graveyard", "equals": True},
            "action_json": {"set_list": "graveyard"},
            "enabled": 1
        },
        {
            "id": "auto-followup-route",
            "name": "Route Follow-up Investors",
            "trigger_type": "flag_change",
            "condition_json": {"field": "follow_up", "equals": True},
            "action_json": {"set_list": "follow_up"},
            "enabled": 1
        }
    ]
    for r in defaults:
        conn.execute("""
            INSERT INTO fundraising_automation_rules (id, name, trigger_type, condition_json, action_json, enabled, updated_at)
            VALUES (?, ?, ?, ?, ?, ?, ?)
            ON CONFLICT(id) DO UPDATE SET
                name = excluded.name,
                trigger_type = excluded.trigger_type,
                condition_json = excluded.condition_json,
                action_json = excluded.action_json,
                updated_at = excluded.updated_at
        """, (r["id"], r["name"], r["trigger_type"], json.dumps(r["condition_json"]), json.dumps(r["action_json"]), r["enabled"], now()))

def run_fundraising_automations(conn):
    ensure_default_automation_rules(conn)
    rules = rows_to_list(conn.execute("SELECT * FROM fundraising_automation_rules WHERE enabled = 1 ORDER BY id").fetchall())
    investors = rows_to_list(conn.execute("SELECT * FROM fundraising_investors").fetchall())
    previous = {}
    for m in rows_to_list(conn.execute("SELECT investor_id, list_key FROM fundraising_list_memberships").fetchall()):
        previous.setdefault(m['investor_id'], set()).add(m['list_key'])

    desired = {}
    for inv in investors:
        inv_id = inv['id']
        lists = {'all'}
        if _to_bool(inv.get('graveyard')):
            lists.add('graveyard')
        else:
            lists.add('main')
        if _to_bool(inv.get('follow_up')):
            lists.add('follow_up')
        desired[inv_id] = lists

    conn.execute("DELETE FROM fundraising_list_memberships")
    for inv_id, lists in desired.items():
        for key in sorted(lists):
            conn.execute("""
                INSERT INTO fundraising_list_memberships (id, investor_id, list_key, source, updated_at)
                VALUES (?, ?, ?, 'automation', ?)
            """, (generate_id(), inv_id, key, now()))

    for inv in investors:
        inv_id = inv['id']
        before = previous.get(inv_id, set())
        after = desired.get(inv_id, set())
        if before == after:
            continue
        added = sorted(list(after - before))
        removed = sorted(list(before - after))
        if added or removed:
            conn.execute("""
                INSERT INTO fundraising_automation_runs (id, rule_id, investor_id, status, result_json, created_at)
                VALUES (?, ?, ?, 'applied', ?, ?)
            """, (
                generate_id(),
                None,
                inv_id,
                json.dumps({"lists_added": added, "lists_removed": removed}),
                now()
            ))

def _ensure_organization_by_name(conn, org_name, actor_user_id=None):
    name = str(org_name or '').strip()
    if not name:
        return None
    existing = conn.execute("SELECT id FROM organizations WHERE lower(name) = lower(?) LIMIT 1", (name,)).fetchone()
    if existing:
        return existing['id']
    org_id = generate_id()
    conn.execute(
        "INSERT INTO organizations (id, name, type, created_by, updated_at) VALUES (?, ?, 'other', ?, ?)",
        (org_id, name, actor_user_id, now())
    )
    return org_id

def _upsert_contact_from_fundraising(conn, investor_name, contact, actor_user_id=None):
    if not isinstance(contact, dict):
        return None
    full_name = str(contact.get('name') or '').strip()
    email = str(contact.get('email') or '').strip()
    title = str(contact.get('title') or '').strip()
    source = str(contact.get('source') or '').strip()
    city = str(contact.get('city') or '').strip()
    state = str(contact.get('state') or '').strip()
    country = str(contact.get('country') or '').strip()
    location_query = str(contact.get('location_query') or '').strip()
    linkedin_url = str(contact.get('linkedin_url') or '').strip()
    if not full_name and not email:
        return None
    first_name, last_name = _split_full_name(full_name)
    if not first_name and email:
        first_name = email.split('@')[0]
    org_id = _ensure_organization_by_name(conn, investor_name, actor_user_id)

    existing = None
    if email:
        existing = conn.execute(
            "SELECT * FROM contacts WHERE lower(email) = lower(?) ORDER BY updated_at DESC LIMIT 1",
            (email,)
        ).fetchone()
    if not existing and first_name:
        if org_id:
            existing = conn.execute(
                """
                SELECT * FROM contacts
                WHERE lower(first_name) = lower(?) AND lower(last_name) = lower(?) AND organization_id = ?
                ORDER BY updated_at DESC LIMIT 1
                """,
                (first_name, last_name, org_id)
            ).fetchone()
        else:
            existing = conn.execute(
                """
                SELECT * FROM contacts
                WHERE lower(first_name) = lower(?) AND lower(last_name) = lower(?) AND organization_id IS NULL
                ORDER BY updated_at DESC LIMIT 1
                """,
                (first_name, last_name)
            ).fetchone()

    if existing:
        next_first = first_name or str(existing['first_name'] or '')
        next_last = last_name if (last_name or full_name) else str(existing['last_name'] or '')
        next_email = email or str(existing['email'] or '')
        next_title = title or str(existing['title'] or '')
        next_source = source or str(existing['source'] or '')
        next_city = city or str(existing['city'] or '')
        next_state = state or str(existing['state'] or '')
        next_country = country or str(existing['country'] or '')
        next_location_query = location_query or str(existing['location_query'] or '')
        next_linkedin = linkedin_url or str(existing['linkedin_url'] or '')
        next_org = org_id or existing['organization_id']
        conn.execute("""
            UPDATE contacts
            SET first_name = ?, last_name = ?, email = ?, title = ?,
                organization_id = ?, source = ?, contact_type = 'investor', city = ?, state = ?, country = ?, location_query = ?, linkedin_url = ?, updated_at = ?
            WHERE id = ?
        """, (next_first, next_last, next_email, next_title, next_org, next_source, next_city, next_state, next_country, next_location_query, next_linkedin, now(), existing['id']))
        return existing['id']

    contact_id = generate_id()
    conn.execute("""
        INSERT INTO contacts (
            id, first_name, last_name, email, title, organization_id, source, contact_type, status, city, state, country, location_query, linkedin_url, created_by, updated_at
        ) VALUES (?, ?, ?, ?, ?, ?, ?, 'investor', 'active', ?, ?, ?, ?, ?, ?, ?)
    """, (
        contact_id,
        first_name or 'Unknown',
        last_name or '',
        email,
        title,
        org_id,
        source,
        city,
        state,
        country,
        location_query,
        linkedin_url,
        actor_user_id,
        now()
    ))
    return contact_id

def _sync_contact_to_fundraising_state(conn, contact_row, actor_user_id=None, remove=False):
    if not isinstance(contact_row, dict):
        return
    row = conn.execute("SELECT * FROM fundraising_state WHERE id = 'main'").fetchone()
    if not row:
        return
    try:
        grid = json.loads(row['grid_json']) if row['grid_json'] else {}
    except Exception:
        grid = {}
    grid = sanitize_fundraising_grid(grid)
    columns = grid.get('columns', [])
    rows = grid.get('rows', [])
    if not isinstance(columns, list) or not isinstance(rows, list):
        return
    if not any(isinstance(c, dict) and c.get('id') == 'contacts' for c in columns):
        return

    org_name = str(contact_row.get('organization_name') or '').strip()
    email = str(contact_row.get('email') or '').strip()
    full_name = ' '.join([str(contact_row.get('first_name') or '').strip(), str(contact_row.get('last_name') or '').strip()]).strip()
    title = str(contact_row.get('title') or '').strip()
    source = str(contact_row.get('source') or '').strip()
    city = str(contact_row.get('city') or '').strip()
    state = str(contact_row.get('state') or '').strip()
    country = str(contact_row.get('country') or '').strip()
    location_query = str(contact_row.get('location_query') or '').strip()
    if not full_name and not email:
        return

    target_row_indexes = []
    org_norm = _normalize_text(org_name)
    for idx, inv in enumerate(rows):
        if not isinstance(inv, dict):
            continue
        if org_norm and _normalize_text(inv.get('investor_name')) == org_norm:
            target_row_indexes.append(idx)
    if not target_row_indexes:
        for idx, inv in enumerate(rows):
            if not isinstance(inv, dict):
                continue
            contacts = inv.get('contacts')
            if not isinstance(contacts, list):
                continue
            for c in contacts:
                if not isinstance(c, dict):
                    continue
                if email and _normalize_text(c.get('email')) == _normalize_text(email):
                    target_row_indexes.append(idx)
                    break
                if full_name and _normalize_text(c.get('name')) == _normalize_text(full_name):
                    target_row_indexes.append(idx)
                    break

    if not target_row_indexes:
        return

    changed = False
    email_norm = _normalize_text(email)
    name_norm = _normalize_text(full_name)
    for idx in target_row_indexes:
        inv = rows[idx]
        contacts = inv.get('contacts')
        if not isinstance(contacts, list):
            contacts = []
        next_contacts = list(contacts)
        match_index = -1
        for c_idx, c in enumerate(next_contacts):
            if not isinstance(c, dict):
                continue
            c_email_norm = _normalize_text(c.get('email'))
            c_name_norm = _normalize_text(c.get('name'))
            if email_norm and c_email_norm and c_email_norm == email_norm:
                match_index = c_idx
                break
            if name_norm and c_name_norm == name_norm:
                match_index = c_idx
                break

        if remove:
            if match_index >= 0:
                next_contacts.pop(match_index)
                inv['contacts'] = next_contacts
                changed = True
            continue

        if match_index >= 0:
            existing = next_contacts[match_index] if isinstance(next_contacts[match_index], dict) else {}
            next_contacts[match_index] = {
                **existing,
                "name": full_name or existing.get('name') or '',
                "email": email or existing.get('email') or '',
                "title": title or existing.get('title') or '',
                "city": city or str(existing.get('city') or ''),
                "state": state or str(existing.get('state') or ''),
                "country": country or str(existing.get('country') or ''),
                "location_query": location_query or str(existing.get('location_query') or '')
            }
        else:
            next_contacts.append({
                "name": full_name,
                "email": email,
                "title": title,
                "city": city,
                "state": state,
                "country": country,
                "location_query": location_query
            })
        inv['contacts'] = next_contacts
        if source and not str(inv.get('lead_source') or '').strip():
            inv['lead_source'] = source
        changed = True

    if not changed:
        return

    next_views = []
    try:
        next_views = json.loads(row['views_json']) if row['views_json'] else []
    except Exception:
        next_views = []
    next_views = sanitize_grid_views(next_views)
    next_version = int(row['version'] or 1) + 1
    conn.execute("""
        UPDATE fundraising_state
        SET grid_json = ?, views_json = ?, version = ?, updated_by = ?, updated_at = ?
        WHERE id = 'main'
    """, (json.dumps(grid), json.dumps(next_views), next_version, actor_user_id, now()))
    sync_fundraising_relational(conn, grid, next_views, actor_user_id=actor_user_id)

def _backfill_grid_contact_ids(conn):
    """One-time backfill for migration 0004: populate fundraising_contacts.contact_id
    by re-running the grid→relational sync once. Fires only when the column exists
    AND some row still lacks a contact_id, so it runs once after the migration and is
    a no-op thereafter. Safe + idempotent: the fundraising_* tables are derived and
    rebuilt on every sync, and _upsert_contact_from_fundraising matches existing
    contacts by email/name (never creates a duplicate on re-run)."""
    try:
        need = conn.execute("SELECT 1 FROM fundraising_contacts WHERE contact_id IS NULL LIMIT 1").fetchone()
    except sqlite3.OperationalError:
        return  # contact_id column not present (migration 0004 not applied)
    if not need:
        return
    row = conn.execute("SELECT grid_json, views_json FROM fundraising_state WHERE id = 'main'").fetchone()
    if not row or not row[0]:
        return
    try:
        grid = json.loads(row[0])
        views = json.loads(row[1]) if row[1] else []
    except Exception:
        return
    sync_fundraising_relational(conn, sanitize_fundraising_grid(grid), views)
    conn.commit()
    print("[backfill] populated fundraising_contacts.contact_id from grid sync")


def sync_fundraising_relational(conn, grid, views, actor_user_id=None):
    columns = grid.get('columns', []) if isinstance(grid, dict) else []
    rows = grid.get('rows', []) if isinstance(grid, dict) else []
    views = views if isinstance(views, list) else []

    fund_columns = []
    for idx, col in enumerate(columns):
        if not isinstance(col, dict):
            continue
        col_id = str(col.get('id') or '').strip()
        if not col_id:
            continue
        is_fund = bool(col.get('isFund')) or col.get('type') == 'currency'
        if is_fund:
            fund_columns.append((idx, col))

    seen_fund_col_ids = set()
    fund_id_by_col = {}
    for idx, col in fund_columns:
        col_id = str(col.get('id'))
        label = str(col.get('label') or col_id).strip()
        seen_fund_col_ids.add(col_id)
        existing = conn.execute("SELECT id FROM fundraising_funds WHERE column_id = ?", (col_id,)).fetchone()
        fund_id = existing['id'] if existing else generate_id()
        conn.execute("""
            INSERT INTO fundraising_funds (id, column_id, fund_name, display_order, active, updated_at)
            VALUES (?, ?, ?, ?, 1, ?)
            ON CONFLICT(column_id) DO UPDATE SET
                fund_name = excluded.fund_name,
                display_order = excluded.display_order,
                active = 1,
                updated_at = excluded.updated_at
        """, (fund_id, col_id, label, idx, now()))
        fund_id_by_col[col_id] = fund_id

    if seen_fund_col_ids:
        placeholders = ','.join(['?'] * len(seen_fund_col_ids))
        conn.execute(f"UPDATE fundraising_funds SET active = 0, updated_at = ? WHERE column_id NOT IN ({placeholders})", [now(), *list(seen_fund_col_ids)])
    else:
        conn.execute("UPDATE fundraising_funds SET active = 0, updated_at = ?", (now(),))

    seen_source_row_ids = set()
    for row in rows:
        if not isinstance(row, dict):
            continue
        source_row_id = str(row.get('id') or '').strip()
        if not source_row_id:
            continue
        seen_source_row_ids.add(source_row_id)
        investor_name = str(row.get('investor_name') or '').strip() or 'Untitled Investor'
        notes = str(row.get('notes') or '')
        lead = str(row.get('lead') or '')
        lead_source = str(row.get('lead_source') or row.get('combined_lead_source') or '').strip()
        total_invested = 0.0
        for _, col in fund_columns:
            total_invested += _to_number(row.get(str(col.get('id'))))

        existing = conn.execute("SELECT id FROM fundraising_investors WHERE source_row_id = ?", (source_row_id,)).fetchone()
        investor_id = existing['id'] if existing else generate_id()
        conn.execute("""
            INSERT INTO fundraising_investors (
                id, investor_name, notes, lead, lead_source, priority, follow_up, graveyard,
                source_row_id, total_invested, updated_at
            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
            ON CONFLICT(source_row_id) DO UPDATE SET
                investor_name = excluded.investor_name,
                notes = excluded.notes,
                lead = excluded.lead,
                lead_source = CASE
                    WHEN COALESCE(trim(fundraising_investors.lead_source), '') = '' THEN excluded.lead_source
                    ELSE fundraising_investors.lead_source
                END,
                priority = excluded.priority,
                follow_up = excluded.follow_up,
                graveyard = excluded.graveyard,
                total_invested = excluded.total_invested,
                updated_at = excluded.updated_at
        """, (
            investor_id,
            investor_name,
            notes,
            lead,
            lead_source,
            1 if _to_bool(row.get('priority')) else 0,
            1 if _to_bool(row.get('follow_up')) else 0,
            1 if _to_bool(row.get('graveyard')) else 0,
            source_row_id,
            total_invested,
            now()
        ))

        fresh = conn.execute("SELECT id FROM fundraising_investors WHERE source_row_id = ?", (source_row_id,)).fetchone()
        investor_id = fresh['id']

        conn.execute("DELETE FROM fundraising_contacts WHERE investor_id = ?", (investor_id,))
        contacts = row.get('contacts')
        if isinstance(contacts, list):
            for i, c in enumerate(contacts):
                if not isinstance(c, dict):
                    continue
                full_name = str(c.get('name') or '').strip()
                email = str(c.get('email') or '').strip()
                if not full_name and not email:
                    continue
                contact_payload = dict(c)
                if lead_source and not str(contact_payload.get('source') or '').strip():
                    contact_payload['source'] = lead_source
                linked_contact_id = _upsert_contact_from_fundraising(conn, investor_name, contact_payload, actor_user_id=actor_user_id)
                conn.execute("""
                    INSERT INTO fundraising_contacts (
                        id, investor_id, full_name, email, title, city, state, country, location_query, sort_order, contact_id, updated_at
                    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                """, (
                    generate_id(), investor_id, full_name, email, str(c.get('title') or ''),
                    str(c.get('city') or ''), str(c.get('state') or ''), str(c.get('country') or ''),
                    str(c.get('location_query') or ''), i, linked_contact_id, now()
                ))
        elif isinstance(contacts, str) and contacts.strip():
            linked_contact_id = _upsert_contact_from_fundraising(conn, investor_name, {"name": contacts.strip(), "email": "", "title": "", "source": lead_source}, actor_user_id=actor_user_id)
            conn.execute("""
                INSERT INTO fundraising_contacts (
                    id, investor_id, full_name, email, title, city, state, country, location_query, sort_order, contact_id, updated_at
                ) VALUES (?, ?, ?, '', '', '', '', '', '', 0, ?, ?)
            """, (generate_id(), investor_id, contacts.strip(), linked_contact_id, now()))

        conn.execute("DELETE FROM fundraising_commitments WHERE investor_id = ?", (investor_id,))
        for _, col in fund_columns:
            col_id = str(col.get('id'))
            fund_id = fund_id_by_col.get(col_id)
            if not fund_id:
                continue
            amount = _to_number(row.get(col_id))
            if abs(amount) < 1e-9:
                continue
            conn.execute("""
                INSERT INTO fundraising_commitments (id, investor_id, fund_id, amount, updated_at)
                VALUES (?, ?, ?, ?, ?)
            """, (generate_id(), investor_id, fund_id, amount, now()))

    if seen_source_row_ids:
        placeholders = ','.join(['?'] * len(seen_source_row_ids))
        conn.execute(f"DELETE FROM fundraising_investors WHERE source_row_id NOT IN ({placeholders})", list(seen_source_row_ids))
    else:
        conn.execute("DELETE FROM fundraising_investors")

    conn.execute("DELETE FROM fundraising_views")
    for v in views:
        if not isinstance(v, dict):
            continue
        view_id = str(v.get('id') or '').strip()
        if not view_id:
            continue
        conn.execute("""
            INSERT INTO fundraising_views (
                id, name, filters_json, quick_search, hidden_columns_json, column_filters_json, updated_at
            ) VALUES (?, ?, ?, ?, ?, ?, ?)
        """, (
            view_id,
            str(v.get('name') or view_id),
            json.dumps(v.get('filters') if isinstance(v.get('filters'), dict) else {}),
            str(v.get('quickSearch') or ''),
            json.dumps(v.get('hiddenColumns') if isinstance(v.get('hiddenColumns'), list) else []),
            json.dumps(v.get('columnFilters') if isinstance(v.get('columnFilters'), list) else []),
            now()
        ))
    run_fundraising_automations(conn)

def find_intake_match(conn, q, email=None):
    """Find an existing fundraising-grid investor for the intake bot's new-vs-existing hint.

    Scans the canonical grid blob (not the derived tables) so the returned `id` is the grid
    row id that handle_log_fundraising_communication matches on — keeping the bot's proposal
    consistent with where the write actually lands (no duplicate-investor risk). Matches by
    normalized investor_name first (the write's own key), then falls back to a contact email.
    Deleted investors are absent from the blob; graveyarded ones remain (a note on them is
    still valid), so no extra filtering is needed."""
    row = conn.execute("SELECT grid_json FROM fundraising_state WHERE id = 'main'").fetchone()
    if not row or not row['grid_json']:
        return None
    try:
        grid = json.loads(row['grid_json'])
    except Exception:
        return None
    rows = grid.get('rows', []) if isinstance(grid, dict) else []
    wanted_name = _normalize_text(q) if q else ''
    wanted_email = (email or '').strip().lower()
    email_hit = None
    for r in rows:
        if not isinstance(r, dict):
            continue
        rid = str(r.get('id') or '').strip()
        if not rid:
            continue
        name = str(r.get('investor_name') or '').strip()
        if wanted_name and _normalize_text(name) == wanted_name:
            return {"id": rid, "investor_name": name, "matched_on": "name"}
        if wanted_email and email_hit is None:
            contacts = r.get('contacts')
            if isinstance(contacts, list):
                for c in contacts:
                    if isinstance(c, dict) and str(c.get('email') or '').strip().lower() == wanted_email:
                        email_hit = {"id": rid, "investor_name": name, "matched_on": "email"}
                        break
    return email_hit


def ensure_fundraising_state_row(conn):
    existing = conn.execute("SELECT * FROM fundraising_state WHERE id = 'main'").fetchone()
    if not existing:
        default_grid = {
            "columns": deep_copy_json(DEFAULT_FUNDRAISING_COLUMNS),
            "rows": deep_copy_json(DEFAULT_FUNDRAISING_ROWS)
        }
        default_views = sanitize_grid_views(deep_copy_json(DEFAULT_GRID_VIEWS))
        conn.execute("""
            INSERT INTO fundraising_state (id, grid_json, views_json, version, updated_by)
            VALUES ('main', ?, ?, 1, NULL)
        """, (json.dumps(default_grid), json.dumps(default_views)))
        sync_fundraising_relational(conn, default_grid, default_views)
        conn.commit()
        return

    try:
        grid = json.loads(existing['grid_json']) if existing['grid_json'] else {}
    except Exception:
        grid = {}
    grid = sanitize_fundraising_grid(grid)
    try:
        views = json.loads(existing['views_json']) if existing['views_json'] else []
    except Exception:
        views = []
    rel_count = conn.execute("SELECT COUNT(*) AS c FROM fundraising_investors").fetchone()['c']
    if rel_count == 0:
        sync_fundraising_relational(conn, grid, sanitize_grid_views(views if isinstance(views, list) else deep_copy_json(DEFAULT_GRID_VIEWS)))
        conn.commit()

def list_backups():
    backup_dir = os.path.join(DATA_DIR, "backups")
    os.makedirs(backup_dir, exist_ok=True)
    entries = []
    for name in os.listdir(backup_dir):
        if not name.endswith('.json'):
            continue
        path = os.path.join(backup_dir, name)
        if not os.path.isfile(path):
            continue
        st = os.stat(path)
        if "pre_restore" in name:
            kind = "pre_restore"
        elif "_auto_" in name:
            kind = "auto"
        else:
            kind = "backup"
        entries.append({
            "filename": name,
            "path": path,
            "size_bytes": st.st_size,
            "modified_at": datetime.utcfromtimestamp(st.st_mtime).isoformat() + "Z",
            "kind": kind
        })
    entries.sort(key=lambda x: x["modified_at"], reverse=True)
    return entries

def create_fundraising_backup_file(state_row, kind="backup"):
    backup_dir = os.path.join(DATA_DIR, "backups")
    os.makedirs(backup_dir, exist_ok=True)
    ts = datetime.utcnow().strftime('%Y%m%dT%H%M%SZ')
    if kind == "pre_restore":
        filename = f"fundraising_state_pre_restore_{ts}.json"
    elif kind == "auto":
        filename = f"fundraising_state_auto_{ts}.json"
    else:
        filename = f"fundraising_state_{ts}.json"
    path = os.path.join(backup_dir, filename)
    try:
        grid = json.loads(state_row['grid_json']) if state_row and state_row['grid_json'] else {}
    except Exception:
        grid = {}
    try:
        views = json.loads(state_row['views_json']) if state_row and state_row['views_json'] else []
    except Exception:
        views = []
    payload = {
        "backup_at": now(),
        "version": state_row['version'] if state_row else 1,
        "updated_at": state_row['updated_at'] if state_row else None,
        "grid": grid,
        "views": views
    }
    with open(path, 'w', encoding='utf-8') as f:
        json.dump(payload, f, ensure_ascii=True, indent=2)
    return {
        "filename": filename,
        "path": path,
        "version": payload["version"],
        "kind": kind
    }

def apply_backup_retention(policy):
    entries = [b for b in list_backups() if b.get('kind') in ('backup', 'auto')]
    keep = max(1, int(policy.get('max_backups') or 60))
    cutoff = datetime.utcnow() - timedelta(days=max(1, int(policy.get('retention_days') or 30)))

    for b in entries:
        ts = parse_iso_utc(b.get('modified_at'))
        if ts and ts.replace(tzinfo=None) < cutoff and b.get('kind') == 'auto':
            try:
                os.remove(b['path'])
            except Exception:
                pass

    entries = [b for b in list_backups() if b.get('kind') in ('backup', 'auto')]
    for stale in entries[keep:]:
        if stale.get('kind') != 'auto':
            continue
        try:
            os.remove(stale['path'])
        except Exception:
            pass

def compute_restore_diff(current_grid, current_views, next_grid, next_views):
    current_columns = current_grid.get('columns', []) if isinstance(current_grid, dict) else []
    next_columns = next_grid.get('columns', []) if isinstance(next_grid, dict) else []
    current_rows = current_grid.get('rows', []) if isinstance(current_grid, dict) else []
    next_rows = next_grid.get('rows', []) if isinstance(next_grid, dict) else []

    curr_col_map = {str(c.get('id')): c for c in current_columns if isinstance(c, dict) and c.get('id')}
    next_col_map = {str(c.get('id')): c for c in next_columns if isinstance(c, dict) and c.get('id')}
    curr_row_map = {str(r.get('id')): r for r in current_rows if isinstance(r, dict) and r.get('id')}
    next_row_map = {str(r.get('id')): r for r in next_rows if isinstance(r, dict) and r.get('id')}

    added_columns = [next_col_map[k] for k in next_col_map.keys() - curr_col_map.keys()]
    removed_columns = [curr_col_map[k] for k in curr_col_map.keys() - next_col_map.keys()]

    changed_column_types = []
    for key in curr_col_map.keys() & next_col_map.keys():
        before_type = curr_col_map[key].get('type')
        after_type = next_col_map[key].get('type')
        if before_type != after_type:
            changed_column_types.append({
                "id": key,
                "label": next_col_map[key].get('label') or curr_col_map[key].get('label') or key,
                "before_type": before_type,
                "after_type": after_type
            })

    common_cols = list(curr_col_map.keys() & next_col_map.keys())
    cell_changes_count = 0
    changed_rows = set()
    for rid in curr_row_map.keys() & next_row_map.keys():
        before = curr_row_map[rid]
        after = next_row_map[rid]
        row_changed = False
        for cid in common_cols:
            if before.get(cid) != after.get(cid):
                cell_changes_count += 1
                row_changed = True
        if row_changed:
            changed_rows.add(rid)

    curr_view_ids = {str(v.get('id')) for v in current_views if isinstance(v, dict) and v.get('id')}
    next_view_ids = {str(v.get('id')) for v in next_views if isinstance(v, dict) and v.get('id')}
    curr_view_names = {str(v.get('id')): v.get('name') for v in current_views if isinstance(v, dict) and v.get('id')}
    next_view_names = {str(v.get('id')): v.get('name') for v in next_views if isinstance(v, dict) and v.get('id')}

    return {
        "columns_added": [{"id": c.get('id'), "label": c.get('label')} for c in added_columns[:25]],
        "columns_removed": [{"id": c.get('id'), "label": c.get('label')} for c in removed_columns[:25]],
        "columns_type_changed": changed_column_types[:25],
        "rows_added_count": len(next_row_map.keys() - curr_row_map.keys()),
        "rows_removed_count": len(curr_row_map.keys() - next_row_map.keys()),
        "rows_changed_count": len(changed_rows),
        "cell_changes_count": cell_changes_count,
        "views_added": [{"id": i, "name": next_view_names.get(i)} for i in list(next_view_ids - curr_view_ids)[:25]],
        "views_removed": [{"id": i, "name": curr_view_names.get(i)} for i in list(curr_view_ids - next_view_ids)[:25]]
    }

def sanitize_grid_views(views):
    if not isinstance(views, list):
        return deep_copy_json(DEFAULT_GRID_VIEWS)
    out = []
    for view in views:
        if not isinstance(view, dict) or str(view.get('id') or '') == 'view-longshot':
            continue
        next_view = deep_copy_json(view)
        filters = next_view.get('filters')
        if isinstance(filters, dict) and 'longshotOnly' in filters:
            filters.pop('longshotOnly', None)
        if not isinstance(filters, dict):
            filters = {}
            next_view['filters'] = filters
        view_id = str(next_view.get('id') or '')
        if view_id == 'view-graveyard':
            filters['graveyardOnly'] = True
            filters['includeGraveyard'] = True
        else:
            filters['graveyardOnly'] = bool(filters.get('graveyardOnly', False))
        if view_id == 'view-followup':
            filters['followUpOnly'] = True
        else:
            filters['followUpOnly'] = bool(filters.get('followUpOnly', False))
        out.append(next_view)
    return out if out else deep_copy_json(DEFAULT_GRID_VIEWS)

def sanitize_fundraising_grid(grid):
    if not isinstance(grid, dict):
        return {
            "columns": deep_copy_json(DEFAULT_FUNDRAISING_COLUMNS),
            "rows": deep_copy_json(DEFAULT_FUNDRAISING_ROWS)
        }
    columns = grid.get('columns')
    rows = grid.get('rows')
    if not isinstance(columns, list):
        columns = deep_copy_json(DEFAULT_FUNDRAISING_COLUMNS)
    if not isinstance(rows, list):
        rows = deep_copy_json(DEFAULT_FUNDRAISING_ROWS)

    clean_columns = []
    seen = set()
    for col in columns:
        if not isinstance(col, dict):
            continue
        col_id = str(col.get('id') or '').strip()
        if not col_id or col_id == 'longshot_followup' or col_id in seen:
            continue
        seen.add(col_id)
        clean_columns.append(col)

    clean_rows = []
    for row in rows:
        if not isinstance(row, dict):
            continue
        next_row = dict(row)
        next_row.pop('longshot_followup', None)
        clean_rows.append(next_row)

    return {"columns": clean_columns, "rows": clean_rows}

def maybe_run_scheduled_backup():
    conn = get_db()
    try:
        ensure_fundraising_state_row(conn)
        policy = load_backup_policy(conn)
        if not policy.get('enabled'):
            return
        interval = timedelta(hours=int(policy.get('interval_hours') or 24))
        last_run = parse_iso_utc(policy.get('last_run_at'))
        current = datetime.utcnow()
        if last_run and (current - last_run.replace(tzinfo=None)) < interval:
            return
        row = conn.execute("SELECT * FROM fundraising_state WHERE id = 'main'").fetchone()
        if not row:
            return
        create_fundraising_backup_file(row, kind="auto")
        policy['last_run_at'] = now()
        save_backup_policy(conn, policy)
        apply_backup_retention(policy)
        conn.commit()
    finally:
        conn.close()

def start_backup_scheduler():
    def _loop():
        while True:
            try:
                maybe_run_scheduled_backup()
            except Exception as exc:
                print(f"[scheduler] backup check failed: {exc}")
            time.sleep(60)
    thread = threading.Thread(target=_loop, daemon=True)
    thread.start()

# ─── Request Handler ──────────────────────────────────────────────────────────

PIPELINE_STAGES = ['lead', 'outreach', 'meeting', 'due_diligence', 'committed', 'funded']
CONTACT_TYPES = ['investor', 'prospect', 'advisor', 'other']
COMM_TYPES = ['email', 'call', 'meeting', 'note', 'text']

DEFAULT_GRID_VIEWS = [
    {"id": "view-main", "name": "Main Fundraising", "filters": {"includeGraveyard": False, "graveyardOnly": False, "followUpOnly": False, "lead": ""}, "quickSearch": "", "hiddenColumns": [], "columnFilters": []},
    {"id": "view-followup", "name": "Follow-up List", "filters": {"includeGraveyard": False, "graveyardOnly": False, "followUpOnly": True, "lead": ""}, "quickSearch": "", "hiddenColumns": [], "columnFilters": []},
    {"id": "view-graveyard", "name": "Graveyard", "filters": {"includeGraveyard": True, "graveyardOnly": True, "followUpOnly": False, "lead": ""}, "quickSearch": "", "hiddenColumns": [], "columnFilters": []},
    {"id": "view-all", "name": "All Investors", "filters": {"includeGraveyard": True, "graveyardOnly": False, "followUpOnly": False, "lead": ""}, "quickSearch": "", "hiddenColumns": [], "columnFilters": []}
]

DEFAULT_FUNDRAISING_COLUMNS = [
    {"id": "investor_name", "label": "Investor Name", "type": "text", "width": 220},
    {"id": "contacts", "label": "Contacts", "type": "contacts", "width": 260},
    {"id": "log_action", "label": "Log", "type": "action", "readOnly": True, "width": 90},
    {"id": "notes", "label": "Notes / Communication / Outreach", "type": "longtext", "width": 420},
    {"id": "lead_source", "label": "Lead Source", "type": "text", "width": 180},
    {"id": "notes_last_modified", "label": "Notes Last Modified", "type": "date", "readOnly": True, "width": 180},
    {"id": "last_communication_date", "label": "Last Communication Date", "type": "date", "readOnly": True, "width": 195},
    {"id": "priority", "label": "Priority", "type": "checkbox", "width": 110},
    {"id": "follow_up", "label": "Follow up", "type": "checkbox", "width": 110},
    {"id": "lead", "label": "Lead", "type": "select", "options": ["JK", "Grant", "MB", "Parker", "Other"], "width": 130},
    {"id": "graveyard", "label": "Graveyard", "type": "checkbox", "width": 115},
    {"id": "fund_i", "label": "Fund I", "type": "currency", "isFund": True, "width": 130},
    {"id": "fund_ii", "label": "Fund II", "type": "currency", "isFund": True, "width": 130},
    {"id": "fund_iii", "label": "Fund III", "type": "currency", "isFund": True, "width": 130},
    {"id": "tactical_fund", "label": "Tactical Fund", "type": "currency", "isFund": True, "width": 140},
    {"id": "pawn_to_e4", "label": "Pawn to E4", "type": "currency", "isFund": True, "width": 130},
    {"id": "ten31_terahash", "label": "Ten31 Terahash", "type": "currency", "isFund": True, "width": 150},
    {"id": "sats_and_stats", "label": "Sats and Stats", "type": "currency", "isFund": True, "width": 140},
    {"id": "pawn_to_f4", "label": "Pawn to f4", "type": "currency", "isFund": True, "width": 130},
    {"id": "join_the_fold", "label": "Join the Fold", "type": "currency", "isFund": True, "width": 130},
    {"id": "total_invested", "label": "Total invested", "type": "rollup", "readOnly": True, "width": 150},
    {"id": "tactical_fund_commit_date", "label": "Tactical Fund Commit Date", "type": "date", "width": 180}
]

DEFAULT_FUNDRAISING_ROWS = []

class CRMHandler(BaseHTTPRequestHandler):
    """Main HTTP request handler for the CRM API."""
    # Class-level state shared across all handler threads. Protected by
    # _abuse_lock; see rate_limited() and record_404() for usage.
    _rate_limit_buckets = {}   # (scope, ip) -> [timestamps]
    _404_buckets = {}          # ip -> [timestamps] of recent 404 responses
    _banned_ips = {}           # ip -> ban_until_epoch
    _abuse_lock = threading.Lock()

    def log_message(self, format, *args):
        """Override to use cleaner logging."""
        sys.stderr.write(f"[{datetime.now().strftime('%H:%M:%S')}] {args[0]}\n")

    # ── Request Parsing ──

    def get_body(self):
        # Cache parsed body on the request handler instance so repeated
        # calls don't try to re-read an already-consumed stream. Handler
        # instances are one-per-request in ThreadingHTTPServer, so the
        # cache is naturally request-scoped.
        if hasattr(self, '_cached_body'):
            return self._cached_body
        content_length = int(self.headers.get('Content-Length', 0))
        if content_length == 0:
            self._cached_body = {}
            return self._cached_body
        body = self.rfile.read(content_length)
        try:
            self._cached_body = json.loads(body.decode('utf-8'))
        except json.JSONDecodeError:
            self._cached_body = {}
        return self._cached_body

    def get_query_params(self):
        parsed = urlparse(self.path)
        return {k: v[0] if len(v) == 1 else v for k, v in parse_qs(parsed.query).items()}

    def get_path(self):
        return urlparse(self.path).path

    def get_user(self):
        auth = self.headers.get('Authorization', '')
        if not auth.startswith('Bearer '):
            return None
        token = auth[7:]
        payload = decode_token(token)
        if not payload:
            return None
        user_id = str(payload.get('user_id') or '').strip()
        if not user_id:
            return None
        conn = get_db()
        row = conn.execute(
            "SELECT id as user_id, username, role, is_active FROM users WHERE id = ?",
            (user_id,)
        ).fetchone()
        conn.close()
        if not row or int(row['is_active'] or 0) != 1:
            return None
        return {
            "user_id": row['user_id'],
            "username": row['username'],
            "role": row['role']
        }

    def get_client_ip(self):
        fwd = self.headers.get('X-Forwarded-For', '')
        if fwd:
            return fwd.split(',')[0].strip()
        return str(self.client_address[0])

    def rate_limited(self, scope, limit_per_minute):
        now_ts = time.time()
        bucket_key = f"{scope}:{self.get_client_ip()}"
        with self._abuse_lock:
            bucket = self._rate_limit_buckets.get(bucket_key, [])
            cutoff = now_ts - 60.0
            bucket = [t for t in bucket if t >= cutoff]
            if len(bucket) >= max(1, int(limit_per_minute)):
                self._rate_limit_buckets[bucket_key] = bucket
                return True
            bucket.append(now_ts)
            self._rate_limit_buckets[bucket_key] = bucket
            return False

    def is_banned(self):
        """Return True if the client IP is currently in the abuse blacklist.

        Uses a coarse class-level dict — fine for a handful of scanners hitting
        a small team CRM. Auto-expires entries when their ban window passes.
        """
        ip = self.get_client_ip()
        now_ts = time.time()
        with self._abuse_lock:
            until = self._banned_ips.get(ip)
            if until is None:
                return False
            if now_ts >= until:
                self._banned_ips.pop(ip, None)
                return False
            return True

    def record_404(self):
        """Track 404s per IP and auto-ban IPs that exceed the burst threshold.

        Called from send_error_json whenever we send a 404. A scanner probing
        /.env, /.git/config, /swagger, /actuator/env etc. will trip this fast
        and get parked on the blacklist for ABUSE_BAN_SEC seconds.
        """
        ip = self.get_client_ip()
        now_ts = time.time()
        with self._abuse_lock:
            bucket = self._404_buckets.get(ip, [])
            cutoff = now_ts - ABUSE_404_WINDOW_SEC
            bucket = [t for t in bucket if t >= cutoff]
            bucket.append(now_ts)
            if len(bucket) >= ABUSE_404_THRESHOLD:
                self._banned_ips[ip] = now_ts + ABUSE_BAN_SEC
                self._404_buckets.pop(ip, None)
                sys.stderr.write(
                    f"[abuse] Banning {ip} for {ABUSE_BAN_SEC}s after "
                    f"{len(bucket)} 404s in {ABUSE_404_WINDOW_SEC}s\n"
                )
            else:
                self._404_buckets[ip] = bucket

    # ── Response Helpers ──

    def send_json(self, data, status=200):
        self.send_response(status)
        self.send_header('Content-Type', 'application/json')
        self.send_header('Access-Control-Allow-Origin', CORS_ORIGIN)
        self.send_header('Access-Control-Allow-Methods', 'GET, POST, PUT, PATCH, DELETE, OPTIONS')
        self.send_header('Access-Control-Allow-Headers', 'Content-Type, Authorization')
        self.end_headers()
        self.wfile.write(json.dumps(data, default=str).encode('utf-8'))

    def send_error_json(self, message, status=400):
        # Record 404s for the abuser-detection blacklist before responding.
        # We do this on send (not on routing) so that any code path that 404s
        # contributes to the burst counter, including unknown POST paths.
        if status == 404:
            try:
                self.record_404()
            except Exception:
                pass
        self.send_json({"error": message}, status)

    def send_file(self, filepath, content_type='text/html'):
        try:
            with open(filepath, 'rb') as f:
                content = f.read()
            self.send_response(200)
            self.send_header('Content-Type', content_type)
            self.send_header('Content-Length', str(len(content)))
            self.end_headers()
            self.wfile.write(content)
        except FileNotFoundError:
            self.send_error_json("File not found", 404)

    # ── Routing ──

    def do_OPTIONS(self):
        if self.is_banned():
            return self.send_error_json("Too many requests", 429)
        self.send_response(200)
        self.send_header('Access-Control-Allow-Origin', CORS_ORIGIN)
        self.send_header('Access-Control-Allow-Methods', 'GET, POST, PUT, PATCH, DELETE, OPTIONS')
        self.send_header('Access-Control-Allow-Headers', 'Content-Type, Authorization')
        self.end_headers()

    def do_GET(self):
        # Short-circuit known abusers before any work, including file reads.
        if self.is_banned():
            return self.send_error_json("Too many requests", 429)
        # Generic per-IP GET rate limit. Generous enough for a normal user
        # (page load fans out ~15 GETs + heartbeats every ~6s) but blocks a
        # scanner blasting hundreds of paths per second.
        if self.rate_limited('get', GET_RATE_LIMIT_PER_MIN):
            return self.send_error_json("Too many requests", 429)

        path = self.get_path()

        # ─── Gmail integration routes (feature-flag-guarded) ─────────
        try:
            from email_integration.routes import try_handle as _email_try_handle
            if _email_try_handle(self):
                return
        except ImportError:
            pass

        # Serve frontend
        if path == '/' or path == '/index.html':
            return self.send_file(os.path.join(FRONTEND_DIR, 'index.html'))
        if path.startswith('/assets/'):
            filepath = os.path.join(FRONTEND_DIR, path.lstrip('/'))
            # Containment check: get_path()/urlparse does NOT normalize '..', so without
            # this an unauthenticated GET /assets/../../data/crm.db (raw client) would read
            # any file the process can — the LP DB, the JWT secret, the Gmail key. Resolve
            # and require the target stay under FRONTEND_ROOT; 404 (not 403) so it looks like
            # any other miss and still trips the scanner abuse counter.
            _real = os.path.realpath(filepath)
            if _real != FRONTEND_ROOT and not _real.startswith(FRONTEND_ROOT + os.sep):
                return self.send_error_json("File not found", 404)
            ext = os.path.splitext(path)[1].lower()
            content_types = {
                '.css': 'text/css',
                '.js': 'application/javascript',
                '.svg': 'image/svg+xml',
                '.png': 'image/png',
                '.jpg': 'image/jpeg',
                '.jpeg': 'image/jpeg',
                '.gif': 'image/gif',
                '.webp': 'image/webp',
                '.ico': 'image/x-icon',
            }
            ct = content_types.get(ext, 'application/octet-stream')
            return self.send_file(filepath, ct)

        # API routes
        if path == '/api/health':
            return self.send_json({"status": "ok", "timestamp": now()})
        if path == '/api/bootstrap/status':
            return self.handle_bootstrap_status()

        # Auth required from here
        user = self.get_user()
        if not user and path.startswith('/api/'):
            return self.send_error_json("Authentication required", 401)

        params = self.get_query_params()

        # Contacts
        if path == '/api/contacts':
            return self.handle_list_contacts(user, params)
        if re.match(r'^/api/contacts/[^/]+$', path):
            contact_id = path.split('/')[-1]
            return self.handle_get_contact(user, contact_id)
        if re.match(r'^/api/contacts/[^/]+/communications$', path):
            contact_id = path.split('/')[-2]
            return self.handle_list_contact_communications(user, contact_id, params)

        # Organizations
        if path == '/api/organizations':
            return self.handle_list_organizations(user, params)
        if re.match(r'^/api/organizations/[^/]+$', path):
            org_id = path.split('/')[-1]
            return self.handle_get_organization(user, org_id)

        # Opportunities
        if path == '/api/opportunities':
            return self.handle_list_opportunities(user, params)
        if re.match(r'^/api/opportunities/[^/]+$', path):
            opp_id = path.split('/')[-1]
            return self.handle_get_opportunity(user, opp_id)

        # Communications
        if path == '/api/communications':
            return self.handle_list_communications(user, params)

        # Reports
        if path == '/api/reports/dashboard':
            return self.handle_dashboard_report(user)
        if path == '/api/reports/pipeline':
            return self.handle_pipeline_report(user)
        if path == '/api/reports/activity':
            return self.handle_activity_report(user, params)

        # Export
        if path == '/api/export/contacts':
            return self.handle_export_contacts(user, params)

        # Feature requests
        if path == '/api/feature-requests':
            return self.handle_list_feature_requests(user, params)

        # Fundraising grid state
        if path == '/api/fundraising/state':
            return self.handle_get_fundraising_state(user)
        if path == '/api/fundraising/collab/state':
            return self.handle_get_fundraising_collab_state(user)
        if path == '/api/fundraising/export':
            return self.handle_export_fundraising_state(user)
        if path == '/api/fundraising/backups':
            return self.handle_list_fundraising_backups(user)
        if path == '/api/fundraising/backup-policy':
            return self.handle_get_backup_policy(user)
        if path == '/api/admin/digest/policy':
            return self.handle_get_digest_policy(user)
        if path == '/api/fundraising/relational-summary':
            return self.handle_get_fundraising_relational_summary(user)
        if path == '/api/fundraising/automations':
            return self.handle_list_fundraising_automations(user)
        if path == '/api/fundraising/automation-runs':
            return self.handle_list_fundraising_automation_runs(user, params)
        if path == '/api/fundraising/activity':
            return self.handle_get_fundraising_activity(user, params)
        if path == '/api/security/status':
            return self.handle_security_status(user)
        if path == '/api/system/status':
            return self.handle_system_status(user)
        if path == '/api/activity/proposals':
            return self.handle_list_activity_proposals(user)
        if path == '/api/outreach/investors':
            return self.handle_list_outreach_investors(user)
        if path == '/api/outreach/radar':
            return self.handle_outreach_radar(user)

        # Matrix intake bot — new-vs-existing lookup for its in-thread proposal
        if path == '/api/intake/match':
            return self.handle_intake_match(user, params)

        # Users
        if path == '/api/users':
            return self.handle_list_users(user)

        # Audit log
        if path == '/api/audit-log':
            return self.handle_list_audit_log(user, params)

        # ─── Architect thesis (Phase 1) ───
        if path == '/api/thesis/lines':
            return self.handle_list_thesis_lines(user)
        if path == '/api/thesis/versions':
            return self.handle_list_thesis_review_queue(user)
        if re.match(r'^/api/thesis/versions/[^/]+$', path):
            return self.handle_get_thesis_version(user, path.split('/')[-1])
        if re.match(r'^/api/thesis/[^/]+/canonical$', path):
            return self.handle_get_canonical_thesis(user, path.split('/')[-2])
        if path == '/api/architect/status':
            return self.handle_architect_status(user)
        if re.match(r'^/api/thesis/nodes/[^/]+/variants$', path):
            return self.handle_get_node_variants(user, path.split('/')[-2])
        if re.match(r'^/api/thesis/[^/]+/tree$', path):
            return self.handle_get_thesis_tree(user, path.split('/')[-2])

        # ─── Entity-merge review queue ───
        if path == '/api/entities/merge-candidates':
            return self.handle_list_merge_candidates(user, params)

        self.send_error_json("Not found", 404)

    def do_POST(self):
        if self.is_banned():
            return self.send_error_json("Too many requests", 429)

        path = self.get_path()
        body = self.get_body()

        if self.rate_limited('write', WRITE_RATE_LIMIT_PER_MIN):
            return self.send_error_json("Too many requests", 429)

        # ─── Gmail integration routes (feature-flag-guarded) ─────────
        try:
            from email_integration.routes import try_handle as _email_try_handle
            if _email_try_handle(self):
                return
        except ImportError:
            pass

        # Auth (no token needed)
        if path == '/api/auth/login':
            if self.rate_limited('login', LOGIN_RATE_LIMIT_PER_MIN):
                return self.send_error_json("Too many login attempts", 429)
            return self.handle_login(body)
        if path == '/api/auth/register':
            return self.handle_register(body)

        # Auth required
        user = self.get_user()
        if not user:
            return self.send_error_json("Authentication required", 401)

        if path == '/api/contacts':
            return self.handle_create_contact(user, body)
        if path == '/api/organizations':
            return self.handle_create_organization(user, body)
        if path == '/api/opportunities':
            return self.handle_create_opportunity(user, body)
        if path == '/api/communications':
            return self.handle_create_communication(user, body)
        if path == '/api/import/csv':
            return self.handle_import_csv(user, body)
        if path == '/api/feature-requests':
            return self.handle_create_feature_request(user, body)
        if path == '/api/fundraising/log-communication':
            return self.handle_log_fundraising_communication(user, body)
        if path == '/api/fundraising/collab/heartbeat':
            return self.handle_fundraising_collab_heartbeat(user, body)
        if path == '/api/admin/users':
            return self.handle_admin_create_user(user, body)
        if path == '/api/admin/reset-all-data':
            return self.handle_admin_reset_all_data(user, body)
        if path == '/api/admin/digest/test-email':
            return self.handle_admin_send_test_email(user, body)
        if path == '/api/admin/digest/send-now':
            return self.handle_admin_send_digest_now(user, body)
        if path == '/api/admin/digest/preview':
            return self.handle_admin_digest_preview(user, body)
        if path == '/api/fundraising/backup':
            return self.handle_backup_fundraising_state(user)
        if path == '/api/fundraising/restore-preview':
            return self.handle_preview_fundraising_restore(user, body)
        if path == '/api/fundraising/restore':
            return self.handle_restore_fundraising_state(user, body)
        if path == '/api/fundraising/backup-verify':
            return self.handle_verify_fundraising_backups(user)

        # ─── Architect thesis review (Phase 1, human approval gate) ───
        if re.match(r'^/api/thesis/versions/[^/]+/review$', path):
            return self.handle_thesis_review(user, path.split('/')[-2], body)
        # ─── Architect generation (Claude) ───
        if re.match(r'^/api/thesis/nodes/[^/]+/generate$', path):
            return self.handle_generate_options(user, path.split('/')[-2], body)
        if re.match(r'^/api/thesis/nodes/[^/]+/feedback$', path):
            return self.handle_node_feedback(user, path.split('/')[-2], body)
        if path == '/api/architect/ground':
            return self.handle_architect_ground(user, body)
        if path == '/api/outreach/draft':
            return self.handle_outreach_draft(user, body)
        if path == '/api/outreach/gmail-draft':
            return self.handle_outreach_gmail_draft(user, body)
        if re.match(r'^/api/activity/proposals/[^/]+/approve$', path):
            return self.handle_decide_activity_proposal(user, path.split('/')[-2], 'approve', body)
        if re.match(r'^/api/activity/proposals/[^/]+/dismiss$', path):
            return self.handle_decide_activity_proposal(user, path.split('/')[-2], 'dismiss', body)
        if re.match(r'^/api/thesis/nodes/[^/]+/choose$', path):
            return self.handle_choose_variant(user, path.split('/')[-2])
        if re.match(r'^/api/thesis/lines/[^/]+/approve$', path):
            return self.handle_approve_line(user, path.split('/')[-2])
        if path == '/api/thesis/lines':
            return self.handle_create_thesis_line(user, body)
        if re.match(r'^/api/thesis/lines/[^/]+/nodes$', path):
            return self.handle_add_thesis_node(user, path.split('/')[-2], body)

        # ─── UI-triggered index jobs + entity-merge decisions (Phase 1) ───
        if path == '/api/index/rebuild':
            return self.handle_index_job(user, 'rebuild_index')
        if path == '/api/index/update':
            return self.handle_index_job(user, 'update_index')
        if path == '/api/entities/find-duplicates':
            return self.handle_index_job(user, 'find_duplicates')
        if re.match(r'^/api/entities/merge-candidates/[^/]+$', path):
            return self.handle_decide_merge_candidate(user, path.split('/')[-1], body)

        self.send_error_json("Not found", 404)

    def do_PUT(self):
        if self.is_banned():
            return self.send_error_json("Too many requests", 429)
        path = self.get_path()
        body = self.get_body()
        if self.rate_limited('write', WRITE_RATE_LIMIT_PER_MIN):
            return self.send_error_json("Too many requests", 429)
        user = self.get_user()
        if not user:
            return self.send_error_json("Authentication required", 401)

        if re.match(r'^/api/contacts/[^/]+$', path):
            return self.handle_update_contact(user, path.split('/')[-1], body)
        if re.match(r'^/api/organizations/[^/]+$', path):
            return self.handle_update_organization(user, path.split('/')[-1], body)
        if re.match(r'^/api/opportunities/[^/]+$', path):
            return self.handle_update_opportunity(user, path.split('/')[-1], body)
        if re.match(r'^/api/communications/[^/]+$', path):
            return self.handle_update_communication(user, path.split('/')[-1], body)
        if path == '/api/fundraising/state':
            return self.handle_update_fundraising_state(user, body)
        if re.match(r'^/api/thesis/nodes/[^/]+$', path):
            return self.handle_edit_thesis_node(user, path.split('/')[-1], body)

        self.send_error_json("Not found", 404)

    def do_PATCH(self):
        if self.is_banned():
            return self.send_error_json("Too many requests", 429)
        path = self.get_path()
        body = self.get_body()
        if self.rate_limited('write', WRITE_RATE_LIMIT_PER_MIN):
            return self.send_error_json("Too many requests", 429)
        user = self.get_user()
        if not user:
            return self.send_error_json("Authentication required", 401)

        if re.match(r'^/api/opportunities/[^/]+/stage$', path):
            opp_id = path.split('/')[-2]
            return self.handle_update_stage(user, opp_id, body)
        if re.match(r'^/api/feature-requests/[^/]+$', path):
            fr_id = path.split('/')[-1]
            return self.handle_update_feature_request(user, fr_id, body)
        if re.match(r'^/api/admin/users/[^/]+$', path):
            target_user_id = path.split('/')[-1]
            return self.handle_admin_update_user(user, target_user_id, body)
        if path == '/api/fundraising/backup-policy':
            return self.handle_update_backup_policy(user, body)
        if path == '/api/admin/digest/policy':
            return self.handle_update_digest_policy(user, body)
        if re.match(r'^/api/fundraising/automations/[^/]+$', path):
            rule_id = path.split('/')[-1]
            return self.handle_update_fundraising_automation_rule(user, rule_id, body)

        self.send_error_json("Not found", 404)

    def do_DELETE(self):
        if self.is_banned():
            return self.send_error_json("Too many requests", 429)
        path = self.get_path()
        if self.rate_limited('write', WRITE_RATE_LIMIT_PER_MIN):
            return self.send_error_json("Too many requests", 429)
        user = self.get_user()
        if not user:
            return self.send_error_json("Authentication required", 401)

        if re.match(r'^/api/contacts/[^/]+$', path):
            return self.handle_delete_contact(user, path.split('/')[-1])
        if re.match(r'^/api/organizations/[^/]+$', path):
            return self.handle_delete_organization(user, path.split('/')[-1])
        if re.match(r'^/api/opportunities/[^/]+$', path):
            return self.handle_delete_opportunity(user, path.split('/')[-1])
        if re.match(r'^/api/communications/[^/]+$', path):
            return self.handle_delete_communication(user, path.split('/')[-1])
        if re.match(r'^/api/thesis/nodes/[^/]+$', path):
            return self.handle_retire_thesis_node(user, path.split('/')[-1])
        self.send_error_json("Not found", 404)

    # ═══════════════════════════════════════════════════════════════════════════
    # AUTH HANDLERS
    # ═══════════════════════════════════════════════════════════════════════════

    def handle_bootstrap_status(self):
        conn = get_db()
        count = int(conn.execute("SELECT COUNT(*) as c FROM users").fetchone()['c'])
        conn.close()
        return self.send_json({
            "data": {
                "user_count": count,
                "setup_required": count == 0
            }
        })

    def handle_login(self, body):
        username = body.get('username', '').strip()
        password = body.get('password', '')
        if not username or not password:
            return self.send_error_json("Username and password required")

        conn = get_db()
        user = conn.execute("SELECT * FROM users WHERE username = ? AND is_active = 1", (username,)).fetchone()
        conn.close()

        if not user or not verify_password(password, user['password_hash']):
            return self.send_error_json("Invalid credentials", 401)

        token = create_token(user['id'], user['username'], user['role'])
        return self.send_json({
            "token": token,
            "user": {
                "id": user['id'],
                "username": user['username'],
                "email": user['email'],
                "full_name": user['full_name'],
                "role": user['role']
            }
        })

    def handle_register(self, body):
        required = ['username', 'password', 'email', 'full_name']
        for field in required:
            if not body.get(field, '').strip():
                return self.send_error_json(f"{field} is required")
        if len(str(body.get('password') or '').strip()) < 8:
            return self.send_error_json("password must be at least 8 characters")

        conn = get_db()
        user_count = conn.execute("SELECT COUNT(*) as c FROM users").fetchone()['c']
        if user_count > 0:
            conn.close()
            return self.send_error_json("Registration is disabled. Ask an admin for an invite.", 403)

        existing = conn.execute("SELECT id FROM users WHERE username = ? OR email = ?",
                                (body['username'], body['email'])).fetchone()
        if existing:
            conn.close()
            return self.send_error_json("Username or email already exists")

        user_id = generate_id()
        # Bootstrap first user as admin
        role = 'admin' if user_count == 0 else body.get('role', 'member')

        conn.execute(
            "INSERT INTO users (id, username, email, password_hash, full_name, role) VALUES (?, ?, ?, ?, ?, ?)",
            (user_id, body['username'], body['email'], hash_password(body['password']), body['full_name'], role)
        )
        conn.commit()
        conn.close()

        token = create_token(user_id, body['username'], role)
        return self.send_json({
            "token": token,
            "user": {
                "id": user_id,
                "username": body['username'],
                "email": body['email'],
                "full_name": body['full_name'],
                "role": role
            }
        }, 201)

    # ═══════════════════════════════════════════════════════════════════════════
    # CONTACT HANDLERS
    # ═══════════════════════════════════════════════════════════════════════════

    def handle_list_contacts(self, user, params):
        conn = get_db()
        query = """
            SELECT c.*, o.name as organization_name,
                   (SELECT COUNT(*) FROM communications WHERE contact_id = c.id AND deleted_at IS NULL) as comm_count,
                   (SELECT MAX(communication_date) FROM communications WHERE contact_id = c.id AND deleted_at IS NULL) as last_contact_date
            FROM contacts c
            LEFT JOIN organizations o ON c.organization_id = o.id
            WHERE 1=1 AND c.deleted_at IS NULL
        """
        args = []

        if params.get('type'):
            query += " AND c.contact_type = ?"
            args.append(params['type'])
        if params.get('status'):
            query += " AND c.status = ?"
            args.append(params['status'])
        if params.get('search'):
            search = f"%{params['search']}%"
            query += " AND (c.first_name LIKE ? OR c.last_name LIKE ? OR c.email LIKE ? OR o.name LIKE ? OR c.source LIKE ?)"
            args.extend([search, search, search, search, search])
        if params.get('organization_id'):
            query += " AND c.organization_id = ?"
            args.append(params['organization_id'])
        if params.get('tag'):
            query += " AND c.tags LIKE ?"
            args.append(f'%"{params["tag"]}"%')

        sort = params.get('sort', 'updated_at')
        order = 'DESC' if params.get('order', 'desc').lower() == 'desc' else 'ASC'
        allowed_sorts = ['first_name', 'last_name', 'email', 'created_at', 'updated_at', 'contact_type', 'source']
        if sort in allowed_sorts:
            query += f" ORDER BY c.{sort} {order}"
        else:
            query += f" ORDER BY c.updated_at DESC"

        # Pagination
        limit = min(int(params.get('limit', 50)), 500)
        offset = int(params.get('offset', 0))
        count_query = f"SELECT COUNT(*) as total FROM ({query})"
        total = conn.execute(count_query, args).fetchone()['total']

        query += " LIMIT ? OFFSET ?"
        args.extend([limit, offset])

        contacts = rows_to_list(conn.execute(query, args).fetchall())
        conn.close()

        return self.send_json({
            "data": contacts,
            "total": total,
            "limit": limit,
            "offset": offset
        })

    def handle_get_contact(self, user, contact_id):
        conn = get_db()
        contact = conn.execute("""
            SELECT c.*, o.name as organization_name
            FROM contacts c
            LEFT JOIN organizations o ON c.organization_id = o.id
            WHERE c.id = ? AND c.deleted_at IS NULL
        """, (contact_id,)).fetchone()

        if not contact:
            conn.close()
            return self.send_error_json("Contact not found", 404)

        result = row_to_dict(contact)

        # Get related data
        result['communications'] = rows_to_list(conn.execute(
            """SELECT cm.*, u.full_name as created_by_name
               FROM communications cm LEFT JOIN users u ON cm.created_by = u.id
               WHERE cm.contact_id = ? AND cm.deleted_at IS NULL ORDER BY cm.communication_date DESC LIMIT 20""",
            (contact_id,)
        ).fetchall())

        result['opportunities'] = rows_to_list(conn.execute(
            "SELECT * FROM opportunities WHERE contact_id = ? AND deleted_at IS NULL ORDER BY updated_at DESC",
            (contact_id,)
        ).fetchall())

        conn.close()
        return self.send_json({"data": result})

    def handle_create_contact(self, user, body):
        if not body.get('first_name') or not body.get('last_name'):
            return self.send_error_json("first_name and last_name are required")

        contact_id = generate_id()
        conn = get_db()
        organization_id = body.get('organization_id')
        if not organization_id and body.get('organization'):
            organization_id = _ensure_organization_by_name(conn, body.get('organization'), user['user_id'])

        tags = json.dumps(body.get('tags', []))
        conn.execute("""
            INSERT INTO contacts (id, first_name, last_name, email, phone, mobile, title,
                organization_id, contact_type, status, source, tags, notes, linkedin_url,
                city, state, country, location_query, preferred_contact, created_by)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
        """, (
            contact_id, body['first_name'], body['last_name'],
            body.get('email'), body.get('phone'), body.get('mobile'),
            body.get('title'), organization_id,
            body.get('contact_type', 'prospect'), body.get('status', 'active'),
            body.get('source'), tags, body.get('notes'),
            body.get('linkedin_url'), body.get('city'), body.get('state'),
            body.get('country'), body.get('location_query'),
            body.get('preferred_contact', 'email'),
            user['user_id']
        ))

        contact = row_to_dict(conn.execute("""
            SELECT c.*, o.name as organization_name
            FROM contacts c LEFT JOIN organizations o ON c.organization_id = o.id
            WHERE c.id = ?
        """, (contact_id,)).fetchone())
        _sync_contact_to_fundraising_state(conn, contact, actor_user_id=user['user_id'], remove=False)
        log_audit(conn, user['user_id'], 'contact', contact_id, 'create')
        conn.commit()
        conn.close()
        return self.send_json({"data": contact}, 201)

    def handle_update_contact(self, user, contact_id, body):
        conn = get_db()
        existing = conn.execute("""
            SELECT c.*, o.name as organization_name
            FROM contacts c LEFT JOIN organizations o ON c.organization_id = o.id
            WHERE c.id = ?
        """, (contact_id,)).fetchone()
        if not existing:
            conn.close()
            return self.send_error_json("Contact not found", 404)
        previous_contact = row_to_dict(existing)

        updatable = ['first_name', 'last_name', 'email', 'phone', 'mobile', 'title',
                     'organization_id', 'contact_type', 'status', 'source', 'notes',
                     'linkedin_url', 'city', 'state', 'country', 'location_query', 'preferred_contact']
        sets = []
        args = []
        for field in updatable:
            if field in body:
                sets.append(f"{field} = ?")
                args.append(body[field])
        if 'organization' in body and 'organization_id' not in body:
            org_id = _ensure_organization_by_name(conn, body.get('organization'), user['user_id'])
            sets.append("organization_id = ?")
            args.append(org_id)

        if 'tags' in body:
            sets.append("tags = ?")
            args.append(json.dumps(body['tags']))

        if not sets:
            conn.close()
            return self.send_error_json("No fields to update")

        sets.append("updated_at = ?")
        args.append(now())
        args.append(contact_id)

        conn.execute(f"UPDATE contacts SET {', '.join(sets)} WHERE id = ?", args)

        contact = row_to_dict(conn.execute("""
            SELECT c.*, o.name as organization_name
            FROM contacts c LEFT JOIN organizations o ON c.organization_id = o.id
            WHERE c.id = ?
        """, (contact_id,)).fetchone())
        _sync_contact_to_fundraising_state(conn, previous_contact, actor_user_id=user['user_id'], remove=True)
        _sync_contact_to_fundraising_state(conn, contact, actor_user_id=user['user_id'], remove=False)
        log_audit(conn, user['user_id'], 'contact', contact_id, 'update', body)
        conn.commit()
        conn.close()
        return self.send_json({"data": contact})

    def handle_delete_contact(self, user, contact_id):
        conn = get_db()
        existing = conn.execute("""
            SELECT c.*, o.name as organization_name
            FROM contacts c LEFT JOIN organizations o ON c.organization_id = o.id
            WHERE c.id = ?
        """, (contact_id,)).fetchone()
        if not existing:
            conn.close()
            return self.send_error_json("Contact not found", 404)

        _sync_contact_to_fundraising_state(conn, row_to_dict(existing), actor_user_id=user['user_id'], remove=True)
        # Soft-delete (guardrail #3 — never hard-delete): mark deleted_at and
        # cascade to the contact's opportunities, communications, and lp_profile.
        _ts = now()
        conn.execute("UPDATE contacts SET deleted_at = ?, updated_at = ? WHERE id = ?", (_ts, _ts, contact_id))
        conn.execute("UPDATE opportunities SET deleted_at = ? WHERE contact_id = ? AND deleted_at IS NULL", (_ts, contact_id))
        conn.execute("UPDATE communications SET deleted_at = ? WHERE contact_id = ? AND deleted_at IS NULL", (_ts, contact_id))
        conn.execute("UPDATE lp_profiles SET deleted_at = ? WHERE contact_id = ? AND deleted_at IS NULL", (_ts, contact_id))
        log_audit(conn, user['user_id'], 'contact', contact_id, 'delete')
        conn.commit()
        conn.close()
        return self.send_json({"message": "Contact deleted"})

    # ═══════════════════════════════════════════════════════════════════════════
    # ORGANIZATION HANDLERS
    # ═══════════════════════════════════════════════════════════════════════════

    def handle_list_organizations(self, user, params):
        conn = get_db()
        query = """
            SELECT o.*,
                   (SELECT COUNT(*) FROM contacts WHERE organization_id = o.id AND deleted_at IS NULL) as contact_count,
                   (SELECT COALESCE(SUM(commitment_amount), 0) FROM opportunities WHERE organization_id = o.id AND stage = 'funded' AND deleted_at IS NULL) as total_funded
            FROM organizations o WHERE 1=1 AND o.deleted_at IS NULL
        """
        args = []
        if params.get('search'):
            search = f"%{params['search']}%"
            query += " AND (o.name LIKE ? OR o.industry LIKE ?)"
            args.extend([search, search])
        if params.get('type'):
            query += " AND o.type = ?"
            args.append(params['type'])

        query += " ORDER BY o.name ASC"
        limit = min(int(params.get('limit', 50)), 500)
        offset = int(params.get('offset', 0))
        total = conn.execute(f"SELECT COUNT(*) as total FROM ({query})", args).fetchone()['total']
        query += " LIMIT ? OFFSET ?"
        args.extend([limit, offset])

        orgs = rows_to_list(conn.execute(query, args).fetchall())
        conn.close()
        return self.send_json({"data": orgs, "total": total, "limit": limit, "offset": offset})

    def handle_get_organization(self, user, org_id):
        conn = get_db()
        org = conn.execute("SELECT * FROM organizations WHERE id = ? AND deleted_at IS NULL", (org_id,)).fetchone()
        if not org:
            conn.close()
            return self.send_error_json("Organization not found", 404)

        result = row_to_dict(org)
        result['contacts'] = rows_to_list(conn.execute(
            "SELECT * FROM contacts WHERE organization_id = ? AND deleted_at IS NULL ORDER BY last_name", (org_id,)
        ).fetchall())
        result['opportunities'] = rows_to_list(conn.execute(
            "SELECT * FROM opportunities WHERE organization_id = ? AND deleted_at IS NULL ORDER BY updated_at DESC", (org_id,)
        ).fetchall())
        conn.close()
        return self.send_json({"data": result})

    def handle_create_organization(self, user, body):
        if not body.get('name'):
            return self.send_error_json("name is required")

        org_id = generate_id()
        conn = get_db()
        tags = json.dumps(body.get('tags', []))
        conn.execute("""
            INSERT INTO organizations (id, name, type, industry, website, phone, email,
                address, city, state, country, description, tags, created_by)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
        """, (
            org_id, body['name'], body.get('type', 'other'), body.get('industry'),
            body.get('website'), body.get('phone'), body.get('email'),
            body.get('address'), body.get('city'), body.get('state'),
            body.get('country'), body.get('description'), tags, user['user_id']
        ))
        log_audit(conn, user['user_id'], 'organization', org_id, 'create')
        conn.commit()
        org = row_to_dict(conn.execute("SELECT * FROM organizations WHERE id = ?", (org_id,)).fetchone())
        conn.close()
        return self.send_json({"data": org}, 201)

    def handle_update_organization(self, user, org_id, body):
        conn = get_db()
        existing = conn.execute("SELECT id FROM organizations WHERE id = ?", (org_id,)).fetchone()
        if not existing:
            conn.close()
            return self.send_error_json("Organization not found", 404)

        updatable = ['name', 'type', 'industry', 'website', 'phone', 'email',
                     'address', 'city', 'state', 'country', 'description']
        sets = []
        args = []
        for field in updatable:
            if field in body:
                sets.append(f"{field} = ?")
                args.append(body[field])
        if 'tags' in body:
            sets.append("tags = ?")
            args.append(json.dumps(body['tags']))

        if sets:
            sets.append("updated_at = ?")
            args.append(now())
            args.append(org_id)
            conn.execute(f"UPDATE organizations SET {', '.join(sets)} WHERE id = ?", args)
            log_audit(conn, user['user_id'], 'organization', org_id, 'update', body)
            conn.commit()

        org = row_to_dict(conn.execute("SELECT * FROM organizations WHERE id = ?", (org_id,)).fetchone())
        conn.close()
        return self.send_json({"data": org})

    def handle_delete_organization(self, user, org_id):
        conn = get_db()
        existing = conn.execute("SELECT id FROM organizations WHERE id = ?", (org_id,)).fetchone()
        if not existing:
            conn.close()
            return self.send_error_json("Organization not found", 404)

        conn.execute("UPDATE contacts SET organization_id = NULL WHERE organization_id = ?", (org_id,))
        conn.execute("UPDATE organizations SET deleted_at = ?, updated_at = ? WHERE id = ?", (now(), now(), org_id))
        log_audit(conn, user['user_id'], 'organization', org_id, 'delete')
        conn.commit()
        conn.close()
        return self.send_json({"message": "Organization deleted"})

    # ═══════════════════════════════════════════════════════════════════════════
    # OPPORTUNITY (PIPELINE) HANDLERS
    # ═══════════════════════════════════════════════════════════════════════════

    def handle_list_opportunities(self, user, params):
        conn = get_db()
        query = """
            SELECT op.*, c.first_name, c.last_name, c.email as contact_email,
                   o.name as organization_name, u.full_name as owner_name
            FROM opportunities op
            LEFT JOIN contacts c ON op.contact_id = c.id
            LEFT JOIN organizations o ON op.organization_id = o.id
            LEFT JOIN users u ON op.owner_id = u.id
            WHERE 1=1 AND op.deleted_at IS NULL
        """
        args = []
        if params.get('stage'):
            query += " AND op.stage = ?"
            args.append(params['stage'])
        if params.get('owner_id'):
            query += " AND op.owner_id = ?"
            args.append(params['owner_id'])
        if params.get('search'):
            search = f"%{params['search']}%"
            query += " AND (op.name LIKE ? OR c.first_name LIKE ? OR c.last_name LIKE ?)"
            args.extend([search, search, search])
        if params.get('priority'):
            query += " AND op.priority = ?"
            args.append(params['priority'])
        if params.get('fund_name'):
            query += " AND op.fund_name = ?"
            args.append(params['fund_name'])

        query += " ORDER BY op.updated_at DESC"
        limit = min(int(params.get('limit', 100)), 500)
        offset = int(params.get('offset', 0))
        total = conn.execute(f"SELECT COUNT(*) as total FROM ({query})", args).fetchone()['total']
        query += " LIMIT ? OFFSET ?"
        args.extend([limit, offset])

        opps = rows_to_list(conn.execute(query, args).fetchall())
        conn.close()
        return self.send_json({"data": opps, "total": total, "limit": limit, "offset": offset})

    def handle_get_opportunity(self, user, opp_id):
        conn = get_db()
        opp = conn.execute("""
            SELECT op.*, c.first_name, c.last_name, c.email as contact_email,
                   o.name as organization_name, u.full_name as owner_name
            FROM opportunities op
            LEFT JOIN contacts c ON op.contact_id = c.id
            LEFT JOIN organizations o ON op.organization_id = o.id
            LEFT JOIN users u ON op.owner_id = u.id
            WHERE op.id = ? AND op.deleted_at IS NULL
        """, (opp_id,)).fetchone()

        if not opp:
            conn.close()
            return self.send_error_json("Opportunity not found", 404)

        result = row_to_dict(opp)
        result['communications'] = rows_to_list(conn.execute(
            """SELECT cm.*, u.full_name as created_by_name
               FROM communications cm LEFT JOIN users u ON cm.created_by = u.id
               WHERE cm.opportunity_id = ? AND cm.deleted_at IS NULL ORDER BY cm.communication_date DESC""",
            (opp_id,)
        ).fetchall())

        # Stage history from audit log
        result['stage_history'] = rows_to_list(conn.execute(
            """SELECT al.*, u.full_name as user_name FROM audit_log al
               LEFT JOIN users u ON al.user_id = u.id
               WHERE al.entity_type = 'opportunity' AND al.entity_id = ?
               AND al.action = 'stage_change'
               ORDER BY al.created_at DESC""",
            (opp_id,)
        ).fetchall())

        conn.close()
        return self.send_json({"data": result})

    def handle_create_opportunity(self, user, body):
        if not body.get('name') or not body.get('contact_id'):
            return self.send_error_json("name and contact_id are required")

        opp_id = generate_id()
        conn = get_db()

        # Verify contact exists
        contact = conn.execute("SELECT id, organization_id FROM contacts WHERE id = ?",
                               (body['contact_id'],)).fetchone()
        if not contact:
            conn.close()
            return self.send_error_json("Contact not found", 404)

        org_id = body.get('organization_id', contact['organization_id'])

        conn.execute("""
            INSERT INTO opportunities (id, name, contact_id, organization_id, stage,
                commitment_amount, expected_amount, probability, expected_close_date,
                fund_name, description, next_step, owner_id, priority)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
        """, (
            opp_id, body['name'], body['contact_id'], org_id,
            body.get('stage', 'lead'),
            body.get('commitment_amount', 0), body.get('expected_amount', 0),
            body.get('probability', 10), body.get('expected_close_date'),
            body.get('fund_name'), body.get('description'), body.get('next_step'),
            body.get('owner_id', user['user_id']), body.get('priority', 'medium')
        ))
        log_audit(conn, user['user_id'], 'opportunity', opp_id, 'create')
        conn.commit()

        opp = row_to_dict(conn.execute("""
            SELECT op.*, c.first_name, c.last_name, o.name as organization_name
            FROM opportunities op
            LEFT JOIN contacts c ON op.contact_id = c.id
            LEFT JOIN organizations o ON op.organization_id = o.id
            WHERE op.id = ?
        """, (opp_id,)).fetchone())
        conn.close()
        return self.send_json({"data": opp}, 201)

    def handle_update_opportunity(self, user, opp_id, body):
        conn = get_db()
        existing = conn.execute("SELECT * FROM opportunities WHERE id = ?", (opp_id,)).fetchone()
        if not existing:
            conn.close()
            return self.send_error_json("Opportunity not found", 404)

        updatable = ['name', 'contact_id', 'organization_id', 'stage', 'commitment_amount',
                     'expected_amount', 'probability', 'expected_close_date', 'fund_name',
                     'description', 'next_step', 'owner_id', 'priority', 'lost_reason']
        sets = []
        args = []

        old_stage = existing['stage']
        new_stage = body.get('stage', old_stage)

        for field in updatable:
            if field in body:
                sets.append(f"{field} = ?")
                args.append(body[field])

        if sets:
            sets.append("updated_at = ?")
            args.append(now())
            args.append(opp_id)
            conn.execute(f"UPDATE opportunities SET {', '.join(sets)} WHERE id = ?", args)

            if new_stage != old_stage:
                log_audit(conn, user['user_id'], 'opportunity', opp_id, 'stage_change',
                          {"from": old_stage, "to": new_stage})
            else:
                log_audit(conn, user['user_id'], 'opportunity', opp_id, 'update', body)
            conn.commit()

        opp = row_to_dict(conn.execute("""
            SELECT op.*, c.first_name, c.last_name, o.name as organization_name, u.full_name as owner_name
            FROM opportunities op
            LEFT JOIN contacts c ON op.contact_id = c.id
            LEFT JOIN organizations o ON op.organization_id = o.id
            LEFT JOIN users u ON op.owner_id = u.id
            WHERE op.id = ?
        """, (opp_id,)).fetchone())
        conn.close()
        return self.send_json({"data": opp})

    def handle_update_stage(self, user, opp_id, body):
        new_stage = body.get('stage', '')
        if new_stage not in PIPELINE_STAGES:
            return self.send_error_json(f"Invalid stage. Must be one of: {', '.join(PIPELINE_STAGES)}")

        conn = get_db()
        existing = conn.execute("SELECT stage FROM opportunities WHERE id = ?", (opp_id,)).fetchone()
        if not existing:
            conn.close()
            return self.send_error_json("Opportunity not found", 404)

        old_stage = existing['stage']
        conn.execute("UPDATE opportunities SET stage = ?, updated_at = ? WHERE id = ?",
                     (new_stage, now(), opp_id))
        log_audit(conn, user['user_id'], 'opportunity', opp_id, 'stage_change',
                  {"from": old_stage, "to": new_stage})
        conn.commit()

        opp = row_to_dict(conn.execute("""
            SELECT op.*, c.first_name, c.last_name, o.name as organization_name, u.full_name as owner_name
            FROM opportunities op
            LEFT JOIN contacts c ON op.contact_id = c.id
            LEFT JOIN organizations o ON op.organization_id = o.id
            LEFT JOIN users u ON op.owner_id = u.id
            WHERE op.id = ?
        """, (opp_id,)).fetchone())
        conn.close()
        return self.send_json({"data": opp})

    def handle_delete_opportunity(self, user, opp_id):
        conn = get_db()
        existing = conn.execute("SELECT id FROM opportunities WHERE id = ?", (opp_id,)).fetchone()
        if not existing:
            conn.close()
            return self.send_error_json("Opportunity not found", 404)

        conn.execute("UPDATE opportunities SET deleted_at = ?, updated_at = ? WHERE id = ?", (now(), now(), opp_id))
        log_audit(conn, user['user_id'], 'opportunity', opp_id, 'delete')
        conn.commit()
        conn.close()
        return self.send_json({"message": "Opportunity deleted"})

    # ═══════════════════════════════════════════════════════════════════════════
    # COMMUNICATION HANDLERS
    # ═══════════════════════════════════════════════════════════════════════════

    def handle_list_communications(self, user, params):
        conn = get_db()
        query = """
            SELECT cm.*, c.first_name, c.last_name, u.full_name as created_by_name
            FROM communications cm
            LEFT JOIN contacts c ON cm.contact_id = c.id
            LEFT JOIN users u ON cm.created_by = u.id
            WHERE 1=1 AND cm.deleted_at IS NULL
        """
        args = []
        if params.get('contact_id'):
            query += " AND cm.contact_id = ?"
            args.append(params['contact_id'])
        if params.get('type'):
            query += " AND cm.type = ?"
            args.append(params['type'])
        if params.get('search'):
            search = f"%{params['search']}%"
            query += " AND (cm.subject LIKE ? OR cm.body LIKE ?)"
            args.extend([search, search])

        query += " ORDER BY cm.communication_date DESC"
        limit = min(int(params.get('limit', 50)), 500)
        offset = int(params.get('offset', 0))
        total = conn.execute(f"SELECT COUNT(*) as total FROM ({query})", args).fetchone()['total']
        query += " LIMIT ? OFFSET ?"
        args.extend([limit, offset])

        comms = rows_to_list(conn.execute(query, args).fetchall())
        conn.close()
        return self.send_json({"data": comms, "total": total, "limit": limit, "offset": offset})

    def handle_list_contact_communications(self, user, contact_id, params):
        params['contact_id'] = contact_id
        return self.handle_list_communications(user, params)

    def handle_create_communication(self, user, body):
        if not body.get('contact_id'):
            return self.send_error_json("contact_id is required")

        comm_id = generate_id()
        conn = get_db()

        contact = conn.execute("SELECT id FROM contacts WHERE id = ?", (body['contact_id'],)).fetchone()
        if not contact:
            conn.close()
            return self.send_error_json("Contact not found", 404)

        attendees = json.dumps(body.get('attendees', []))
        conn.execute("""
            INSERT INTO communications (id, contact_id, opportunity_id, type, subject, body,
                communication_date, duration_minutes, outcome, next_action, next_action_date,
                attendees, created_by)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
        """, (
            comm_id, body['contact_id'], body.get('opportunity_id'),
            body.get('type', 'note'), body.get('subject'), body.get('body'),
            body.get('communication_date', now()),
            body.get('duration_minutes'), body.get('outcome'),
            body.get('next_action'), body.get('next_action_date'),
            attendees, user['user_id']
        ))

        # Update contact's updated_at
        conn.execute("UPDATE contacts SET updated_at = ? WHERE id = ?", (now(), body['contact_id']))

        log_audit(conn, user['user_id'], 'communication', comm_id, 'create')
        conn.commit()

        comm = row_to_dict(conn.execute("""
            SELECT cm.*, c.first_name, c.last_name, u.full_name as created_by_name
            FROM communications cm
            LEFT JOIN contacts c ON cm.contact_id = c.id
            LEFT JOIN users u ON cm.created_by = u.id
            WHERE cm.id = ?
        """, (comm_id,)).fetchone())
        conn.close()
        return self.send_json({"data": comm}, 201)

    def handle_log_fundraising_communication(self, user, body):
        row_id = str(body.get('row_id') or '').strip()
        investor_name_in = str(body.get('investor_name') or '').strip()
        contact_in = body.get('contact')
        append_note = bool(body.get('append_note', True))
        create_investor_if_missing = bool(body.get('create_investor_if_missing', False))
        # Provenance: where this logged communication originated (grid UI vs the Matrix
        # intake bot). Default preserves prior behavior; callers may override.
        comm_source = (str(body.get('source') or 'fundraising_grid').strip() or 'fundraising_grid')[:64]

        if not row_id and not investor_name_in:
            return self.send_error_json("row_id or investor_name is required")

        conn = get_db()
        ensure_fundraising_state_row(conn)
        state = conn.execute("SELECT * FROM fundraising_state WHERE id = 'main'").fetchone()
        if not state:
            conn.close()
            return self.send_error_json("Fundraising state not found", 404)

        try:
            grid = json.loads(state['grid_json']) if state['grid_json'] else {}
        except Exception:
            grid = {}
        grid = sanitize_fundraising_grid(grid)
        rows = grid.get('rows', [])
        if not isinstance(rows, list):
            rows = []

        target_index = -1
        target_row = None
        for idx, row in enumerate(rows):
            if not isinstance(row, dict):
                continue
            if row_id and str(row.get('id') or '').strip() == row_id:
                target_index = idx
                target_row = row
                break
        if target_row is None and investor_name_in:
            wanted = _normalize_text(investor_name_in)
            for idx, row in enumerate(rows):
                if not isinstance(row, dict):
                    continue
                if _normalize_text(row.get('investor_name')) == wanted:
                    target_index = idx
                    target_row = row
                    break
        if target_row is None and investor_name_in and create_investor_if_missing:
            new_row = {
                "id": f"inv-{int(time.time() * 1000)}-{uuid.uuid4().hex[:6]}",
                "investor_name": investor_name_in,
                "contacts": [contact_in] if isinstance(contact_in, dict) else [],
                "notes": "",
                "notes_last_modified": "",
                "last_communication_date": "",
                "lead": "",
                "priority": False,
                "follow_up": False,
                "graveyard": False
            }
            columns = grid.get('columns') if isinstance(grid.get('columns'), list) else []
            for col in columns:
                if not isinstance(col, dict):
                    continue
                col_id = str(col.get('id') or '').strip()
                if not col_id or col_id == 'longshot_followup' or col_id in new_row:
                    continue
                col_type = str(col.get('type') or 'text').strip().lower()
                if col_type == 'checkbox':
                    new_row[col_id] = False
                elif col_type in ('currency', 'number'):
                    new_row[col_id] = 0
                elif col_type == 'contacts':
                    new_row[col_id] = []
                else:
                    new_row[col_id] = ''
            rows.append(new_row)
            target_index = len(rows) - 1
            target_row = new_row
        if target_row is None:
            conn.close()
            return self.send_error_json("Fundraising row not found", 404)

        investor_name = str(target_row.get('investor_name') or investor_name_in or '').strip()
        contacts = target_row.get('contacts')
        if not isinstance(contact_in, dict):
            contact_in = None
        if contact_in is None and isinstance(contacts, list) and contacts:
            first = contacts[0]
            contact_in = first if isinstance(first, dict) else None
        if not isinstance(contact_in, dict):
            conn.close()
            return self.send_error_json("contact is required")

        contact_id = _upsert_contact_from_fundraising(conn, investor_name, contact_in, actor_user_id=user['user_id'])
        if not contact_id:
            conn.close()
            return self.send_error_json("Could not resolve contact for communication")

        comm_id = generate_id()
        comm_type = str(body.get('type') or 'note').strip() or 'note'
        comm_subject = str(body.get('subject') or '').strip()
        comm_body = str(body.get('body') or '').strip()
        comm_outcome = str(body.get('outcome') or '').strip()
        next_action = str(body.get('next_action') or '').strip()
        next_action_date = str(body.get('next_action_date') or '').strip()
        comm_date = str(body.get('communication_date') or now()).strip() or now()
        conn.execute("""
            INSERT INTO communications (id, contact_id, opportunity_id, type, subject, body,
                communication_date, duration_minutes, outcome, next_action, next_action_date,
                attendees, created_by)
            VALUES (?, ?, NULL, ?, ?, ?, ?, NULL, ?, ?, ?, '[]', ?)
        """, (
            comm_id,
            contact_id,
            comm_type,
            comm_subject,
            comm_body,
            comm_date,
            comm_outcome or None,
            next_action or None,
            next_action_date or None,
            user['user_id']
        ))
        conn.execute("UPDATE contacts SET updated_at = ? WHERE id = ?", (now(), contact_id))
        log_audit(conn, user['user_id'], 'communication', comm_id, 'create', {"source": comm_source})

        iso_day = now()[:10]
        target_row['last_communication_date'] = iso_day
        if append_note:
            contact_name = str(contact_in.get('name') or '').strip()
            summary = comm_subject or comm_body[:120]
            type_tag = "" if comm_type == "note" else f"[{comm_type}] "
            note_line = f"{iso_day} {type_tag}{contact_name}: {summary}".strip()
            existing_notes = str(target_row.get('notes') or '')
            target_row['notes'] = f"{existing_notes}\n{note_line}".strip() if existing_notes.strip() else note_line
            target_row['notes_last_modified'] = iso_day
        rows[target_index] = target_row

        try:
            views = json.loads(state['views_json']) if state['views_json'] else []
        except Exception:
            views = []
        views = sanitize_grid_views(views)
        next_version = int(state['version'] or 1) + 1
        conn.execute("""
            UPDATE fundraising_state
            SET grid_json = ?, views_json = ?, version = ?, updated_by = ?, updated_at = ?
            WHERE id = 'main'
        """, (json.dumps(grid), json.dumps(views), next_version, user['user_id'], now()))
        sync_fundraising_relational(conn, grid, views, actor_user_id=user['user_id'])
        conn.commit()

        comm = row_to_dict(conn.execute("""
            SELECT cm.*, c.first_name, c.last_name, u.full_name as created_by_name
            FROM communications cm
            LEFT JOIN contacts c ON cm.contact_id = c.id
            LEFT JOIN users u ON cm.created_by = u.id
            WHERE cm.id = ?
        """, (comm_id,)).fetchone())
        conn.close()
        return self.send_json({"data": {"communication": comm, "row": target_row, "version": next_version}}, 201)

    def handle_intake_match(self, user, params):
        """Read-only: does an investor matching this intake already exist? Used by the
        Matrix intake bot to label its in-thread proposal new-vs-existing. Returns the
        grid row id so an approved note lands on exactly that investor."""
        q = str(params.get('q') or '').strip()
        email = str(params.get('email') or '').strip()
        if not q and not email:
            return self.send_error_json("q or email is required")
        conn = get_db()
        try:
            match = find_intake_match(conn, q, email)
        finally:
            conn.close()
        return self.send_json({"data": {"match": match}})

    def handle_update_communication(self, user, comm_id, body):
        conn = get_db()
        existing = conn.execute("SELECT id FROM communications WHERE id = ?", (comm_id,)).fetchone()
        if not existing:
            conn.close()
            return self.send_error_json("Communication not found", 404)

        updatable = ['type', 'subject', 'body', 'communication_date', 'duration_minutes',
                     'outcome', 'next_action', 'next_action_date']
        sets = []
        args = []
        for field in updatable:
            if field in body:
                sets.append(f"{field} = ?")
                args.append(body[field])
        if 'attendees' in body:
            sets.append("attendees = ?")
            args.append(json.dumps(body['attendees']))

        if sets:
            sets.append("updated_at = ?")
            args.append(now())
            args.append(comm_id)
            conn.execute(f"UPDATE communications SET {', '.join(sets)} WHERE id = ?", args)
            log_audit(conn, user['user_id'], 'communication', comm_id, 'update')
            conn.commit()

        comm = row_to_dict(conn.execute("SELECT * FROM communications WHERE id = ?", (comm_id,)).fetchone())
        conn.close()
        return self.send_json({"data": comm})

    def handle_delete_communication(self, user, comm_id):
        conn = get_db()
        existing = conn.execute("SELECT id FROM communications WHERE id = ?", (comm_id,)).fetchone()
        if not existing:
            conn.close()
            return self.send_error_json("Communication not found", 404)

        conn.execute("UPDATE communications SET deleted_at = ?, updated_at = ? WHERE id = ?", (now(), now(), comm_id))
        log_audit(conn, user['user_id'], 'communication', comm_id, 'delete')
        conn.commit()
        conn.close()
        return self.send_json({"message": "Communication deleted"})

    # ═══════════════════════════════════════════════════════════════════════════
    # REPORT HANDLERS
    # ═══════════════════════════════════════════════════════════════════════════

    def handle_dashboard_report(self, user):
        conn = get_db()

        # Key metrics
        total_lps = conn.execute("SELECT COUNT(*) as c FROM contacts WHERE contact_type = 'investor'").fetchone()['c']
        total_prospects = conn.execute("SELECT COUNT(*) as c FROM contacts WHERE contact_type = 'prospect'").fetchone()['c']
        total_contacts = conn.execute("SELECT COUNT(*) as c FROM contacts").fetchone()['c']

        # Committed capital comes from the canonical fundraising grid (per-investor
        # rollup of per-fund commitments). Graveyarded (written-off) investors are
        # excluded so the headline reflects live committed capital — a deliberate
        # divergence from /api/fundraising/relational-summary, which sums all rows.
        # The legacy lp_profiles table is retired; the grid tracks commitments, not a
        # separate "funded" amount, so total_funded is no longer reported.
        total_committed = conn.execute(
            "SELECT COALESCE(SUM(total_invested), 0) as total FROM fundraising_investors WHERE graveyard = 0"
        ).fetchone()['total']

        pipeline_value = conn.execute(
            "SELECT COALESCE(SUM(expected_amount), 0) as total FROM opportunities WHERE stage NOT IN ('funded', 'lost')"
        ).fetchone()['total']

        active_opportunities = conn.execute(
            "SELECT COUNT(*) as c FROM opportunities WHERE stage NOT IN ('funded', 'lost')"
        ).fetchone()['c']

        # Pipeline by stage
        pipeline_stages = rows_to_list(conn.execute("""
            SELECT stage, COUNT(*) as count, COALESCE(SUM(expected_amount), 0) as total_value,
                   COALESCE(SUM(commitment_amount), 0) as committed_value
            FROM opportunities
            WHERE stage != 'lost'
            GROUP BY stage
            ORDER BY CASE stage
                WHEN 'lead' THEN 1 WHEN 'outreach' THEN 2 WHEN 'meeting' THEN 3
                WHEN 'due_diligence' THEN 4 WHEN 'committed' THEN 5 WHEN 'funded' THEN 6
            END
        """).fetchall())

        # Recent communications
        recent_comms = rows_to_list(conn.execute("""
            SELECT cm.*, c.first_name, c.last_name, u.full_name as created_by_name
            FROM communications cm
            LEFT JOIN contacts c ON cm.contact_id = c.id
            LEFT JOIN users u ON cm.created_by = u.id
            ORDER BY cm.communication_date DESC LIMIT 10
        """).fetchall())

        # Upcoming actions
        upcoming = rows_to_list(conn.execute("""
            SELECT cm.*, c.first_name, c.last_name
            FROM communications cm
            LEFT JOIN contacts c ON cm.contact_id = c.id
            WHERE cm.next_action IS NOT NULL AND cm.next_action != ''
            AND cm.next_action_date >= date('now')
            ORDER BY cm.next_action_date ASC LIMIT 10
        """).fetchall())

        # Recent stage changes
        recent_stage_changes = rows_to_list(conn.execute("""
            SELECT al.*, u.full_name as user_name, op.name as opportunity_name
            FROM audit_log al
            LEFT JOIN users u ON al.user_id = u.id
            LEFT JOIN opportunities op ON al.entity_id = op.id
            WHERE al.entity_type = 'opportunity' AND al.action = 'stage_change'
            ORDER BY al.created_at DESC LIMIT 10
        """).fetchall())

        # Comms this month
        comms_this_month = conn.execute("""
            SELECT COUNT(*) as c FROM communications
            WHERE communication_date >= date('now', 'start of month')
        """).fetchone()['c']

        # Meetings this month
        meetings_this_month = conn.execute("""
            SELECT COUNT(*) as c FROM communications
            WHERE type = 'meeting' AND communication_date >= date('now', 'start of month')
        """).fetchone()['c']

        conn.close()
        return self.send_json({
            "data": {
                "metrics": {
                    "total_lps": total_lps,
                    "total_prospects": total_prospects,
                    "total_contacts": total_contacts,
                    "total_committed": total_committed,
                    "pipeline_value": pipeline_value,
                    "active_opportunities": active_opportunities,
                    "comms_this_month": comms_this_month,
                    "meetings_this_month": meetings_this_month
                },
                "pipeline_stages": pipeline_stages,
                "recent_communications": recent_comms,
                "upcoming_actions": upcoming,
                "recent_stage_changes": recent_stage_changes
            }
        })

    def handle_pipeline_report(self, user):
        conn = get_db()
        stages = rows_to_list(conn.execute("""
            SELECT stage, COUNT(*) as count,
                   COALESCE(SUM(expected_amount), 0) as total_expected,
                   COALESCE(SUM(commitment_amount), 0) as total_committed,
                   COALESCE(AVG(probability), 0) as avg_probability
            FROM opportunities
            GROUP BY stage
            ORDER BY CASE stage
                WHEN 'lead' THEN 1 WHEN 'outreach' THEN 2 WHEN 'meeting' THEN 3
                WHEN 'due_diligence' THEN 4 WHEN 'committed' THEN 5 WHEN 'funded' THEN 6
            END
        """).fetchall())

        by_owner = rows_to_list(conn.execute("""
            SELECT u.full_name as owner, op.stage, COUNT(*) as count,
                   COALESCE(SUM(op.expected_amount), 0) as total_expected
            FROM opportunities op
            LEFT JOIN users u ON op.owner_id = u.id
            GROUP BY op.owner_id, op.stage
            ORDER BY u.full_name, op.stage
        """).fetchall())

        by_priority = rows_to_list(conn.execute("""
            SELECT priority, COUNT(*) as count,
                   COALESCE(SUM(expected_amount), 0) as total_expected
            FROM opportunities
            WHERE stage NOT IN ('funded', 'lost')
            GROUP BY priority
        """).fetchall())

        conn.close()
        return self.send_json({
            "data": {
                "by_stage": stages,
                "by_owner": by_owner,
                "by_priority": by_priority
            }
        })

    def handle_activity_report(self, user, params):
        conn = get_db()
        days = int(params.get('days', 30))

        by_user = rows_to_list(conn.execute("""
            SELECT u.full_name, cm.type, COUNT(*) as count
            FROM communications cm
            LEFT JOIN users u ON cm.created_by = u.id
            WHERE cm.communication_date >= date('now', ?)
            GROUP BY cm.created_by, cm.type
            ORDER BY u.full_name, cm.type
        """, (f'-{days} days',)).fetchall())

        by_day = rows_to_list(conn.execute("""
            SELECT date(communication_date) as date, type, COUNT(*) as count
            FROM communications
            WHERE communication_date >= date('now', ?)
            GROUP BY date(communication_date), type
            ORDER BY date DESC
        """, (f'-{days} days',)).fetchall())

        conn.close()
        return self.send_json({
            "data": {
                "by_user": by_user,
                "by_day": by_day,
                "period_days": days
            }
        })

    # ═══════════════════════════════════════════════════════════════════════════
    # IMPORT / EXPORT HANDLERS
    # ═══════════════════════════════════════════════════════════════════════════

    def handle_import_csv(self, user, body):
        """Import contacts from CSV data (sent as JSON array or raw CSV text)."""
        csv_data = body.get('data', [])
        entity_type = body.get('entity_type', 'contacts')
        mapping = body.get('mapping', {})
        dry_run = body.get('dry_run', False)
        update_existing = bool(body.get('update_existing', True))
        action_overrides_raw = body.get('action_overrides', {}) or {}

        if not csv_data:
            return self.send_error_json("No data provided. Send 'data' as array of objects.")

        conn = get_db()
        results = {"created": 0, "updated": 0, "skipped": 0, "errors": [], "matches": []}
        # Keep in-memory email matches so dry-run mirrors real behavior for
        # duplicate emails appearing multiple times in the same CSV batch.
        batch_email_matches = {}

        try:
            for i, row in enumerate(csv_data):
                try:
                    # Apply field mapping
                    mapped = {}
                    for csv_col, crm_field in mapping.items():
                        if csv_col in row:
                            mapped[crm_field] = row[csv_col]

                    # Use mapped data, fall back to raw row
                    data = mapped if mapping else row

                    if entity_type == 'contacts':
                        first_name = data.get('first_name', '').strip()
                        last_name = data.get('last_name', '').strip()

                        # Try splitting a 'name' field
                        if not first_name and not last_name and data.get('name'):
                            parts = data['name'].strip().split(' ', 1)
                            first_name = parts[0]
                            last_name = parts[1] if len(parts) > 1 else ''

                        if not first_name:
                            results['errors'].append(f"Row {i+1}: Missing first_name")
                            results['skipped'] += 1
                            continue

                        email = data.get('email', '').strip()
                        email_key = email.lower()
                        linkedin_url = data.get('linkedin_url', data.get('linkedin', '')).strip()
                        city = data.get('city', '').strip()
                        state = data.get('state', '').strip()
                        country = data.get('country', '').strip()
                        location_query = data.get('location_query', '').strip()
                        raw_location = data.get('location', data.get('city_location', data.get('city/location', ''))).strip()
                        if raw_location:
                            p_city, p_state, p_country, p_query = _parse_location_text(raw_location)
                            city = city or p_city
                            state = state or p_state
                            country = country or p_country
                            location_query = location_query or p_query

                        # Check for existing contact by email
                        existing = None
                        existing_summary = None
                        if email:
                            if email_key in batch_email_matches:
                                existing_summary = batch_email_matches[email_key]
                                existing = {"id": existing_summary.get('id')}
                            else:
                                existing = conn.execute("""
                                    SELECT c.id, c.first_name, c.last_name, c.email, o.name as organization_name
                                    FROM contacts c
                                    LEFT JOIN organizations o ON c.organization_id = o.id
                                    WHERE lower(c.email) = lower(?)
                                    ORDER BY c.updated_at DESC
                                    LIMIT 1
                                """, (email,)).fetchone()
                                if existing:
                                    existing_summary = {
                                        "id": existing['id'],
                                        "name": f"{str(existing['first_name'] or '').strip()} {str(existing['last_name'] or '').strip()}".strip(),
                                        "email": str(existing['email'] or ''),
                                        "organization": str(existing['organization_name'] or '')
                                    }
                                    batch_email_matches[email_key] = existing_summary

                        # Handle organization
                        org_id = None
                        org_name = data.get('organization', data.get('company', '')).strip()
                        if org_name:
                            org = conn.execute("SELECT id FROM organizations WHERE name = ?", (org_name,)).fetchone()
                            if org:
                                org_id = org['id']
                            elif not dry_run:
                                org_id = generate_id()
                                conn.execute(
                                    "INSERT INTO organizations (id, name, created_by) VALUES (?, ?, ?)",
                                    (org_id, org_name, user['user_id'])
                                )

                        action_override = None
                        if isinstance(action_overrides_raw, dict):
                            action_override = action_overrides_raw.get(str(i + 1)) or action_overrides_raw.get(i + 1)
                        default_action = 'update' if update_existing else 'skip'
                        action = action_override if action_override in ('update', 'skip', 'create_duplicate') else default_action
                        if existing:
                            incoming_name = f"{first_name} {last_name}".strip()
                            results['matches'].append({
                                "row": i + 1,
                                "incoming_name": incoming_name,
                                "incoming_email": email,
                                "incoming_organization": org_name,
                                "existing_id": existing_summary.get('id') if isinstance(existing_summary, dict) else existing['id'],
                                "existing_name": existing_summary.get('name') if isinstance(existing_summary, dict) else '',
                                "existing_email": existing_summary.get('email') if isinstance(existing_summary, dict) else email,
                                "existing_organization": existing_summary.get('organization') if isinstance(existing_summary, dict) else '',
                                "default_action": default_action,
                                "action": action
                            })

                        if not dry_run:
                            if existing:
                                if action == 'update':
                                    conn.execute("""
                                        UPDATE contacts SET first_name=?, last_name=?, phone=?, title=?,
                                        organization_id=COALESCE(?, organization_id),
                                        contact_type=COALESCE(?, contact_type),
                                        linkedin_url=COALESCE(?, linkedin_url),
                                        city=COALESCE(?, city),
                                        state=COALESCE(?, state),
                                        country=COALESCE(?, country),
                                        location_query=COALESCE(?, location_query),
                                        updated_at=?
                                        WHERE id=?
                                    """, (first_name, last_name, data.get('phone'),
                                          data.get('title'), org_id,
                                          data.get('contact_type'),
                                          linkedin_url if linkedin_url else None,
                                          city if city else None,
                                          state if state else None,
                                          country if country else None,
                                          location_query if location_query else None,
                                      now(), existing['id']))
                                    if email:
                                        batch_email_matches[email_key] = {
                                            "id": existing['id'],
                                            "name": f"{first_name} {last_name}".strip(),
                                            "email": email,
                                            "organization": org_name
                                        }
                                    updated_contact = row_to_dict(conn.execute("""
                                        SELECT c.*, o.name as organization_name
                                        FROM contacts c LEFT JOIN organizations o ON c.organization_id = o.id
                                        WHERE c.id = ?
                                    """, (existing['id'],)).fetchone())
                                    _sync_contact_to_fundraising_state(conn, updated_contact, actor_user_id=user['user_id'], remove=False)
                                    results['updated'] += 1
                                elif action == 'create_duplicate':
                                    contact_id = generate_id()
                                    conn.execute("""
                                        INSERT INTO contacts (id, first_name, last_name, email, phone,
                                            title, organization_id, contact_type, status, source,
                                            linkedin_url, city, state, country, location_query, created_by)
                                        VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'active', 'import', ?, ?, ?, ?, ?, ?)
                                    """, (contact_id, first_name, last_name, email,
                                          data.get('phone'), data.get('title'), org_id,
                                          data.get('contact_type', 'prospect'), linkedin_url,
                                          city, state, country, location_query, user['user_id']))
                                    if email:
                                        batch_email_matches[email_key] = {
                                            "id": contact_id,
                                            "name": f"{first_name} {last_name}".strip(),
                                            "email": email,
                                            "organization": org_name
                                        }
                                    created_contact = row_to_dict(conn.execute("""
                                        SELECT c.*, o.name as organization_name
                                        FROM contacts c LEFT JOIN organizations o ON c.organization_id = o.id
                                        WHERE c.id = ?
                                    """, (contact_id,)).fetchone())
                                    _sync_contact_to_fundraising_state(conn, created_contact, actor_user_id=user['user_id'], remove=False)
                                    results['created'] += 1
                                else:
                                    results['skipped'] += 1
                                    results['errors'].append(f"Row {i+1}: Existing contact matched by email; skipped")
                            else:
                                contact_id = generate_id()
                                conn.execute("""
                                    INSERT INTO contacts (id, first_name, last_name, email, phone,
                                        title, organization_id, contact_type, status, source,
                                        linkedin_url, city, state, country, location_query, created_by)
                                    VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'active', 'import', ?, ?, ?, ?, ?, ?)
                                """, (contact_id, first_name, last_name, email,
                                      data.get('phone'), data.get('title'), org_id,
                                      data.get('contact_type', 'prospect'), linkedin_url,
                                      city, state, country, location_query, user['user_id']))
                                if email:
                                    batch_email_matches[email_key] = {
                                        "id": contact_id,
                                        "name": f"{first_name} {last_name}".strip(),
                                        "email": email,
                                        "organization": org_name
                                    }
                                created_contact = row_to_dict(conn.execute("""
                                    SELECT c.*, o.name as organization_name
                                    FROM contacts c LEFT JOIN organizations o ON c.organization_id = o.id
                                    WHERE c.id = ?
                                """, (contact_id,)).fetchone())
                                _sync_contact_to_fundraising_state(conn, created_contact, actor_user_id=user['user_id'], remove=False)
                                results['created'] += 1
                        else:
                            if existing:
                                if action == 'update':
                                    results['updated'] += 1
                                    if email:
                                        batch_email_matches[email_key] = {
                                            "id": existing['id'],
                                            "name": f"{first_name} {last_name}".strip(),
                                            "email": email,
                                            "organization": org_name
                                        }
                                elif action == 'create_duplicate':
                                    results['created'] += 1
                                    if email:
                                        batch_email_matches[email_key] = {
                                            "id": f"dryrun-{i+1}",
                                            "name": f"{first_name} {last_name}".strip(),
                                            "email": email,
                                            "organization": org_name
                                        }
                                else:
                                    results['skipped'] += 1
                                    results['errors'].append(f"Row {i+1}: Existing contact matched by email; would be skipped")
                            else:
                                results['created'] += 1
                                if email:
                                    # Simulate that the row now exists for subsequent duplicate-email rows.
                                    batch_email_matches[email_key] = {
                                        "id": f"dryrun-{i+1}",
                                        "name": f"{first_name} {last_name}".strip(),
                                        "email": email,
                                        "organization": org_name
                                    }

                except Exception as e:
                    results['errors'].append(f"Row {i+1}: {str(e)}")
                    results['skipped'] += 1

            if not dry_run:
                conn.commit()

        except Exception as e:
            conn.rollback()
            conn.close()
            return self.send_error_json(f"Import failed: {str(e)}", 500)

        conn.close()
        return self.send_json({
            "data": results,
            "dry_run": dry_run
        })

    def handle_export_contacts(self, user, params):
        conn = get_db()
        contacts = rows_to_list(conn.execute("""
            SELECT c.*, o.name as organization_name
            FROM contacts c
            LEFT JOIN organizations o ON c.organization_id = o.id
            ORDER BY c.last_name, c.first_name
        """).fetchall())
        conn.close()
        return self.send_json({"data": contacts})

    # ═══════════════════════════════════════════════════════════════════════════
    # TAGS / USERS / AUDIT
    # ═══════════════════════════════════════════════════════════════════════════

    def handle_list_tags(self, user):
        conn = get_db()
        tags = rows_to_list(conn.execute("SELECT * FROM tags ORDER BY name").fetchall())
        conn.close()
        return self.send_json({"data": tags})

    def handle_create_tag(self, user, body):
        if not body.get('name'):
            return self.send_error_json("name is required")
        conn = get_db()
        tag_id = generate_id()
        try:
            conn.execute("INSERT INTO tags (id, name, color) VALUES (?, ?, ?)",
                         (tag_id, body['name'], body.get('color', '#6366f1')))
            conn.commit()
        except sqlite3.IntegrityError:
            conn.close()
            return self.send_error_json("Tag already exists")
        tag = row_to_dict(conn.execute("SELECT * FROM tags WHERE id = ?", (tag_id,)).fetchone())
        conn.close()
        return self.send_json({"data": tag}, 201)

    def handle_delete_tag(self, user, tag_id):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)
        conn = get_db()
        existing = conn.execute("SELECT id FROM tags WHERE id = ?", (tag_id,)).fetchone()
        if not existing:
            conn.close()
            return self.send_error_json("Tag not found", 404)
        conn.execute("DELETE FROM tags WHERE id = ?", (tag_id,))
        log_audit(conn, user['user_id'], 'tag', tag_id, 'delete')
        conn.commit()
        conn.close()
        return self.send_json({"message": "Tag deleted"})

    def handle_system_status(self, user):
        """System / search-index health for the in-app status view (DB-derived)."""
        conn = get_db()
        out = {}
        try:
            live = "deleted_at IS NULL"
            out['canonical_entities'] = {
                'investor': conn.execute(f"SELECT COUNT(*) FROM canonical_entities WHERE entity_kind='investor' AND {live}").fetchone()[0],
                'person': conn.execute(f"SELECT COUNT(*) FROM canonical_entities WHERE entity_kind='person' AND {live}").fetchone()[0],
            }
            out['entity_links'] = conn.execute("SELECT COUNT(*) FROM entity_links").fetchone()[0]
        except Exception:
            out['canonical_entities'] = None
        try:
            r = conn.execute("SELECT ts, payload FROM interaction_log WHERE action='ingest.sync' ORDER BY ts DESC LIMIT 1").fetchone()
            out['last_index_sync'] = ({'ts': r['ts'], **json.loads(r['payload'] or '{}')} if r else None)
        except Exception:
            out['last_index_sync'] = None
        try:
            out['thesis'] = {
                'lines': conn.execute("SELECT COUNT(*) FROM thesis_lines WHERE deleted_at IS NULL").fetchone()[0],
                'canonical_versions': conn.execute("SELECT COUNT(*) FROM thesis_versions WHERE status='canonical'").fetchone()[0],
                'in_review': conn.execute("SELECT COUNT(*) FROM thesis_versions WHERE status='in_review'").fetchone()[0],
            }
        except Exception:
            out['thesis'] = None
        try:
            out['recent_activity'] = [dict(r) for r in conn.execute(
                "SELECT ts, actor_type, actor_id, action FROM interaction_log ORDER BY ts DESC LIMIT 12")]
        except Exception:
            out['recent_activity'] = []
        try:
            # Count only candidates whose both sides are still live (mirror the
            # review queue in entity_merge.list_candidates) — self-healed twins
            # whose duplicate was soft-deleted no longer count as pending work.
            out['pending_merge_candidates'] = conn.execute(
                """SELECT COUNT(*) FROM entity_merge_candidates mc
                     JOIN canonical_entities a ON a.id=mc.entity_a AND a.deleted_at IS NULL
                     JOIN canonical_entities b ON b.id=mc.entity_b AND b.deleted_at IS NULL
                    WHERE mc.status='pending'""").fetchone()[0]
        except Exception:
            out['pending_merge_candidates'] = None
        out['index_job'] = entity_jobs.get_status() if entity_jobs else None
        # Raw source-record counts, so the resolved canonical numbers can be
        # sanity-checked against what's actually in the CRM.
        try:
            out['source_counts'] = {
                'contacts': conn.execute("SELECT COUNT(*) FROM contacts WHERE deleted_at IS NULL").fetchone()[0],
                'organizations': conn.execute("SELECT COUNT(*) FROM organizations WHERE deleted_at IS NULL").fetchone()[0],
                'fundraising_investors': conn.execute("SELECT COUNT(*) FROM fundraising_investors").fetchone()[0],
                'fundraising_contacts': conn.execute("SELECT COUNT(*) FROM fundraising_contacts").fetchone()[0],
            }
        except Exception:
            out['source_counts'] = None
        # Storage usage — DB file(s), email attachments, backups, and disk headroom,
        # so growth can be watched over time. Best-effort; never fails the status call.
        try:
            import shutil

            def _fsize(p):
                try:
                    return os.path.getsize(p)
                except OSError:
                    return 0

            def _dirsize(d):
                total = 0
                for root, _dirs, files in os.walk(d):
                    for f in files:
                        try:
                            total += os.path.getsize(os.path.join(root, f))
                        except OSError:
                            pass
                return total

            du = shutil.disk_usage(DATA_DIR)
            out['storage'] = {
                'database_bytes': sum(_fsize(DB_PATH + s) for s in ("", "-wal", "-shm")),
                'attachments_bytes': _dirsize(os.path.join(DATA_DIR, "email_attachments")),
                'backups_bytes': _dirsize(os.path.join(DATA_DIR, "backups")),
                'disk_total_bytes': du.total,
                'disk_used_bytes': du.used,
                'disk_free_bytes': du.free,
            }
        except Exception:
            out['storage'] = None
        conn.close()
        self.send_json({"data": out})

    def handle_list_activity_proposals(self, user):
        if not require_admin(user):
            return self.send_error_json("Admin required", 403)
        conn = get_db()
        try:
            return self.send_json({"proposals": list_email_activity_proposals(conn, status="pending")})
        finally:
            conn.close()

    def handle_decide_activity_proposal(self, user, proposal_id, decision, body):
        if not require_admin(user):
            return self.send_error_json("Admin required", 403)
        conn = get_db()
        try:
            res = decide_email_activity_proposal(conn, proposal_id, decision,
                                                 user['user_id'], (body or {}).get('note'))
        finally:
            conn.close()
        if res.get("error"):
            code = {"not_found": 404, "already_decided": 409}.get(res["error"], 400)
            return self.send_error_json(res["error"], code)
        return self.send_json({"data": res})

    # ─── UI-triggered index jobs + entity-merge review (Phase 1) ───
    def handle_index_job(self, user, kind):
        if not require_admin(user):
            return self.send_error_json("Admin required", 403)
        if entity_jobs is None:
            return self.send_error_json("Jobs unavailable", 503)
        res = entity_jobs.start(kind, DB_PATH)
        if res.get('error'):
            return self.send_error_json(res['error'], 409)
        return self.send_json({"data": res})

    def handle_list_merge_candidates(self, user, params):
        if not require_admin(user):
            return self.send_error_json("Admin required", 403)
        if entity_merge is None:
            return self.send_error_json("Unavailable", 503)
        return self.send_json(entity_merge.list_candidates(DB_PATH, params.get('status', 'pending')))

    def handle_decide_merge_candidate(self, user, candidate_id, body):
        if not require_admin(user):
            return self.send_error_json("Admin required", 403)
        if entity_merge is None:
            return self.send_error_json("Unavailable", 503)
        res = entity_merge.decide(DB_PATH, candidate_id, (body or {}).get('decision'), user['user_id'])
        if res.get('error'):
            return self.send_error_json(res['error'], 400)
        return self.send_json({"data": res})

    # ─── Architect thesis authoring (Phase 1) ───
    def handle_get_thesis_tree(self, user, line_key):
        if _architect_tools is None:
            return self.send_error_json("Unavailable", 503)
        return self.send_json(_architect_tools.get_thesis(line_key, DB_PATH))

    def handle_create_thesis_line(self, user, body):
        if not require_admin(user):
            return self.send_error_json("Admin required", 403)
        if _architect_tools is None:
            return self.send_error_json("Unavailable", 503)
        body = body or {}
        if not body.get('line_key') or not body.get('name'):
            return self.send_error_json("line_key and name required", 400)
        return self.send_json({"data": _architect_tools.create_thesis_line(
            body['line_key'], body['name'], segment_key=body.get('segment_key'),
            is_core=bool(body.get('is_core')), description=body.get('description'), db=DB_PATH)})

    def handle_add_thesis_node(self, user, line_key, body):
        if not require_admin(user):
            return self.send_error_json("Admin required", 403)
        if _architect_tools is None:
            return self.send_error_json("Unavailable", 503)
        conn = get_db()
        row = conn.execute("SELECT id FROM thesis_lines WHERE line_key=? AND deleted_at IS NULL", (line_key,)).fetchone()
        conn.close()
        if not row:
            return self.send_error_json("Line not found", 404)
        body = body or {}
        return self.send_json({"data": _architect_tools.upsert_thesis_node(
            row['id'], body.get('node_type', 'claim'), body.get('body', ''), title=body.get('title'),
            parent_id=body.get('parent_id'), node_id=body.get('node_id'),
            variant_group=body.get('variant_group'), change_reason=body.get('change_reason'), db=DB_PATH)})

    # ─── Architect agent (Phase 1, runs on Claude) ───
    def handle_architect_status(self, user):
        if _architect_agent is None:
            return self.send_error_json("Unavailable", 503)
        return self.send_json({"data": _architect_agent.status()})

    def handle_get_node_variants(self, user, node_id):
        if _architect_tools is None:
            return self.send_error_json("Unavailable", 503)
        return self.send_json(_architect_tools.get_node_variants(node_id, DB_PATH))

    def _architect_line_key(self, node_id):
        conn = get_db()
        row = conn.execute("SELECT l.line_key FROM thesis_nodes n JOIN thesis_lines l ON l.id=n.line_id WHERE n.id=?",
                           (node_id,)).fetchone()
        conn.close()
        return row['line_key'] if row else None

    def handle_edit_thesis_node(self, user, node_id, body):
        """Manual human edit of a node's title/body (no Architect)."""
        if not require_admin(user):
            return self.send_error_json("Admin required", 403)
        if _architect_tools is None:
            return self.send_error_json("Unavailable", 503)
        node = _architect_tools.get_node(node_id, db=DB_PATH)
        if node.get('error'):
            return self.send_error_json("Node not found", 404)
        body = body or {}
        res = _architect_tools.upsert_thesis_node(
            node['line_id'], node['node_type'], body.get('body', node.get('body') or ''),
            title=body.get('title', node.get('title')), node_id=node_id,
            change_reason='manual edit', actor_id=user['user_id'], actor_type='human', db=DB_PATH)
        return self.send_json({"data": res})

    def handle_retire_thesis_node(self, user, node_id):
        """Soft-delete a node + its subtree."""
        if not require_admin(user):
            return self.send_error_json("Admin required", 403)
        if _architect_tools is None:
            return self.send_error_json("Unavailable", 503)
        res = _architect_tools.retire_node(node_id, actor_id=user['user_id'], db=DB_PATH)
        if res.get('error'):
            return self.send_error_json("Node not found", 404)
        return self.send_json({"data": res})

    def handle_choose_variant(self, user, node_id):
        """'Use this option' — keep this variant, retire its siblings."""
        if not require_admin(user):
            return self.send_error_json("Admin required", 403)
        if _architect_tools is None:
            return self.send_error_json("Unavailable", 503)
        res = _architect_tools.choose_variant(node_id, actor_id=user['user_id'], db=DB_PATH)
        if res.get('error'):
            return self.send_error_json("Node not found", 404)
        return self.send_json({"data": res})

    def handle_approve_line(self, user, line_key):
        """One-click 'approve as current': record this admin's approval on the line's
        in-review version (creating + submitting one from the live tree if none exists);
        promotes to canonical once the required distinct approvals are reached."""
        if not require_admin(user):
            return self.send_error_json("Admin required", 403)
        if _architect_tools is None or thesis_review is None:
            return self.send_error_json("Unavailable", 503)
        conn = get_db()
        line = conn.execute("SELECT id FROM thesis_lines WHERE line_key=? AND deleted_at IS NULL", (line_key,)).fetchone()
        v = None
        if line:
            v = conn.execute("SELECT id FROM thesis_versions WHERE line_id=? AND status='in_review' ORDER BY version_no DESC LIMIT 1",
                             (line['id'],)).fetchone()
        conn.close()
        if not line:
            return self.send_error_json("Line not found", 404)
        if v:
            version_id = v['id']
        else:
            ver = _architect_tools.create_thesis_version(line_key, rationale="Approved as current (Workshop)",
                                                         created_by=user['user_id'], db=DB_PATH)
            if ver.get('error'):
                return self.send_error_json(ver['error'], 400)
            _architect_tools.submit_version_for_review(ver['id'], db=DB_PATH)
            version_id = ver['id']
        res = thesis_review.record_review(DB_PATH, version_id, user['user_id'], 'approve')
        return self.send_json({"data": res})

    def handle_generate_options(self, user, node_id, body):
        if not require_admin(user):
            return self.send_error_json("Admin required", 403)
        if _architect_agent is None:
            return self.send_error_json("Unavailable", 503)
        lk = self._architect_line_key(node_id)
        if not lk:
            return self.send_error_json("Node not found", 404)
        body = body or {}
        try:
            res = _architect_agent.generate_options(lk, node_id, int(body.get('n', 3) or 3),
                                                    body.get('guidance', '') or '', DB_PATH)
        except Exception as exc:
            return self.send_error_json(str(exc), 502)
        if res.get('error'):
            return self.send_error_json(res.get('raw') or res['error'], 502)
        return self.send_json({"data": res})

    def handle_node_feedback(self, user, node_id, body):
        if not require_admin(user):
            return self.send_error_json("Admin required", 403)
        if _architect_agent is None:
            return self.send_error_json("Unavailable", 503)
        lk = self._architect_line_key(node_id)
        if not lk:
            return self.send_error_json("Node not found", 404)
        body = body or {}
        try:
            res = _architect_agent.revise(lk, node_id, body.get('feedback', '') or '',
                                          int(body.get('n', 2) or 2), DB_PATH)
        except Exception as exc:
            return self.send_error_json(str(exc), 502)
        if res.get('error'):
            return self.send_error_json(res.get('raw') or res['error'], 502)
        return self.send_json({"data": res})

    def _ground_feedback_corpus(self, conn, limit=60):
        """Raw LP-feedback prose for grounding, newest-first, balanced across sources:
        matched email bodies (the richest objection signal), logged communications, and
        fundraising grid notes. Sensitive Tier-2-heavy text; ONLY ever passed into the
        redaction boundary, never to Claude directly."""
        # Email bodies are capped per item (long threads/quote-chains) to keep the local
        # minimize tractable; only `matched` emails (tied to a known investor/contact) are
        # pulled. Sources are round-robin merged so email is always represented even when
        # communications/notes are plentiful, rather than crowded out by a flat LIMIT.
        sources = (
            "SELECT SUBSTR(body_text,1,4000) FROM emails WHERE match_status='matched' "
            "AND body_text IS NOT NULL AND TRIM(body_text)<>'' ORDER BY sent_at DESC LIMIT ?",
            "SELECT body FROM communications WHERE body IS NOT NULL AND TRIM(body)<>'' "
            "ORDER BY communication_date DESC LIMIT ?",
            "SELECT notes FROM fundraising_investors WHERE notes IS NOT NULL AND TRIM(notes)<>'' LIMIT ?",
        )
        buckets = []
        for q in sources:
            try:
                buckets.append([r[0] for r in conn.execute(q, (limit,))])
            except Exception:
                buckets.append([])   # table absent (e.g. email integration not migrated) -> skip
        items, i = [], 0
        while len(items) < limit and any(i < len(b) for b in buckets):
            for b in buckets:
                if i < len(b):
                    items.append(b[i])
                    if len(items) >= limit:
                        break
            i += 1
        return items

    def handle_architect_ground(self, user, body):
        """Ground an objection register in real LP feedback THROUGH the redaction boundary
        (Workstream D). Retrieval + minimization + scrub stay local; only the de-identified
        register reaches Claude; the re-hydrated draft is for human review (guardrail #4)."""
        if not require_admin(user):
            return self.send_error_json("Admin required", 403)
        if _architect_grounding is None:
            return self.send_error_json("Unavailable", 503)
        body = body or {}
        segment_key = body.get('segment_key')
        feedback = body.get('feedback_items')
        conn = get_db()
        try:
            if not feedback:
                feedback = self._ground_feedback_corpus(conn)
            if not feedback:
                return self.send_error_json("No LP feedback found to ground against", 404)
            res = _architect_grounding.ground_objections(feedback, segment_key=segment_key,
                                                         db_path=DB_PATH, conn=conn)
        except Exception as exc:
            return self.send_error_json(str(exc), 502)
        finally:
            conn.close()
        return self.send_json({"data": res})

    def handle_list_outreach_investors(self, user):
        conn = get_db()
        try:
            rows = conn.execute("SELECT id, investor_name FROM fundraising_investors "
                                "ORDER BY investor_name LIMIT 2000").fetchall()
            return self.send_json({"investors": [{"id": r["id"], "name": r["investor_name"]} for r in rows]})
        finally:
            conn.close()

    def handle_outreach_radar(self, user):
        """Deterministic 'who needs attention' scan (reasons are checkable, not LLM guesses)."""
        if _outreach_agent is None:
            return self.send_error_json("Outreach agent unavailable", 503)
        conn = get_db()
        try:
            try:
                own = [r[0] for r in conn.execute("SELECT email_address FROM email_accounts")]
            except Exception:
                own = []
            items = _outreach_agent.follow_up_radar(conn, own, now())
            return self.send_json({"items": items})
        finally:
            conn.close()

    def handle_outreach_draft(self, user, body):
        """Draft tailored LP outreach through the redaction boundary (draft-only —
        a human reviews/edits/sends; guardrails #4, #6)."""
        if _outreach_agent is None:
            return self.send_error_json("Outreach agent unavailable", 503)
        body = body or {}
        inv = body.get('investor_id')
        if not inv:
            return self.send_error_json("investor_id required", 400)
        conn = get_db()
        try:
            sender_email = None
            try:
                r = conn.execute("SELECT email FROM users WHERE id=?", (user.get('user_id'),)).fetchone()
                sender_email = r[0] if r else None
            except Exception:
                pass
            res = _outreach_agent.draft_outreach(conn, inv, body.get('outreach_type', 'follow_up'),
                                                 body.get('guidance', '') or '', DB_PATH, sender_email=sender_email)
            try:
                conn.execute(
                    "INSERT INTO interaction_log (id, ts, actor_type, actor_id, action, target_type, target_id, payload, source, created_at) "
                    "VALUES (?,?,?,?,?,?,?,?,?,?)",
                    (generate_id(), now(), "human", user.get('user_id'), "outreach.drafted",
                     "fundraising_investor", inv,
                     json.dumps({"type": body.get('outreach_type'), "status": res.get('status')}), "crm_ui", now()))
                conn.commit()
            except Exception:
                pass
        except Exception as exc:
            return self.send_error_json(str(exc), 502)
        finally:
            conn.close()
        return self.send_json({"data": res})

    def handle_outreach_gmail_draft(self, user, body):
        """Create a Gmail DRAFT from an approved outreach draft (in-thread reply when there
        is an active thread). Never sends — the human sends from Gmail (guardrails #4, #6)."""
        body = body or {}
        inv = body.get('investor_id')
        text = body.get('draft') or ''
        if not inv or not text.strip():
            return self.send_error_json("investor_id and draft required", 400)
        try:
            from email_integration import compose as _compose
        except Exception:
            return self.send_error_json("Gmail compose unavailable", 503)
        conn = get_db()
        try:
            sender_email = None
            try:
                r = conn.execute("SELECT email FROM users WHERE id=?", (user.get('user_id'),)).fetchone()
                sender_email = r[0] if r else None
            except Exception:
                pass
            res = _compose.create_outreach_draft(conn, sender_email, inv, text)
            try:
                conn.execute(
                    "INSERT INTO interaction_log (id, ts, actor_type, actor_id, action, target_type, target_id, payload, source, created_at) "
                    "VALUES (?,?,?,?,?,?,?,?,?,?)",
                    (generate_id(), now(), "human", user.get('user_id'), "outreach.gmail_draft_created",
                     "fundraising_investor", inv, json.dumps({"status": res.get('status')}), "crm_ui", now()))
                conn.commit()
            except Exception:
                pass
        finally:
            conn.close()
        return self.send_json({"data": res})

    # ─── Architect thesis (Phase 1) ───
    def handle_list_thesis_lines(self, user):
        if thesis_review is None:
            return self.send_error_json("Thesis module unavailable", 503)
        return self.send_json(thesis_review.list_lines(DB_PATH))

    def handle_list_thesis_review_queue(self, user):
        if thesis_review is None:
            return self.send_error_json("Thesis module unavailable", 503)
        return self.send_json(thesis_review.list_versions_for_review(DB_PATH))

    def handle_get_thesis_version(self, user, version_id):
        if thesis_review is None:
            return self.send_error_json("Thesis module unavailable", 503)
        return self.send_json(thesis_review.get_version(DB_PATH, version_id))

    def handle_get_canonical_thesis(self, user, line_key):
        if thesis_review is None:
            return self.send_error_json("Thesis module unavailable", 503)
        return self.send_json(thesis_review.get_canonical(DB_PATH, line_key))

    def handle_thesis_review(self, user, version_id, body):
        # Promotion to canonical is a human partner action (guardrail #4).
        if not require_admin(user):
            return self.send_error_json("Admin required", 403)
        if thesis_review is None:
            return self.send_error_json("Thesis module unavailable", 503)
        body = body or {}
        res = thesis_review.record_review(DB_PATH, version_id, user['user_id'],
                                          body.get('decision'), body.get('feedback'),
                                          body.get('target_node_id'))
        if res.get('error'):
            return self.send_error_json(res['error'], 400)
        return self.send_json({"data": res})

    def handle_list_users(self, user):
        # The full user directory (names, emails, roles) is admin-only — it is only
        # consumed by the admin section of Settings. The nav already hides it from
        # members; this enforces the same boundary server-side.
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)
        conn = get_db()
        users = rows_to_list(conn.execute(
            "SELECT id, username, email, full_name, role, is_active, created_at FROM users ORDER BY full_name"
        ).fetchall())
        conn.close()
        return self.send_json({"data": users})

    def handle_admin_create_user(self, user, body):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)

        required = ['username', 'password', 'email', 'full_name']
        for field in required:
            if not str(body.get(field, '')).strip():
                return self.send_error_json(f"{field} is required")
        if len(str(body.get('password') or '').strip()) < 8:
            return self.send_error_json("password must be at least 8 characters")

        role = body.get('role', 'member')
        if role not in ('admin', 'member'):
            return self.send_error_json("role must be admin or member")

        conn = get_db()
        existing = conn.execute(
            "SELECT id FROM users WHERE username = ? OR email = ?",
            (body['username'].strip(), body['email'].strip())
        ).fetchone()
        if existing:
            conn.close()
            return self.send_error_json("Username or email already exists")

        user_id = generate_id()
        conn.execute(
            "INSERT INTO users (id, username, email, password_hash, full_name, role) VALUES (?, ?, ?, ?, ?, ?)",
            (
                user_id,
                body['username'].strip(),
                body['email'].strip(),
                hash_password(body['password']),
                body['full_name'].strip(),
                role
            )
        )
        log_audit(conn, user['user_id'], 'user', user_id, 'create', {"username": body['username'].strip(), "role": role})
        conn.commit()
        created = row_to_dict(conn.execute(
            "SELECT id, username, email, full_name, role, is_active, created_at FROM users WHERE id = ?",
            (user_id,)
        ).fetchone())
        conn.close()
        return self.send_json({"data": created}, 201)

    def handle_admin_update_user(self, user, target_user_id, body):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)

        conn = get_db()
        existing = conn.execute(
            "SELECT id, username, full_name, role, is_active FROM users WHERE id = ?",
            (target_user_id,)
        ).fetchone()
        if not existing:
            conn.close()
            return self.send_error_json("User not found", 404)

        sets = []
        args = []

        if 'is_active' in body:
            next_active = 1 if bool(body.get('is_active')) else 0
            # prevent locking out the currently authenticated admin accidentally
            if existing['id'] == user.get('user_id') and next_active == 0:
                conn.close()
                return self.send_error_json("You cannot deactivate your own account", 400)
            sets.append("is_active = ?")
            args.append(next_active)

        if 'role' in body:
            role = str(body.get('role'))
            if role not in ('admin', 'member'):
                conn.close()
                return self.send_error_json("role must be admin or member")
            sets.append("role = ?")
            args.append(role)

        if 'password' in body:
            password = str(body.get('password') or '')
            if len(password.strip()) < 8:
                conn.close()
                return self.send_error_json("password must be at least 8 characters")
            sets.append("password_hash = ?")
            args.append(hash_password(password))

        if not sets:
            conn.close()
            return self.send_error_json("No fields to update")

        sets.append("updated_at = ?")
        args.append(now())
        args.append(target_user_id)
        conn.execute(f"UPDATE users SET {', '.join(sets)} WHERE id = ?", args)
        audit_payload = dict(body)
        if 'password' in audit_payload:
            audit_payload['password'] = '[REDACTED]'
        log_audit(conn, user['user_id'], 'user', target_user_id, 'update', audit_payload)
        conn.commit()
        updated = row_to_dict(conn.execute(
            "SELECT id, username, email, full_name, role, is_active, created_at FROM users WHERE id = ?",
            (target_user_id,)
        ).fetchone())
        conn.close()
        return self.send_json({"data": updated})

    def handle_admin_reset_all_data(self, user, body):
        if not require_admin(user):
            return self.send_error_json("Admin only", 403)

        confirm_phrase = str(body.get('confirm_phrase') or '').strip()
        if confirm_phrase != 'RESET ALL DATA':
            return self.send_error_json("Confirmation phrase must be exactly: RESET ALL DATA", 400)

        conn = get_db()
        try:
            ensure_fundraising_state_row(conn)
            state = conn.execute("SELECT * FROM fundraising_state WHERE id = 'main'").fetchone()
            pre_backup = create_fundraising_backup_file(state, kind="pre_restore") if state else None

            conn.execute("DELETE FROM communications")
            conn.execute("DELETE FROM opportunities")
            conn.execute("DELETE FROM lp_profiles")
            conn.execute("DELETE FROM custom_field_values")
            conn.execute("DELETE FROM custom_fields")
            conn.execute("DELETE FROM feature_requests")
            conn.execute("DELETE FROM contacts")
            conn.execute("DELETE FROM organizations")

            default_grid = {
                "columns": deep_copy_json(DEFAULT_FUNDRAISING_COLUMNS),
                "rows": deep_copy_json(DEFAULT_FUNDRAISING_ROWS)
            }
            default_views = sanitize_grid_views(deep_copy_json(DEFAULT_GRID_VIEWS))
            conn.execute("""
                UPDATE fundraising_state
                SET grid_json = ?, views_json = ?, version = COALESCE(version, 1) + 1, updated_by = ?, updated_at = ?
                WHERE id = 'main'
            """, (json.dumps(default_grid), json.dumps(default_views), user['user_id'], now()))
            sync_fundraising_relational(conn, default_grid, default_views, actor_user_id=user['user_id'])

            log_audit(conn, user['user_id'], 'system', 'all-data', 'reset', {
                "pre_backup": pre_backup['filename'] if pre_backup else None
            })
            conn.commit()
        except Exception as exc:
            conn.rollback()
            conn.close()
            return self.send_error_json(f"Failed to reset data: {str(exc)}", 500)

        conn.close()
        return self.send_json({
            "data": {
                "status": "ok",
                "pre_backup": pre_backup
            }
        })

    def handle_admin_send_test_email(self, user, body):
        """Send a test email through the active transport (Gmail DWD preferred,
        SMTP fallback) to prove the outbound pipe before the daily digest (Phase B)."""
        if not require_admin(user):
            return self.send_error_json("Admin only", 403)

        import digest_mailer

        conn = get_db()
        try:
            # Recipients are restricted to the active-admin set (the real digest
            # audience). An explicit `to` may NARROW to specific admins but can never
            # introduce an outside address — this endpoint is not an open mail relay.
            rows = conn.execute(
                "SELECT email FROM users WHERE role = 'admin' AND is_active = 1 "
                "AND email IS NOT NULL AND TRIM(email) != ''"
            ).fetchall()
            admin_emails = [str(r['email']).strip() for r in rows if str(r['email']).strip()]
            admin_lower = {e.lower() for e in admin_emails}

            to = body.get('to')
            if to:
                requested = [to] if isinstance(to, str) else list(to)
                requested = [str(e).strip() for e in requested if str(e).strip()]
                outside = [e for e in requested if e.lower() not in admin_lower]
                if outside:
                    return self.send_error_json(
                        "Test email may only go to an active admin address; "
                        f"not allowed: {', '.join(outside)}", 400)
                recipients = requested
            else:
                recipients = admin_emails
            if not recipients:
                return self.send_error_json(
                    "No recipient: give an active admin an email address first.", 400)

            subject = "Ten31 CRM — test digest email"
            email_body = (
                "This is a test message from the Ten31 CRM daily-digest mailer.\n\n"
                f"Triggered by {user.get('full_name') or user.get('user_id')} at {now()}.\n\n"
                "If you received this, outbound email works and the daily digest can "
                "be delivered to this address."
            )
            result = digest_mailer.send_digest(conn, recipients, subject, email_body)
        except digest_mailer.NoTransport as exc:
            return self.send_error_json(str(exc), 400)
        except Exception as exc:
            # Never echo the exception to the client — an auth error can carry a
            # credential or token. Log it server-side instead.
            print(f"[digest] test send failed: {type(exc).__name__}: {exc}", file=sys.stderr)
            return self.send_error_json("Send failed — see server logs for details.", 502)
        finally:
            conn.close()

        return self.send_json({"data": {"status": "sent", **result}})

    def handle_admin_digest_preview(self, user, body):
        """Build the activity digest over a chosen window and return it WITHOUT
        sending — the admin-panel preview. Window defaults to the last 24h, or
        {hours: N} / {since: 'YYYY-MM-DD'} (a local date -> that day's midnight).
        Runs the REAL Spark summarization, so widening the window is how you verify
        the summarizer on a quiet day. Never touches the daily cursor."""
        if not require_admin(user):
            return self.send_error_json("Admin only", 403)
        body = body or {}
        import digest_builder
        try:
            since_iso, until_iso = digest_builder.resolve_digest_window(
                hours=body.get('hours'), since=body.get('since'))
        except ValueError as exc:
            return self.send_error_json(str(exc), 400)
        conn = get_db()
        try:
            digest = digest_builder.build_digest(conn, since_iso, until_iso)
        except Exception as exc:
            print(f"[digest] preview failed: {type(exc).__name__}: {exc}", file=sys.stderr)
            return self.send_error_json("Preview failed — see server logs for details.", 502)
        finally:
            conn.close()
        return self.send_json({"data": {**digest, "window": [since_iso, until_iso]}})

    def handle_admin_send_digest_now(self, user, body):
        """Build the REAL activity digest over a chosen window and send it to the
        active-admin set now. Window defaults to the last 24h, or {hours: N} /
        {since: 'YYYY-MM-DD'} — same resolution as the preview. Does NOT touch the
        daily schedule's cursor, so it never suppresses the scheduled send. Content
        is summarized on Spark (local), never Claude."""
        if not require_admin(user):
            return self.send_error_json("Admin only", 403)
        body = body or {}
        import digest_mailer
        import digest_builder
        try:
            since_iso, until_iso = digest_builder.resolve_digest_window(
                hours=body.get('hours'), since=body.get('since'))
        except ValueError as exc:
            return self.send_error_json(str(exc), 400)
        try:
            from email_integration.digest_scheduler import send_digest_window
            result = send_digest_window(since_iso=since_iso, until_iso=until_iso)
        except digest_mailer.NoTransport as exc:
            return self.send_error_json(str(exc), 400)
        except Exception as exc:
            # Never echo the exception — an auth error can carry a token/credential.
            print(f"[digest] send-now failed: {type(exc).__name__}: {exc}", file=sys.stderr)
            return self.send_error_json("Send failed — see server logs for details.", 502)

        return self.send_json({"data": result})

    def handle_get_digest_policy(self, user):
        """Return the live daily-digest policy (enabled + send hour). DB-backed
        (app_settings), set from this same panel — see digest_builder.load_digest_policy."""
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)
        import digest_builder
        conn = get_db()
        try:
            return self.send_json({"data": digest_builder.load_digest_policy(conn)})
        finally:
            conn.close()

    def handle_update_digest_policy(self, user, body):
        """Update the daily-digest policy. Takes effect on the scheduler's next
        cycle (no restart). Recipients stay the active-admin set; sender/transport
        are env/StartOS config, not toggled here."""
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)
        import digest_builder
        conn = get_db()
        try:
            policy = digest_builder.load_digest_policy(conn)
            if 'enabled' in body:
                policy['enabled'] = bool(body.get('enabled'))
            if 'send_hour' in body:
                try:
                    policy['send_hour'] = max(0, min(23, int(body.get('send_hour'))))
                except (ValueError, TypeError):
                    return self.send_error_json("send_hour must be an integer from 0 to 23")
            normalized = {"enabled": bool(policy['enabled']), "send_hour": int(policy['send_hour'])}
            set_app_setting(conn, digest_builder.DIGEST_POLICY_KEY, normalized)
            conn.commit()
            return self.send_json({"data": normalized})
        finally:
            conn.close()

    def handle_list_audit_log(self, user, params):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)
        conn = get_db()
        query = """
            SELECT al.*, u.full_name as user_name
            FROM audit_log al LEFT JOIN users u ON al.user_id = u.id
            WHERE 1=1
        """
        args = []
        if params.get('entity_type'):
            query += " AND al.entity_type = ?"
            args.append(params['entity_type'])
        if params.get('entity_id'):
            query += " AND al.entity_id = ?"
            args.append(params['entity_id'])
        if params.get('action'):
            query += " AND al.action = ?"
            args.append(params['action'])
        if params.get('user_id'):
            query += " AND al.user_id = ?"
            args.append(params['user_id'])
        if params.get('date_from'):
            query += " AND al.created_at >= ?"
            args.append(params['date_from'])
        if params.get('date_to'):
            query += " AND al.created_at <= ?"
            args.append(params['date_to'])
        if params.get('search'):
            search = f"%{str(params.get('search')).strip()}%"
            query += " AND (al.entity_id LIKE ? OR al.entity_type LIKE ? OR al.action LIKE ? OR al.changes LIKE ?)"
            args.extend([search, search, search, search])

        try:
            limit = max(1, min(500, int(params.get('limit') or 100)))
        except Exception:
            limit = 100
        query += " ORDER BY al.created_at DESC LIMIT ?"
        args.append(limit)
        logs = rows_to_list(conn.execute(query, args).fetchall())
        conn.close()
        return self.send_json({"data": logs})

    def handle_get_backup_policy(self, user):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)
        conn = get_db()
        policy = load_backup_policy(conn)
        conn.close()
        last_run_dt = parse_iso_utc(policy.get('last_run_at'))
        next_run_at = None
        if policy.get('enabled'):
            if last_run_dt:
                next_run_at = (last_run_dt.replace(tzinfo=None) + timedelta(hours=int(policy.get('interval_hours') or 24))).isoformat() + "Z"
            else:
                next_run_at = now()
        return self.send_json({"data": {**policy, "next_run_at": next_run_at}})

    def handle_update_backup_policy(self, user, body):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)
        conn = get_db()
        policy = load_backup_policy(conn)
        if 'enabled' in body:
            policy['enabled'] = bool(body.get('enabled'))
        if 'interval_hours' in body:
            try:
                policy['interval_hours'] = max(1, min(168, int(body.get('interval_hours'))))
            except Exception:
                conn.close()
                return self.send_error_json("interval_hours must be an integer from 1 to 168")
        if 'retention_days' in body:
            try:
                policy['retention_days'] = max(1, min(365, int(body.get('retention_days'))))
            except Exception:
                conn.close()
                return self.send_error_json("retention_days must be an integer from 1 to 365")
        if 'max_backups' in body:
            try:
                policy['max_backups'] = max(1, min(1000, int(body.get('max_backups'))))
            except Exception:
                conn.close()
                return self.send_error_json("max_backups must be an integer from 1 to 1000")
        updated = save_backup_policy(conn, policy)
        apply_backup_retention(updated)
        conn.commit()
        conn.close()
        return self.send_json({"data": updated})

    def handle_get_fundraising_relational_summary(self, user):
        conn = get_db()
        counts = {
            "investors": conn.execute("SELECT COUNT(*) AS c FROM fundraising_investors").fetchone()['c'],
            "contacts": conn.execute("SELECT COUNT(*) AS c FROM fundraising_contacts").fetchone()['c'],
            "funds": conn.execute("SELECT COUNT(*) AS c FROM fundraising_funds WHERE active = 1").fetchone()['c'],
            "commitments": conn.execute("SELECT COUNT(*) AS c FROM fundraising_commitments").fetchone()['c'],
            "views": conn.execute("SELECT COUNT(*) AS c FROM fundraising_views").fetchone()['c']
        }
        totals = conn.execute("""
            SELECT
                COALESCE(SUM(total_invested), 0) AS total_invested,
                SUM(CASE WHEN graveyard = 1 THEN 1 ELSE 0 END) AS graveyard_count,
                SUM(CASE WHEN follow_up = 1 THEN 1 ELSE 0 END) AS follow_up_count
            FROM fundraising_investors
        """).fetchone()
        top_funds = rows_to_list(conn.execute("""
            SELECT f.fund_name, f.column_id, COALESCE(SUM(c.amount), 0) AS total_commitment
            FROM fundraising_funds f
            LEFT JOIN fundraising_commitments c ON c.fund_id = f.id
            WHERE f.active = 1
            GROUP BY f.id
            ORDER BY total_commitment DESC
            LIMIT 10
        """).fetchall())
        conn.close()
        return self.send_json({
            "data": {
                "counts": counts,
                "totals": {
                    "total_invested": totals['total_invested'] if totals else 0,
                    "graveyard_count": totals['graveyard_count'] if totals and totals['graveyard_count'] is not None else 0,
                    "follow_up_count": totals['follow_up_count'] if totals and totals['follow_up_count'] is not None else 0
                },
                "top_funds": top_funds
            }
        })

    def handle_list_fundraising_automations(self, user):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)
        conn = get_db()
        ensure_default_automation_rules(conn)
        rules = rows_to_list(conn.execute("""
            SELECT * FROM fundraising_automation_rules
            ORDER BY name ASC
        """).fetchall())
        conn.commit()
        conn.close()
        for r in rules:
            try:
                r['condition_json'] = json.loads(r.get('condition_json') or '{}')
            except Exception:
                r['condition_json'] = {}
            try:
                r['action_json'] = json.loads(r.get('action_json') or '{}')
            except Exception:
                r['action_json'] = {}
        return self.send_json({"data": rules, "total": len(rules)})

    def handle_update_fundraising_automation_rule(self, user, rule_id, body):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)
        conn = get_db()
        existing = conn.execute("SELECT * FROM fundraising_automation_rules WHERE id = ?", (rule_id,)).fetchone()
        if not existing:
            conn.close()
            return self.send_error_json("Automation rule not found", 404)
        sets = []
        args = []
        if 'enabled' in body:
            sets.append("enabled = ?")
            args.append(1 if _to_bool(body.get('enabled')) else 0)
        if 'name' in body:
            sets.append("name = ?")
            args.append(str(body.get('name') or '').strip() or existing['name'])
        if 'condition_json' in body:
            sets.append("condition_json = ?")
            args.append(json.dumps(body.get('condition_json') if isinstance(body.get('condition_json'), dict) else {}))
        if 'action_json' in body:
            sets.append("action_json = ?")
            args.append(json.dumps(body.get('action_json') if isinstance(body.get('action_json'), dict) else {}))
        if not sets:
            conn.close()
            return self.send_error_json("No fields to update")
        sets.append("updated_at = ?")
        args.append(now())
        args.append(rule_id)
        conn.execute(f"UPDATE fundraising_automation_rules SET {', '.join(sets)} WHERE id = ?", args)
        run_fundraising_automations(conn)
        log_audit(conn, user['user_id'], 'fundraising_automation_rule', rule_id, 'update', body)
        conn.commit()
        updated = row_to_dict(conn.execute("SELECT * FROM fundraising_automation_rules WHERE id = ?", (rule_id,)).fetchone())
        conn.close()
        try:
            updated['condition_json'] = json.loads(updated.get('condition_json') or '{}')
        except Exception:
            updated['condition_json'] = {}
        try:
            updated['action_json'] = json.loads(updated.get('action_json') or '{}')
        except Exception:
            updated['action_json'] = {}
        return self.send_json({"data": updated})

    def handle_list_fundraising_automation_runs(self, user, params):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)
        limit = 100
        try:
            limit = max(1, min(500, int(params.get('limit') or 100)))
        except Exception:
            limit = 100
        conn = get_db()
        rows = rows_to_list(conn.execute("""
            SELECT r.*, i.investor_name, ar.name as rule_name
            FROM fundraising_automation_runs r
            LEFT JOIN fundraising_investors i ON r.investor_id = i.id
            LEFT JOIN fundraising_automation_rules ar ON r.rule_id = ar.id
            ORDER BY r.created_at DESC
            LIMIT ?
        """, (limit,)).fetchall())
        conn.close()
        for row in rows:
            try:
                row['result_json'] = json.loads(row.get('result_json') or '{}')
            except Exception:
                row['result_json'] = {}
        return self.send_json({"data": rows, "total": len(rows)})

    def handle_verify_fundraising_backups(self, user):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)
        checked = 0
        valid = 0
        invalid = []
        for b in list_backups():
            checked += 1
            try:
                with open(b['path'], 'r', encoding='utf-8') as f:
                    payload = json.load(f)
                if not isinstance(payload, dict):
                    raise ValueError("payload is not an object")
                if not isinstance(payload.get('grid'), dict):
                    raise ValueError("missing grid")
                if not isinstance(payload.get('views'), list):
                    raise ValueError("missing views")
                if not isinstance(payload['grid'].get('columns'), list) or not isinstance(payload['grid'].get('rows'), list):
                    raise ValueError("grid missing columns/rows")
                valid += 1
            except Exception as exc:
                invalid.append({"filename": b['filename'], "error": str(exc)})
        return self.send_json({
            "data": {
                "checked": checked,
                "valid": valid,
                "invalid": invalid,
                "invalid_count": len(invalid)
            }
        })

    def handle_get_fundraising_activity(self, user, params):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)
        limit = 120
        try:
            limit = max(20, min(500, int(params.get('limit') or 120)))
        except Exception:
            limit = 120
        conn = get_db()
        audit_rows = rows_to_list(conn.execute("""
            SELECT al.id, al.created_at, 'audit' AS source, al.entity_type, al.entity_id, al.action, u.full_name as actor_name
            FROM audit_log al
            LEFT JOIN users u ON al.user_id = u.id
            ORDER BY al.created_at DESC
            LIMIT ?
        """, (limit,)).fetchall())
        auto_rows = rows_to_list(conn.execute("""
            SELECT r.id, r.created_at, 'automation' AS source, 'fundraising_automation' AS entity_type, COALESCE(i.source_row_id, r.investor_id) AS entity_id,
                   r.status AS action, ar.name AS actor_name
            FROM fundraising_automation_runs r
            LEFT JOIN fundraising_automation_rules ar ON r.rule_id = ar.id
            LEFT JOIN fundraising_investors i ON r.investor_id = i.id
            ORDER BY r.created_at DESC
            LIMIT ?
        """, (limit,)).fetchall())
        conn.close()
        backup_rows = []
        for b in list_backups()[:limit]:
            backup_rows.append({
                "id": f"backup:{b['filename']}",
                "created_at": b['modified_at'],
                "source": "backup",
                "entity_type": "fundraising_backup",
                "entity_id": b['filename'],
                "action": b['kind'],
                "actor_name": "system"
            })
        combined = audit_rows + auto_rows + backup_rows
        combined.sort(key=lambda x: str(x.get('created_at') or ''), reverse=True)
        return self.send_json({"data": combined[:limit], "total": len(combined[:limit])})

    def handle_security_status(self, user):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)
        has_custom_secret = bool(os.environ.get("CRM_SECRET_KEY"))
        return self.send_json({
            "data": {
                "env": ENV,
                "auth_required": True,
                "cors_origin": CORS_ORIGIN,
                "has_custom_secret": has_custom_secret,
                "warnings": [] if has_custom_secret else ["CRM_SECRET_KEY is not explicitly set"],
                "rate_limits": {
                    "login_per_min": LOGIN_RATE_LIMIT_PER_MIN,
                    "write_per_min": WRITE_RATE_LIMIT_PER_MIN
                }
            }
        })

    # ═══════════════════════════════════════════════════════════════════════════
    # FEATURE REQUESTS
    # ═══════════════════════════════════════════════════════════════════════════

    def handle_list_feature_requests(self, user, params):
        conn = get_db()
        query = """
            SELECT fr.*, u.full_name as requested_by_name
            FROM feature_requests fr
            LEFT JOIN users u ON fr.requested_by_user_id = u.id
            WHERE 1=1
        """
        args = []

        if params.get('status'):
            query += " AND fr.status = ?"
            args.append(params['status'])

        if params.get('search'):
            search = f"%{params['search']}%"
            query += " AND (fr.title LIKE ? OR fr.description LIKE ? OR fr.requested_by LIKE ?)"
            args.extend([search, search, search])

        query += " ORDER BY fr.created_at DESC"
        rows = rows_to_list(conn.execute(query, args).fetchall())
        conn.close()
        return self.send_json({"data": rows, "total": len(rows)})

    def handle_create_feature_request(self, user, body):
        title = str(body.get('title', '')).strip()
        if not title:
            return self.send_error_json("title is required")

        req_id = generate_id()
        requested_by = str(body.get('requested_by') or user.get('username') or '').strip()
        conn = get_db()
        conn.execute("""
            INSERT INTO feature_requests (
                id, title, description, page, category, priority, status,
                requested_by, requested_by_user_id
            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
        """, (
            req_id,
            title,
            body.get('description'),
            body.get('page'),
            body.get('category', 'general'),
            body.get('priority', 'medium'),
            body.get('status', 'new'),
            requested_by,
            user['user_id']
        ))
        log_audit(conn, user['user_id'], 'feature_request', req_id, 'create', {"title": title})
        conn.commit()
        row = row_to_dict(conn.execute("SELECT * FROM feature_requests WHERE id = ?", (req_id,)).fetchone())
        conn.close()
        return self.send_json({"data": row}, 201)

    def handle_update_feature_request(self, user, req_id, body):
        conn = get_db()
        existing = conn.execute("SELECT * FROM feature_requests WHERE id = ?", (req_id,)).fetchone()
        if not existing:
            conn.close()
            return self.send_error_json("Feature request not found", 404)

        updatable = ['title', 'description', 'page', 'category', 'priority', 'status', 'requested_by']
        sets = []
        args = []
        for field in updatable:
            if field in body:
                sets.append(f"{field} = ?")
                args.append(body[field])

        if not sets:
            conn.close()
            return self.send_error_json("No fields to update")

        sets.append("updated_at = ?")
        args.append(now())
        args.append(req_id)
        conn.execute(f"UPDATE feature_requests SET {', '.join(sets)} WHERE id = ?", args)
        log_audit(conn, user['user_id'], 'feature_request', req_id, 'update', body)
        conn.commit()
        row = row_to_dict(conn.execute("SELECT * FROM feature_requests WHERE id = ?", (req_id,)).fetchone())
        conn.close()
        return self.send_json({"data": row})

    # ═══════════════════════════════════════════════════════════════════════════
    # FUNDRAISING STATE (AIRTABLE-LIKE GRID)
    # ═══════════════════════════════════════════════════════════════════════════

    def _ensure_fundraising_state_row(self, conn):
        ensure_fundraising_state_row(conn)

    def handle_get_fundraising_state(self, user):
        conn = get_db()
        self._ensure_fundraising_state_row(conn)
        row = conn.execute("SELECT * FROM fundraising_state WHERE id = 'main'").fetchone()
        conn.close()

        try:
            grid = json.loads(row['grid_json']) if row and row['grid_json'] else {}
        except json.JSONDecodeError:
            grid = {}
        try:
            views = json.loads(row['views_json']) if row and row['views_json'] else []
        except json.JSONDecodeError:
            views = []
        views = sanitize_grid_views(views)

        grid = sanitize_fundraising_grid(grid)
        if not isinstance(views, list):
            views = []
        columns = grid.get('columns', [])
        rows = grid.get('rows', [])

        return self.send_json({
            "data": {
                "grid": {"columns": columns, "rows": rows},
                "views": views if views else deep_copy_json(DEFAULT_GRID_VIEWS),
                "version": row['version'] if row else 1,
                "updated_at": row['updated_at'] if row else None
            }
        })

    def _cleanup_fundraising_collab(self, conn):
        now_epoch = int(time.time())
        conn.execute("DELETE FROM fundraising_presence WHERE expires_at_epoch <= ?", (now_epoch,))
        conn.execute("DELETE FROM fundraising_cell_locks WHERE expires_at_epoch <= ?", (now_epoch,))

    def _list_fundraising_collab_state(self, conn):
        presence_rows = rows_to_list(conn.execute("""
            SELECT user_id, username, full_name, active_view, row_id, col_id, is_editing, cell_key, last_seen_at
            FROM fundraising_presence
            ORDER BY last_seen_at DESC
        """).fetchall())
        lock_rows = rows_to_list(conn.execute("""
            SELECT cell_key, row_id, col_id, locked_by_user_id, locked_by_username, locked_by_full_name, last_seen_at
            FROM fundraising_cell_locks
            ORDER BY last_seen_at DESC
        """).fetchall())
        for row in presence_rows:
            row['is_editing'] = bool(row.get('is_editing'))
        return {"presence": presence_rows, "locks": lock_rows}

    def handle_get_fundraising_collab_state(self, user):
        conn = get_db()
        self._cleanup_fundraising_collab(conn)
        snapshot = self._list_fundraising_collab_state(conn)
        conn.commit()
        conn.close()
        return self.send_json({"data": snapshot})

    def handle_fundraising_collab_heartbeat(self, user, body):
        active_view = str(body.get('active_view') or '').strip()
        selected = body.get('selected') if isinstance(body.get('selected'), dict) else {}
        editing = body.get('editing') if isinstance(body.get('editing'), dict) else {}
        selected_row_id = str(selected.get('row_id') or '').strip()
        selected_col_id = str(selected.get('col_id') or '').strip()
        editing_row_id = str(editing.get('row_id') or '').strip()
        editing_col_id = str(editing.get('col_id') or '').strip()
        is_editing = bool(editing_row_id and editing_col_id)
        ttl_seconds = int(body.get('ttl_seconds') or 25)
        ttl_seconds = max(10, min(120, ttl_seconds))
        now_epoch = int(time.time())
        expires_at_epoch = now_epoch + ttl_seconds
        seen_at = now()
        lock_conflict = None

        conn = get_db()
        self._cleanup_fundraising_collab(conn)

        user_row = conn.execute("SELECT username, full_name FROM users WHERE id = ?", (user['user_id'],)).fetchone()
        username = str(user_row['username']) if user_row and user_row['username'] else str(user.get('username') or '')
        full_name = str(user_row['full_name']) if user_row and user_row['full_name'] else ''
        editing_cell_key = f"{editing_row_id}:{editing_col_id}" if is_editing else None

        if is_editing and editing_cell_key:
            existing_lock = conn.execute("""
                SELECT cell_key, row_id, col_id, locked_by_user_id, locked_by_username, locked_by_full_name, last_seen_at
                FROM fundraising_cell_locks
                WHERE cell_key = ? AND locked_by_user_id != ? AND expires_at_epoch > ?
                LIMIT 1
            """, (editing_cell_key, user['user_id'], now_epoch)).fetchone()
            if existing_lock:
                lock_conflict = row_to_dict(existing_lock)
                is_editing = False
                editing_cell_key = None
            else:
                conn.execute("""
                    INSERT INTO fundraising_cell_locks (
                        cell_key, row_id, col_id, locked_by_user_id, locked_by_username, locked_by_full_name, last_seen_at, expires_at_epoch
                    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
                    ON CONFLICT(cell_key) DO UPDATE SET
                        row_id = excluded.row_id,
                        col_id = excluded.col_id,
                        locked_by_user_id = excluded.locked_by_user_id,
                        locked_by_username = excluded.locked_by_username,
                        locked_by_full_name = excluded.locked_by_full_name,
                        last_seen_at = excluded.last_seen_at,
                        expires_at_epoch = excluded.expires_at_epoch
                """, (editing_cell_key, editing_row_id, editing_col_id, user['user_id'], username, full_name, seen_at, expires_at_epoch))
                conn.execute("""
                    DELETE FROM fundraising_cell_locks
                    WHERE locked_by_user_id = ? AND cell_key != ?
                """, (user['user_id'], editing_cell_key))
        else:
            conn.execute("DELETE FROM fundraising_cell_locks WHERE locked_by_user_id = ?", (user['user_id'],))

        conn.execute("""
            INSERT INTO fundraising_presence (
                user_id, username, full_name, active_view, row_id, col_id, is_editing, cell_key, last_seen_at, expires_at_epoch
            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
            ON CONFLICT(user_id) DO UPDATE SET
                username = excluded.username,
                full_name = excluded.full_name,
                active_view = excluded.active_view,
                row_id = excluded.row_id,
                col_id = excluded.col_id,
                is_editing = excluded.is_editing,
                cell_key = excluded.cell_key,
                last_seen_at = excluded.last_seen_at,
                expires_at_epoch = excluded.expires_at_epoch
        """, (
            user['user_id'],
            username,
            full_name,
            active_view,
            selected_row_id or editing_row_id,
            selected_col_id or editing_col_id,
            1 if is_editing else 0,
            editing_cell_key,
            seen_at,
            expires_at_epoch
        ))

        snapshot = self._list_fundraising_collab_state(conn)
        conn.commit()
        conn.close()

        return self.send_json({
            "data": {
                **snapshot,
                "lock_conflict": lock_conflict
            }
        })

    def handle_update_fundraising_state(self, user, body):
        grid = body.get('grid', {})
        views = body.get('views')
        expected_version = body.get('expected_version')

        if not isinstance(grid, dict):
            return self.send_error_json("grid must be an object")
        if 'columns' not in grid or 'rows' not in grid:
            return self.send_error_json("grid must include columns and rows")
        if not isinstance(grid.get('columns'), list) or not isinstance(grid.get('rows'), list):
            return self.send_error_json("grid.columns and grid.rows must be arrays")
        grid = sanitize_fundraising_grid(grid)
        if views is not None and not isinstance(views, list):
            return self.send_error_json("views must be an array when provided")

        conn = get_db()
        self._ensure_fundraising_state_row(conn)

        current = conn.execute("SELECT version FROM fundraising_state WHERE id = 'main'").fetchone()
        current_version = int(current['version']) if current else 1
        if expected_version is not None and int(expected_version) != current_version:
            snapshot = conn.execute("SELECT version, updated_at, updated_by FROM fundraising_state WHERE id = 'main'").fetchone()
            conn.close()
            return self.send_json({
                "error": "Version conflict",
                "current_version": current_version,
                "current_updated_at": snapshot['updated_at'] if snapshot else None,
                "current_updated_by": snapshot['updated_by'] if snapshot else None
            }, status=409)

        row = conn.execute("SELECT views_json FROM fundraising_state WHERE id = 'main'").fetchone()
        existing_views = []
        if row and row['views_json']:
            try:
                existing_views = json.loads(row['views_json'])
            except json.JSONDecodeError:
                existing_views = deep_copy_json(DEFAULT_GRID_VIEWS)
        existing_views = sanitize_grid_views(existing_views)

        next_views = sanitize_grid_views(views if views is not None else existing_views)
        next_version = current_version + 1
        conn.execute("""
            UPDATE fundraising_state
            SET grid_json = ?, views_json = ?, version = ?, updated_by = ?, updated_at = ?
            WHERE id = 'main'
        """, (json.dumps(grid), json.dumps(next_views), next_version, user['user_id'], now()))
        sync_fundraising_relational(conn, grid, next_views, actor_user_id=user['user_id'])
        log_audit(conn, user['user_id'], 'fundraising_state', 'main', 'update', {"version": next_version})
        conn.commit()
        conn.close()

        return self.send_json({
            "data": {
                "version": next_version,
                "updated_at": now()
            }
        })

    def handle_export_fundraising_state(self, user):
        conn = get_db()
        self._ensure_fundraising_state_row(conn)
        row = conn.execute("SELECT * FROM fundraising_state WHERE id = 'main'").fetchone()
        conn.close()

        if not row:
            return self.send_error_json("Fundraising state not found", 404)

        try:
            grid = json.loads(row['grid_json']) if row['grid_json'] else {}
        except json.JSONDecodeError:
            grid = {}
        try:
            views = json.loads(row['views_json']) if row['views_json'] else []
        except json.JSONDecodeError:
            views = []

        payload = {
            "exported_at": now(),
            "version": row['version'],
            "updated_at": row['updated_at'],
            "grid": grid,
            "views": views
        }
        return self.send_json({"data": payload})

    def handle_list_fundraising_backups(self, user):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)
        entries = list_backups()
        return self.send_json({"data": entries, "total": len(entries)})

    def _extract_fundraising_restore_payload(self, body):
        payload = body.get('payload')
        if payload is None:
            payload = body.get('data')

        if payload is None and body.get('filename'):
            filename = os.path.basename(str(body.get('filename')))
            if not filename.endswith('.json'):
                return None, "filename must point to a .json backup file"
            backup_dir = os.path.join(DATA_DIR, "backups")
            candidate = os.path.join(backup_dir, filename)
            if not os.path.isfile(candidate):
                return None, "backup file not found"
            try:
                with open(candidate, 'r', encoding='utf-8') as f:
                    payload = json.load(f)
            except Exception:
                return None, "backup file is not valid JSON"

        if payload is None:
            payload = body

        if not isinstance(payload, dict):
            return None, "payload must be a JSON object"
        return payload, None

    def _normalize_fundraising_payload(self, payload):
        grid = payload.get('grid')
        views = payload.get('views')

        if not isinstance(grid, dict):
            return None, None, "payload.grid is required"
        if not isinstance(grid.get('columns'), list) or not isinstance(grid.get('rows'), list):
            return None, None, "payload.grid.columns and payload.grid.rows must be arrays"
        if views is None:
            views = deep_copy_json(DEFAULT_GRID_VIEWS)
        if not isinstance(views, list):
            return None, None, "payload.views must be an array"
        return sanitize_fundraising_grid(grid), sanitize_grid_views(views), None

    def handle_preview_fundraising_restore(self, user, body):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)

        payload, err = self._extract_fundraising_restore_payload(body)
        if err:
            return self.send_error_json(err)
        grid, views, err = self._normalize_fundraising_payload(payload)
        if err:
            return self.send_error_json(err)

        conn = get_db()
        self._ensure_fundraising_state_row(conn)
        current = conn.execute("SELECT * FROM fundraising_state WHERE id = 'main'").fetchone()
        conn.close()
        try:
            current_grid = json.loads(current['grid_json']) if current and current['grid_json'] else {}
        except Exception:
            current_grid = {}
        try:
            current_views = json.loads(current['views_json']) if current and current['views_json'] else []
        except Exception:
            current_views = []
        diff = compute_restore_diff(current_grid, current_views, grid, views)

        preview = {
            "columns_count": len(grid.get('columns', [])),
            "rows_count": len(grid.get('rows', [])),
            "views_count": len(views),
            "source": "filename" if body.get('filename') else "payload",
            "diff": diff
        }
        return self.send_json({"data": preview})

    def handle_backup_fundraising_state(self, user):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)

        conn = get_db()
        self._ensure_fundraising_state_row(conn)
        row = conn.execute("SELECT * FROM fundraising_state WHERE id = 'main'").fetchone()
        conn.close()
        if not row:
            return self.send_error_json("Fundraising state not found", 404)

        info = create_fundraising_backup_file(row, kind="backup")
        conn = get_db()
        policy = load_backup_policy(conn)
        apply_backup_retention(policy)
        conn.close()
        return self.send_json({"data": info}, 201)

    def handle_restore_fundraising_state(self, user, body):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)

        payload, err = self._extract_fundraising_restore_payload(body)
        if err:
            return self.send_error_json(err)
        grid, views, err = self._normalize_fundraising_payload(payload)
        if err:
            return self.send_error_json(err)

        conn = get_db()
        self._ensure_fundraising_state_row(conn)
        current = conn.execute("SELECT * FROM fundraising_state WHERE id = 'main'").fetchone()
        if not current:
            conn.close()
            return self.send_error_json("Fundraising state not found", 404)

        # Always create a rollback snapshot before restore
        pre_backup = create_fundraising_backup_file(current, kind="pre_restore")

        next_version = int(current['version']) + 1
        conn.execute("""
            UPDATE fundraising_state
            SET grid_json = ?, views_json = ?, version = ?, updated_by = ?, updated_at = ?
            WHERE id = 'main'
        """, (json.dumps(grid), json.dumps(views), next_version, user['user_id'], now()))
        sync_fundraising_relational(conn, grid, views, actor_user_id=user['user_id'])
        log_audit(conn, user['user_id'], 'fundraising_state', 'main', 'restore', {"version": next_version, "pre_backup": pre_backup['filename']})
        conn.commit()
        conn.close()

        return self.send_json({
            "data": {
                "version": next_version,
                "pre_restore_backup": {
                    "filename": pre_backup['filename'],
                    "path": pre_backup['path']
                }
            }
        }, 201)


# ─── Seed Data ────────────────────────────────────────────────────────────────

def seed_demo_data():
    """Create demo data for testing."""
    conn = get_db()

    # Check if already seeded
    if conn.execute("SELECT COUNT(*) as c FROM users").fetchone()['c'] > 0:
        conn.close()
        return

    print("Seeding demo data...")

    # Create admin user (password: admin123)
    admin_id = generate_id()
    conn.execute("""
        INSERT INTO users (id, username, email, password_hash, full_name, role)
        VALUES (?, 'admin', 'admin@fund.com', ?, 'Fund Admin', 'admin')
    """, (admin_id, hash_password('admin123')))

    # Create a second user
    user2_id = generate_id()
    conn.execute("""
        INSERT INTO users (id, username, email, password_hash, full_name, role)
        VALUES (?, 'grant', 'grant@ten31.xyz', ?, 'Grant', 'admin')
    """, (user2_id, hash_password('password')))

    # Create organizations
    orgs = [
        (generate_id(), "Sovereign Wealth Holdings", "institutional", "Sovereign Wealth", "https://example.com"),
        (generate_id(), "Pacific Capital Partners", "family_office", "Family Office", "https://example.com"),
        (generate_id(), "Northeast Pension Fund", "pension", "Pension Fund", "https://example.com"),
        (generate_id(), "Redwood Endowment", "endowment", "Endowment", "https://example.com"),
        (generate_id(), "Atlas Family Office", "family_office", "Family Office", "https://example.com"),
        (generate_id(), "Summit Insurance Group", "institutional", "Insurance", "https://example.com"),
        (generate_id(), "Cascade Wealth Management", "wealth_management", "Wealth Management", "https://example.com"),
        (generate_id(), "Blue Harbor Foundation", "foundation", "Foundation", "https://example.com"),
    ]
    for org in orgs:
        conn.execute("""
            INSERT INTO organizations (id, name, type, industry, website, created_by)
            VALUES (?, ?, ?, ?, ?, ?)
        """, (*org, admin_id))

    # Create contacts (mix of investors and prospects)
    contacts = [
        # Investors
        (generate_id(), "James", "Chen", "jchen@sovereign.com", "Managing Director", orgs[0][0], "investor"),
        (generate_id(), "Sarah", "Williams", "swilliams@pacificcap.com", "CIO", orgs[1][0], "investor"),
        (generate_id(), "Michael", "Davis", "mdavis@nepension.org", "Investment Director", orgs[2][0], "investor"),
        (generate_id(), "Elizabeth", "Thompson", "ethompson@redwood.edu", "Endowment Manager", orgs[3][0], "investor"),
        (generate_id(), "Robert", "Kim", "rkim@atlas.family", "Principal", orgs[4][0], "investor"),
        (generate_id(), "Patricia", "Anderson", "panderson@summit.com", "VP Investments", orgs[5][0], "investor"),
        # Prospects
        (generate_id(), "David", "Martinez", "dmartinez@cascade.com", "Partner", orgs[6][0], "prospect"),
        (generate_id(), "Jennifer", "Taylor", "jtaylor@blueharbor.org", "Executive Director", orgs[7][0], "prospect"),
        (generate_id(), "William", "Johnson", "wjohnson@gmail.com", "Independent Investor", None, "prospect"),
        (generate_id(), "Maria", "Garcia", "mgarcia@example.com", "Family Office Principal", None, "prospect"),
        (generate_id(), "Thomas", "Brown", "tbrown@example.com", "Wealth Manager", None, "prospect"),
        (generate_id(), "Linda", "Wilson", "lwilson@example.com", "Portfolio Manager", None, "prospect"),
    ]
    for c in contacts:
        conn.execute("""
            INSERT INTO contacts (id, first_name, last_name, email, title, organization_id, contact_type, created_by)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?)
        """, (*c, admin_id))

    # Create opportunities
    opp_data = [
        (contacts[6][0], orgs[6][0], "Cascade Wealth - Fund II", "meeting", 10000000, 10000000, 40, user2_id),
        (contacts[7][0], orgs[7][0], "Blue Harbor - Fund II", "due_diligence", 5000000, 5000000, 60, user2_id),
        (contacts[8][0], None, "William Johnson - Direct", "outreach", 0, 2000000, 20, admin_id),
        (contacts[9][0], None, "Garcia Family Office - Fund II", "lead", 0, 15000000, 10, admin_id),
        (contacts[10][0], None, "Thomas Brown - WM Referral", "meeting", 0, 3000000, 30, user2_id),
        (contacts[11][0], None, "Linda Wilson - PM Intro", "outreach", 0, 5000000, 15, admin_id),
    ]
    for opp in opp_data:
        conn.execute("""
            INSERT INTO opportunities (id, contact_id, organization_id, name, stage,
                commitment_amount, expected_amount, probability, owner_id, fund_name)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'Fund II')
        """, (generate_id(), *opp))

    # Create communications
    comm_data = [
        (contacts[0][0], "meeting", "Q4 Review Meeting", "Discussed fund performance and upcoming distributions.", "2025-12-15", 60, "positive", admin_id),
        (contacts[1][0], "email", "Capital Call Notice", "Sent Q1 capital call documentation.", "2026-01-10", None, "neutral", admin_id),
        (contacts[2][0], "call", "Annual Check-in", "Reviewed portfolio allocation and discussed Fund II.", "2026-01-20", 30, "positive", user2_id),
        (contacts[6][0], "meeting", "Initial Presentation", "Presented Fund II thesis and track record.", "2026-02-01", 90, "positive", user2_id),
        (contacts[7][0], "email", "Due Diligence Materials", "Sent DDQ, audited financials, and PPM.", "2026-02-05", None, "neutral", user2_id),
        (contacts[8][0], "call", "Intro Call", "Initial outreach, discussed investment interests.", "2026-02-08", 20, "positive", admin_id),
        (contacts[9][0], "email", "Introduction Email", "Sent overview deck and one-pager.", "2026-02-10", None, "neutral", admin_id),
        (contacts[6][0], "call", "Follow-up on Presentation", "Cascade team asked about risk management approach.", "2026-02-10", 25, "positive", user2_id),
    ]
    for cm in comm_data:
        conn.execute("""
            INSERT INTO communications (id, contact_id, type, subject, body,
                communication_date, duration_minutes, outcome, created_by)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
        """, (generate_id(), *cm))

    # Create some tags
    tag_data = [
        ("High Priority", "#ef4444"),
        ("Fund I LP", "#3b82f6"),
        ("Fund II Prospect", "#8b5cf6"),
        ("Family Office", "#f59e0b"),
        ("Institutional", "#10b981"),
        ("Re-up Target", "#ec4899"),
    ]
    for tag in tag_data:
        conn.execute("INSERT INTO tags (id, name, color) VALUES (?, ?, ?)",
                     (generate_id(), *tag))

    conn.commit()
    conn.close()
    print("Demo data seeded successfully.")


# ─── Email-activity summary agent ────────────────────────────────────────────
# When a sent/received email is matched to an investor, summarize it to ONE dated,
# marked note on the LOCAL model (sovereign — nothing leaves Ten31, so no redaction
# boundary is needed) and append it to that investor's notes in the fundraising grid.
# Going-forward only: we never summarize email dated before the feature was switched
# on, so the historical backfill does not generate noise.

_ACTIVITY_SINCE_KEY = "email_activity_since"
_ACTIVITY_MARKER = "✉"   # marks an email-derived note (kept editable before approval)


def _fmt_activity_date(sent_at):
    """ISO-ish timestamp -> 'Jun 25, 2026' (falls back to the raw date part)."""
    from datetime import datetime
    datepart = str(sent_at or "")[:10]   # YYYY-MM-DD
    try:
        return datetime.strptime(datepart, "%Y-%m-%d").strftime("%b %-d, %Y")
    except Exception:
        return datepart


def _activity_investor(conn, email_id):
    """Resolve (grid_row_id, investor_name) for a matched email via its highest-
    confidence investor link. Either may be None."""
    link = conn.execute(
        "SELECT fundraising_investor_id, organization_id, contact_id FROM email_investor_links "
        "WHERE email_id=? ORDER BY match_confidence DESC LIMIT 1", (email_id,)).fetchone()
    if not link:
        return None, None
    inv_id = link["fundraising_investor_id"]
    name = None
    if inv_id:
        r = conn.execute("SELECT investor_name FROM fundraising_investors WHERE id=?", (inv_id,)).fetchone()
        name = r["investor_name"] if r else None
    if not name and link["organization_id"]:
        r = conn.execute("SELECT name FROM organizations WHERE id=?", (link["organization_id"],)).fetchone()
        name = r["name"] if r else None
    if not name and link["contact_id"]:
        r = conn.execute("SELECT first_name, last_name FROM contacts WHERE id=?", (link["contact_id"],)).fetchone()
        if r:
            name = f"{r['first_name'] or ''} {r['last_name'] or ''}".strip()
    return inv_id, name


def _summarize_email_gist(subject, body):
    """One short clause describing the email's substance, from the LOCAL model."""
    try:
        sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "ingest"))
        import llm  # noqa: E402
    except Exception:
        return None
    text = (body or "")[:4000]
    if not (subject or text).strip():
        return None
    out = llm.chat(
        f"Subject: {subject or '(none)'}\n\n{text}",
        system=("You summarize an email into a brief CRM note. Reply with ONE clause under 14 words "
                "describing what the email is about. No greeting, no names, no quotes, no trailing period."),
        max_tokens=40, temperature=0.0)
    gist = " ".join((out or "").split()).strip().rstrip(".")
    return gist or None


def _append_grid_note(conn, inv_id, inv_name, note, updated_by=None):
    """Append a note to the matched investor's notes cell in the live grid (newest at
    bottom), bump the grid version, and refresh the relational projection. Best-effort."""
    row = conn.execute("SELECT grid_json, views_json, version FROM fundraising_state WHERE id='main'").fetchone()
    if not row or not row["grid_json"]:
        return False
    try:
        grid = json.loads(row["grid_json"])
    except Exception:
        return False
    rows = grid.get("rows", []) if isinstance(grid, dict) else []
    target = None
    if inv_id:
        target = next((r for r in rows if isinstance(r, dict) and str(r.get("id")) == str(inv_id)), None)
    if target is None and inv_name:
        nn = _normalize_text(inv_name)
        target = next((r for r in rows if isinstance(r, dict)
                       and _normalize_text(str(r.get("investor_name") or "")) == nn), None)
    if target is None:
        return False
    existing = str(target.get("notes") or "").rstrip()
    target["notes"] = (existing + "\n" + note) if existing else note
    try:
        views = json.loads(row["views_json"]) if row["views_json"] else []
    except Exception:
        views = []
    # updated_by has a FK to users(id); stamp the approving user (None -> NULL is fine).
    conn.execute("UPDATE fundraising_state SET grid_json=?, version=?, updated_by=?, updated_at=? WHERE id='main'",
                 (json.dumps(grid), (row["version"] or 0) + 1, updated_by, now()))
    try:
        sync_fundraising_relational(conn, grid, views, actor_user_id=updated_by)
    except Exception:
        pass
    return True


def _activity_note_text(sent_at, direction, gist):
    return f"{_ACTIVITY_MARKER} {_fmt_activity_date(sent_at)} — {direction}: {gist}"


def propose_email_activity_notes(limit=50):
    """Draft a PROPOSED grid note per newly-matched email and queue it for human
    review (status 'pending'). Does NOT touch the grid — approval does that. Idempotent
    (one proposal per email), going-forward only. Safe to call after each Gmail sync."""
    conn = get_db()
    try:
        try:
            conn.execute("SELECT 1 FROM email_activity_proposals LIMIT 1")
        except sqlite3.OperationalError:
            return {"proposed": 0, "skipped": "tables_absent"}
        since = get_app_setting(conn, _ACTIVITY_SINCE_KEY)
        if not since:
            since = now()
            set_app_setting(conn, _ACTIVITY_SINCE_KEY, since)
            conn.commit()
        try:
            rows = conn.execute(
                "SELECT id, subject, body_text, snippet, from_email, sent_at FROM emails "
                "WHERE is_matched=1 AND sent_at >= ? "
                "AND id NOT IN (SELECT email_id FROM email_activity_proposals) "
                "ORDER BY sent_at ASC LIMIT ?", (since, limit)).fetchall()
        except sqlite3.OperationalError:
            return {"proposed": 0, "skipped": "emails_absent"}
        if not rows:
            return {"proposed": 0}
        own = set()
        try:
            own = {(r[0] or "").lower() for r in conn.execute("SELECT email_address FROM email_accounts")}
        except Exception:
            pass
        done = 0
        for r in rows:
            inv_id, inv_name = _activity_investor(conn, r["id"])
            direction = "Sent" if (r["from_email"] or "").lower() in own else "Received"
            gist = _summarize_email_gist(r["subject"], r["body_text"] or r["snippet"] or "")
            if not gist:
                continue  # leave unproposed; a later pass retries once the model answers
            note = _activity_note_text(r["sent_at"], direction, gist)
            conn.execute(
                "INSERT OR IGNORE INTO email_activity_proposals "
                "(id,email_id,investor_id,investor_name,direction,summary,proposed_note,"
                " email_subject,email_date,status,created_at) "
                "VALUES (?,?,?,?,?,?,?,?,?,'pending',?)",
                (generate_id(), r["id"], inv_id, inv_name, direction.lower(), gist, note,
                 r["subject"], r["sent_at"], now()))
            conn.commit()
            done += 1
        return {"proposed": done}
    finally:
        conn.close()


def list_email_activity_proposals(conn, status="pending", limit=200):
    try:
        rows = conn.execute(
            "SELECT id, email_id, investor_id, investor_name, direction, summary, proposed_note, "
            "email_subject, email_date, status, created_at FROM email_activity_proposals "
            "WHERE status=? ORDER BY email_date ASC, created_at ASC LIMIT ?", (status, limit)).fetchall()
        return [dict(r) for r in rows]
    except sqlite3.OperationalError:
        return []


def decide_email_activity_proposal(conn, proposal_id, decision, user_id, edited_note=None):
    """Approve (optionally with an edited note -> append to grid) or dismiss a proposal."""
    p = conn.execute("SELECT * FROM email_activity_proposals WHERE id=?", (proposal_id,)).fetchone()
    if not p:
        return {"error": "not_found"}
    if p["status"] != "pending":
        return {"error": "already_decided", "status": p["status"]}
    if decision == "approve":
        note = (edited_note or "").strip() or p["proposed_note"]
        placed = _append_grid_note(conn, p["investor_id"], p["investor_name"], note, updated_by=user_id)
        conn.execute("UPDATE email_activity_proposals SET status='approved', final_note=?, decided_by=?, decided_at=? WHERE id=?",
                     (note, user_id, now(), proposal_id))
        action, result = "email.activity_approved", {"status": "approved", "placed_in_grid": placed}
    elif decision == "dismiss":
        conn.execute("UPDATE email_activity_proposals SET status='dismissed', decided_by=?, decided_at=? WHERE id=?",
                     (user_id, now(), proposal_id))
        action, result = "email.activity_dismissed", {"status": "dismissed"}
    else:
        return {"error": "bad_decision"}
    conn.execute(
        "INSERT INTO interaction_log (id, ts, actor_type, actor_id, action, target_type, target_id, payload, source, created_at) "
        "VALUES (?,?,?,?,?,?,?,?,?,?)",
        (generate_id(), now(), "human", user_id, action, "fundraising_investor", p["investor_id"],
         json.dumps({"proposal_id": proposal_id}), "crm_ui", now()))
    conn.commit()
    return result


# ─── Main Entry Point ────────────────────────────────────────────────────────

def main():
    if ENV == "production" and not os.environ.get("CRM_SECRET_KEY"):
        print("ERROR: CRM_SECRET_KEY must be set in production mode (CRM_ENV=production).")
        sys.exit(1)
    init_db()
    if SEED_DEMO_DATA:
        seed_demo_data()
    else:
        print("Demo data seeding disabled (set CRM_SEED_DEMO_DATA=1 to enable).")
    start_backup_scheduler()

    # ─── Gmail sync scheduler (feature-flag-guarded) ─────────────────
    if os.environ.get("CRM_GMAIL_INTEGRATION_ENABLED", "").lower() in ("1", "true", "yes", "on"):
        try:
            from email_integration.scheduler import start_sync_scheduler
            # After each Gmail sync, draft proposed activity notes for human review.
            start_sync_scheduler(post_sync=lambda: propose_email_activity_notes())
            print("[email_integration] Gmail sync scheduler started")
        except Exception as _e:
            print(f"[email_integration] failed to start scheduler: {_e}")

    # ─── Daily activity digest scheduler ─────────────────────────────
    # Always started; it reads the digest policy (enabled + send hour) from the DB
    # each cycle, so the Settings → Admin toggle controls it live (no restart).
    try:
        from email_integration.digest_scheduler import start_digest_scheduler
        start_digest_scheduler()
        print("[digest] daily activity digest scheduler started (policy-controlled)")
    except Exception as _e:
        print(f"[digest] failed to start digest scheduler: {_e}")

    # ThreadingHTTPServer lets one slow request (or a wave of scanner probes)
    # not block legit users. SQLite is opened per-request via get_db(), and
    # WAL mode allows concurrent readers + a single writer, so this is safe.
    server = ThreadingHTTPServer((HOST, PORT), CRMHandler)
    server.daemon_threads = True
    print(f"\n{'='*60}")
    print(f"  Venture Fund CRM Server")
    print(f"  Running at http://{HOST}:{PORT}")
    print(f"  Database: {DB_PATH}")
    print(f"{'='*60}")
    if SEED_DEMO_DATA:
        print(f"\n  Demo login: admin / admin123")
        print(f"  Or: grant / password")
    print(f"\n  Press Ctrl+C to stop\n")

    try:
        server.serve_forever()
    except KeyboardInterrupt:
        print("\nShutting down server...")
        server.shutdown()

if __name__ == "__main__":
    main()