ten31-database/backend/server.py

#!/usr/bin/env python3
"""
Venture Fund CRM — Local Self-Hosted Server
Backend API server using Python stdlib + SQLite + bcrypt + PyJWT
"""

import json
import os
import sys
import sqlite3
import hashlib
import hmac
import time
import uuid
import csv
import io
import re
import base64
import threading
from datetime import datetime, timedelta
from http.server import HTTPServer, ThreadingHTTPServer, BaseHTTPRequestHandler
from urllib.parse import urlparse, parse_qs, unquote
from functools import wraps

# Available system packages
try:
    import bcrypt  # type: ignore
    BCRYPT_AVAILABLE = True
except Exception:
    bcrypt = None
    BCRYPT_AVAILABLE = False

try:
    import jwt  # type: ignore
    JWT_AVAILABLE = True
except Exception:
    jwt = None
    JWT_AVAILABLE = False

# Phase-1 Architect: human-gated thesis approval logic (pure stdlib; guarded).
try:
    import thesis_review  # type: ignore
except Exception:
    thesis_review = None

# Phase-1: entity-merge review + UI-triggered index jobs (guarded).
try:
    import entity_merge  # type: ignore
except Exception:
    entity_merge = None
try:
    import entity_jobs  # type: ignore
except Exception:
    entity_jobs = None

# ─── Configuration ────────────────────────────────────────────────────────────

BASE_DIR = os.path.dirname(os.path.abspath(__file__))
PROJECT_DIR = os.path.dirname(BASE_DIR)
DATA_DIR = os.environ.get("CRM_DATA_DIR", os.path.join(PROJECT_DIR, "data"))
FRONTEND_DIR = os.environ.get("CRM_FRONTEND_DIR", os.path.join(PROJECT_DIR, "frontend"))
DB_PATH = os.environ.get("CRM_DB_PATH", os.path.join(DATA_DIR, "crm.db"))
SECRET_KEY = os.environ.get("CRM_SECRET_KEY", "venture-crm-secret-change-in-production-" + str(uuid.uuid4()))
TOKEN_EXPIRY_HOURS = 24
HOST = os.environ.get("CRM_HOST", "0.0.0.0")
PORT = int(os.environ.get("CRM_PORT", "8080"))
CORS_ORIGIN = os.environ.get("CRM_CORS_ORIGIN", "*")
ENV = os.environ.get("CRM_ENV", "development")
LOGIN_RATE_LIMIT_PER_MIN = int(os.environ.get("CRM_LOGIN_RATE_LIMIT_PER_MIN", "20"))
WRITE_RATE_LIMIT_PER_MIN = int(os.environ.get("CRM_WRITE_RATE_LIMIT_PER_MIN", "300"))
GET_RATE_LIMIT_PER_MIN = int(os.environ.get("CRM_GET_RATE_LIMIT_PER_MIN", "600"))
# Auto-ban any IP that racks up too many 404s in a short window — almost always
# a vulnerability scanner blasting common paths (/.env, /.git/config, /swagger,
# /actuator/env, wp-json, etc.). Banned IPs get instant 429s with no DB or
# filesystem work, so they can't keep the single SQLite writer busy.
ABUSE_404_THRESHOLD = int(os.environ.get("CRM_ABUSE_404_THRESHOLD", "15"))
ABUSE_404_WINDOW_SEC = int(os.environ.get("CRM_ABUSE_404_WINDOW_SEC", "60"))
ABUSE_BAN_SEC = int(os.environ.get("CRM_ABUSE_BAN_SEC", "900"))  # 15 minutes
BACKUP_POLICY_SETTING_KEY = "fundraising_backup_policy"
DEFAULT_BACKUP_POLICY = {
    "enabled": True,
    "interval_hours": 24,
    "retention_days": 30,
    "max_backups": 60,
    "last_run_at": None
}
SEED_DEMO_DATA = os.environ.get("CRM_SEED_DEMO_DATA", "").strip().lower() in ("1", "true", "yes", "on")

os.makedirs(DATA_DIR, exist_ok=True)

# ─── Database Setup ───────────────────────────────────────────────────────────

def get_db():
    """Get a database connection with WAL mode and foreign keys enabled."""
    conn = sqlite3.connect(DB_PATH)
    conn.execute("PRAGMA journal_mode=WAL")
    conn.execute("PRAGMA foreign_keys=ON")
    conn.execute("PRAGMA busy_timeout=5000")
    conn.row_factory = sqlite3.Row
    return conn

def init_db():
    """Initialize all database tables."""
    conn = get_db()
    cursor = conn.cursor()

    cursor.executescript("""
        CREATE TABLE IF NOT EXISTS users (
            id TEXT PRIMARY KEY,
            username TEXT NOT NULL UNIQUE,
            email TEXT NOT NULL UNIQUE,
            password_hash TEXT NOT NULL,
            full_name TEXT NOT NULL,
            role TEXT NOT NULL DEFAULT 'member',
            is_active INTEGER DEFAULT 1,
            created_at TEXT DEFAULT (datetime('now')),
            updated_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS organizations (
            id TEXT PRIMARY KEY,
            name TEXT NOT NULL,
            type TEXT DEFAULT 'other',
            industry TEXT,
            website TEXT,
            phone TEXT,
            email TEXT,
            address TEXT,
            city TEXT,
            state TEXT,
            country TEXT,
            description TEXT,
            tags TEXT DEFAULT '[]',
            created_by TEXT REFERENCES users(id),
            created_at TEXT DEFAULT (datetime('now')),
            updated_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS contacts (
            id TEXT PRIMARY KEY,
            first_name TEXT NOT NULL,
            last_name TEXT NOT NULL,
            email TEXT,
            phone TEXT,
            mobile TEXT,
            title TEXT,
            organization_id TEXT REFERENCES organizations(id) ON DELETE SET NULL,
            contact_type TEXT NOT NULL DEFAULT 'prospect',
            status TEXT NOT NULL DEFAULT 'active',
            source TEXT,
            tags TEXT DEFAULT '[]',
            notes TEXT,
            linkedin_url TEXT,
            city TEXT,
            state TEXT,
            country TEXT,
            location_query TEXT,
            preferred_contact TEXT DEFAULT 'email',
            created_by TEXT REFERENCES users(id),
            created_at TEXT DEFAULT (datetime('now')),
            updated_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS opportunities (
            id TEXT PRIMARY KEY,
            name TEXT NOT NULL,
            contact_id TEXT NOT NULL REFERENCES contacts(id) ON DELETE CASCADE,
            organization_id TEXT REFERENCES organizations(id) ON DELETE SET NULL,
            stage TEXT NOT NULL DEFAULT 'lead',
            commitment_amount REAL DEFAULT 0,
            expected_amount REAL DEFAULT 0,
            probability INTEGER DEFAULT 10,
            expected_close_date TEXT,
            fund_name TEXT,
            description TEXT,
            next_step TEXT,
            owner_id TEXT NOT NULL REFERENCES users(id),
            priority TEXT DEFAULT 'medium',
            lost_reason TEXT,
            created_at TEXT DEFAULT (datetime('now')),
            updated_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS communications (
            id TEXT PRIMARY KEY,
            contact_id TEXT NOT NULL REFERENCES contacts(id) ON DELETE CASCADE,
            opportunity_id TEXT REFERENCES opportunities(id) ON DELETE SET NULL,
            type TEXT NOT NULL DEFAULT 'note',
            subject TEXT,
            body TEXT,
            communication_date TEXT NOT NULL,
            duration_minutes INTEGER,
            outcome TEXT,
            next_action TEXT,
            next_action_date TEXT,
            attendees TEXT DEFAULT '[]',
            created_by TEXT NOT NULL REFERENCES users(id),
            created_at TEXT DEFAULT (datetime('now')),
            updated_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS lp_profiles (
            id TEXT PRIMARY KEY,
            contact_id TEXT NOT NULL UNIQUE REFERENCES contacts(id) ON DELETE CASCADE,
            commitment_amount REAL DEFAULT 0,
            funded_amount REAL DEFAULT 0,
            commitment_date TEXT,
            fund_name TEXT,
            investor_type TEXT,
            accredited INTEGER DEFAULT 0,
            legal_docs_signed INTEGER DEFAULT 0,
            signed_date TEXT,
            wire_received INTEGER DEFAULT 0,
            wire_date TEXT,
            k1_sent INTEGER DEFAULT 0,
            preferred_communication TEXT DEFAULT 'email',
            notes TEXT,
            created_at TEXT DEFAULT (datetime('now')),
            updated_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS custom_fields (
            id TEXT PRIMARY KEY,
            name TEXT NOT NULL,
            entity_type TEXT NOT NULL,
            field_type TEXT NOT NULL DEFAULT 'text',
            options TEXT DEFAULT '[]',
            required INTEGER DEFAULT 0,
            display_order INTEGER DEFAULT 0,
            created_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS custom_field_values (
            id TEXT PRIMARY KEY,
            custom_field_id TEXT NOT NULL REFERENCES custom_fields(id) ON DELETE CASCADE,
            entity_id TEXT NOT NULL,
            entity_type TEXT NOT NULL,
            value TEXT,
            updated_at TEXT DEFAULT (datetime('now')),
            UNIQUE(custom_field_id, entity_id, entity_type)
        );

        CREATE TABLE IF NOT EXISTS audit_log (
            id TEXT PRIMARY KEY,
            user_id TEXT REFERENCES users(id),
            entity_type TEXT NOT NULL,
            entity_id TEXT NOT NULL,
            action TEXT NOT NULL,
            changes TEXT,
            created_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS tags (
            id TEXT PRIMARY KEY,
            name TEXT NOT NULL UNIQUE,
            color TEXT DEFAULT '#6366f1',
            created_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS feature_requests (
            id TEXT PRIMARY KEY,
            title TEXT NOT NULL,
            description TEXT,
            page TEXT,
            category TEXT DEFAULT 'general',
            priority TEXT DEFAULT 'medium',
            status TEXT DEFAULT 'new',
            requested_by TEXT,
            requested_by_user_id TEXT REFERENCES users(id),
            created_at TEXT DEFAULT (datetime('now')),
            updated_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS fundraising_state (
            id TEXT PRIMARY KEY,
            grid_json TEXT NOT NULL,
            views_json TEXT NOT NULL,
            version INTEGER NOT NULL DEFAULT 1,
            updated_by TEXT REFERENCES users(id),
            created_at TEXT DEFAULT (datetime('now')),
            updated_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS fundraising_investors (
            id TEXT PRIMARY KEY,
            investor_name TEXT NOT NULL,
            notes TEXT,
            lead TEXT,
            lead_source TEXT,
            priority INTEGER DEFAULT 0,
            follow_up INTEGER DEFAULT 0,
            graveyard INTEGER DEFAULT 0,
            longshot_followup INTEGER DEFAULT 0,
            source_row_id TEXT NOT NULL UNIQUE,
            total_invested REAL DEFAULT 0,
            updated_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS fundraising_contacts (
            id TEXT PRIMARY KEY,
            investor_id TEXT NOT NULL REFERENCES fundraising_investors(id) ON DELETE CASCADE,
            full_name TEXT NOT NULL,
            email TEXT,
            title TEXT,
            city TEXT,
            state TEXT,
            country TEXT,
            location_query TEXT,
            sort_order INTEGER DEFAULT 0,
            updated_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS fundraising_funds (
            id TEXT PRIMARY KEY,
            column_id TEXT NOT NULL UNIQUE,
            fund_name TEXT NOT NULL,
            display_order INTEGER DEFAULT 0,
            active INTEGER DEFAULT 1,
            updated_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS fundraising_commitments (
            id TEXT PRIMARY KEY,
            investor_id TEXT NOT NULL REFERENCES fundraising_investors(id) ON DELETE CASCADE,
            fund_id TEXT NOT NULL REFERENCES fundraising_funds(id) ON DELETE CASCADE,
            amount REAL DEFAULT 0,
            updated_at TEXT DEFAULT (datetime('now')),
            UNIQUE(investor_id, fund_id)
        );

        CREATE TABLE IF NOT EXISTS fundraising_views (
            id TEXT PRIMARY KEY,
            name TEXT NOT NULL,
            filters_json TEXT NOT NULL,
            quick_search TEXT,
            hidden_columns_json TEXT NOT NULL DEFAULT '[]',
            column_filters_json TEXT NOT NULL DEFAULT '[]',
            updated_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS fundraising_automation_rules (
            id TEXT PRIMARY KEY,
            name TEXT NOT NULL,
            trigger_type TEXT NOT NULL DEFAULT 'flag_change',
            condition_json TEXT NOT NULL,
            action_json TEXT NOT NULL,
            enabled INTEGER DEFAULT 1,
            updated_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS fundraising_automation_runs (
            id TEXT PRIMARY KEY,
            rule_id TEXT REFERENCES fundraising_automation_rules(id) ON DELETE SET NULL,
            investor_id TEXT REFERENCES fundraising_investors(id) ON DELETE SET NULL,
            status TEXT NOT NULL DEFAULT 'applied',
            result_json TEXT,
            created_at TEXT DEFAULT (datetime('now'))
        );

        CREATE TABLE IF NOT EXISTS fundraising_list_memberships (
            id TEXT PRIMARY KEY,
            investor_id TEXT NOT NULL REFERENCES fundraising_investors(id) ON DELETE CASCADE,
            list_key TEXT NOT NULL,
            source TEXT NOT NULL DEFAULT 'automation',
            updated_at TEXT DEFAULT (datetime('now')),
            UNIQUE(investor_id, list_key)
        );

        CREATE TABLE IF NOT EXISTS fundraising_presence (
            user_id TEXT PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE,
            username TEXT NOT NULL,
            full_name TEXT,
            active_view TEXT,
            row_id TEXT,
            col_id TEXT,
            is_editing INTEGER DEFAULT 0,
            cell_key TEXT,
            last_seen_at TEXT DEFAULT (datetime('now')),
            expires_at_epoch INTEGER NOT NULL
        );

        CREATE TABLE IF NOT EXISTS fundraising_cell_locks (
            cell_key TEXT PRIMARY KEY,
            row_id TEXT NOT NULL,
            col_id TEXT NOT NULL,
            locked_by_user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE,
            locked_by_username TEXT NOT NULL,
            locked_by_full_name TEXT,
            last_seen_at TEXT DEFAULT (datetime('now')),
            expires_at_epoch INTEGER NOT NULL
        );

        CREATE TABLE IF NOT EXISTS app_settings (
            key TEXT PRIMARY KEY,
            value_json TEXT NOT NULL,
            updated_at TEXT DEFAULT (datetime('now'))
        );

        -- Indexes for performance
        CREATE INDEX IF NOT EXISTS idx_contacts_type ON contacts(contact_type);
        CREATE INDEX IF NOT EXISTS idx_contacts_status ON contacts(status);
        CREATE INDEX IF NOT EXISTS idx_contacts_org ON contacts(organization_id);
        CREATE INDEX IF NOT EXISTS idx_opportunities_stage ON opportunities(stage);
        CREATE INDEX IF NOT EXISTS idx_opportunities_owner ON opportunities(owner_id);
        CREATE INDEX IF NOT EXISTS idx_opportunities_contact ON opportunities(contact_id);
        CREATE INDEX IF NOT EXISTS idx_communications_contact ON communications(contact_id);
        CREATE INDEX IF NOT EXISTS idx_communications_date ON communications(communication_date);
        CREATE INDEX IF NOT EXISTS idx_audit_entity ON audit_log(entity_type, entity_id);
        CREATE INDEX IF NOT EXISTS idx_lp_profiles_contact ON lp_profiles(contact_id);
        CREATE INDEX IF NOT EXISTS idx_feature_requests_status ON feature_requests(status);
        CREATE INDEX IF NOT EXISTS idx_feature_requests_created_at ON feature_requests(created_at);
        CREATE INDEX IF NOT EXISTS idx_fr_investor_name ON fundraising_investors(investor_name);
        CREATE INDEX IF NOT EXISTS idx_fr_investor_lead ON fundraising_investors(lead);
        CREATE INDEX IF NOT EXISTS idx_fr_contacts_investor ON fundraising_contacts(investor_id);
        CREATE INDEX IF NOT EXISTS idx_fr_commitments_investor ON fundraising_commitments(investor_id);
        CREATE INDEX IF NOT EXISTS idx_fr_commitments_fund ON fundraising_commitments(fund_id);
        CREATE INDEX IF NOT EXISTS idx_fr_automation_runs_created ON fundraising_automation_runs(created_at);
        CREATE INDEX IF NOT EXISTS idx_fr_memberships_list ON fundraising_list_memberships(list_key);
        CREATE INDEX IF NOT EXISTS idx_fr_presence_expires ON fundraising_presence(expires_at_epoch);
        CREATE INDEX IF NOT EXISTS idx_fr_locks_expires ON fundraising_cell_locks(expires_at_epoch);
    """)

    # Lightweight schema migrations for existing databases.
    for stmt in [
        "ALTER TABLE contacts ADD COLUMN city TEXT",
        "ALTER TABLE contacts ADD COLUMN state TEXT",
        "ALTER TABLE contacts ADD COLUMN country TEXT",
        "ALTER TABLE contacts ADD COLUMN location_query TEXT",
        "ALTER TABLE fundraising_investors ADD COLUMN lead_source TEXT",
    ]:
        try:
            conn.execute(stmt)
        except sqlite3.OperationalError:
            pass

    # ─── Gmail integration migrations (feature-flag-guarded import) ───
    try:
        from email_integration.db import apply_migrations as _email_apply_migrations
        _email_apply_migrations(cursor)
    except ImportError:
        pass
    except Exception as _e:
        print(f"[email_integration] migration warning: {_e}")

    conn.commit()

    # ─── Core schema migrations (Phase 0+; ordered .sql files w/ ledger) ───
    # Additive/reversible only; tracked in schema_migrations. See core_migrations.py.
    try:
        from core_migrations import apply_core_migrations as _apply_core_migrations
        _apply_core_migrations(conn)
    except Exception as _e:
        print(f"[migrations] core migration warning: {_e}")

    conn.close()
    print(f"Database initialized at {DB_PATH}")

# ─── Auth Helpers ─────────────────────────────────────────────────────────────

def hash_password(password):
    if BCRYPT_AVAILABLE:
        return bcrypt.hashpw(password.encode('utf-8'), bcrypt.gensalt()).decode('utf-8')

    # Stdlib fallback (PBKDF2) when bcrypt is unavailable
    salt = os.urandom(16).hex()
    digest = hashlib.pbkdf2_hmac('sha256', password.encode('utf-8'), bytes.fromhex(salt), 200_000).hex()
    return f"pbkdf2_sha256${salt}${digest}"

def verify_password(password, hashed):
    if BCRYPT_AVAILABLE and not str(hashed).startswith("pbkdf2_sha256$"):
        return bcrypt.checkpw(password.encode('utf-8'), hashed.encode('utf-8'))

    try:
        scheme, salt_hex, digest_hex = str(hashed).split('$', 2)
        if scheme != 'pbkdf2_sha256':
            return False
        check = hashlib.pbkdf2_hmac('sha256', password.encode('utf-8'), bytes.fromhex(salt_hex), 200_000).hex()
        return hmac.compare_digest(check, digest_hex)
    except Exception:
        return False

def _b64url_encode(data: bytes) -> str:
    return base64.urlsafe_b64encode(data).decode('utf-8').rstrip('=')

def _b64url_decode(data: str) -> bytes:
    padding = '=' * (-len(data) % 4)
    return base64.urlsafe_b64decode((data + padding).encode('utf-8'))

def create_token(user_id, username, role):
    payload = {
        "user_id": user_id,
        "username": username,
        "role": role,
        "exp": int(time.time()) + TOKEN_EXPIRY_HOURS * 3600,
        "iat": int(time.time())
    }
    if JWT_AVAILABLE:
        return jwt.encode(payload, SECRET_KEY, algorithm="HS256")

    # Stdlib fallback token format: base64url(payload).hmac_sha256_signature
    payload_bytes = json.dumps(payload, separators=(',', ':'), sort_keys=True).encode('utf-8')
    payload_part = _b64url_encode(payload_bytes)
    signature = hmac.new(SECRET_KEY.encode('utf-8'), payload_part.encode('utf-8'), hashlib.sha256).digest()
    return f"{payload_part}.{_b64url_encode(signature)}"

def decode_token(token):
    if JWT_AVAILABLE:
        try:
            return jwt.decode(token, SECRET_KEY, algorithms=["HS256"])
        except jwt.ExpiredSignatureError:
            return None
        except jwt.InvalidTokenError:
            return None

    try:
        payload_part, sig_part = str(token).split('.', 1)
        expected_sig = hmac.new(SECRET_KEY.encode('utf-8'), payload_part.encode('utf-8'), hashlib.sha256).digest()
        actual_sig = _b64url_decode(sig_part)
        if not hmac.compare_digest(expected_sig, actual_sig):
            return None
        payload = json.loads(_b64url_decode(payload_part).decode('utf-8'))
        exp = int(payload.get('exp', 0))
        if exp and time.time() > exp:
            return None
        return payload
    except Exception:
        return None

# ─── Helper Functions ─────────────────────────────────────────────────────────

def row_to_dict(row):
    if row is None:
        return None
    d = dict(row)
    # Parse JSON fields
    for key in ['tags', 'attendees', 'options']:
        if key in d and isinstance(d[key], str):
            try:
                d[key] = json.loads(d[key])
            except (json.JSONDecodeError, TypeError):
                pass
    return d

def rows_to_list(rows):
    return [row_to_dict(r) for r in rows]

def generate_id():
    return str(uuid.uuid4())[:8]

def now():
    return datetime.utcnow().isoformat() + "Z"

def deep_copy_json(value):
    return json.loads(json.dumps(value))

def parse_iso_utc(ts):
    if not ts or not isinstance(ts, str):
        return None
    try:
        if ts.endswith('Z'):
            ts = ts[:-1] + '+00:00'
        return datetime.fromisoformat(ts)
    except Exception:
        return None

def require_admin(user):
    return bool(user and user.get('role') == 'admin')

def log_audit(conn, user_id, entity_type, entity_id, action, changes=None):
    conn.execute(
        "INSERT INTO audit_log (id, user_id, entity_type, entity_id, action, changes) VALUES (?, ?, ?, ?, ?, ?)",
        (generate_id(), user_id, entity_type, entity_id, action, json.dumps(changes) if changes else None)
    )

def get_app_setting(conn, key, default_value=None):
    row = conn.execute("SELECT value_json FROM app_settings WHERE key = ?", (key,)).fetchone()
    if not row:
        return deep_copy_json(default_value) if default_value is not None else None
    try:
        return json.loads(row['value_json'])
    except Exception:
        return deep_copy_json(default_value) if default_value is not None else None

def set_app_setting(conn, key, value):
    payload = json.dumps(value)
    conn.execute("""
        INSERT INTO app_settings (key, value_json, updated_at)
        VALUES (?, ?, ?)
        ON CONFLICT(key) DO UPDATE SET value_json = excluded.value_json, updated_at = excluded.updated_at
    """, (key, payload, now()))

def load_backup_policy(conn):
    raw = get_app_setting(conn, BACKUP_POLICY_SETTING_KEY, DEFAULT_BACKUP_POLICY)
    if not isinstance(raw, dict):
        raw = {}
    policy = deep_copy_json(DEFAULT_BACKUP_POLICY)
    policy.update(raw)
    policy['enabled'] = bool(policy.get('enabled'))
    policy['interval_hours'] = max(1, min(168, int(policy.get('interval_hours') or 24)))
    policy['retention_days'] = max(1, min(365, int(policy.get('retention_days') or 30)))
    policy['max_backups'] = max(1, min(1000, int(policy.get('max_backups') or 60)))
    policy['last_run_at'] = policy.get('last_run_at') if isinstance(policy.get('last_run_at'), str) else None
    return policy

def save_backup_policy(conn, policy):
    normalized = {
        "enabled": bool(policy.get('enabled')),
        "interval_hours": max(1, min(168, int(policy.get('interval_hours') or 24))),
        "retention_days": max(1, min(365, int(policy.get('retention_days') or 30))),
        "max_backups": max(1, min(1000, int(policy.get('max_backups') or 60))),
        "last_run_at": policy.get('last_run_at') if isinstance(policy.get('last_run_at'), str) else None
    }
    set_app_setting(conn, BACKUP_POLICY_SETTING_KEY, normalized)
    return normalized

def _to_bool(value):
    if isinstance(value, bool):
        return value
    if isinstance(value, (int, float)):
        return value != 0
    if isinstance(value, str):
        v = value.strip().lower()
        return v in ('1', 'true', 'yes', 'y', 'on')
    return False

def _to_number(value):
    if value is None:
        return 0.0
    if isinstance(value, (int, float)):
        return float(value)
    if isinstance(value, str):
        cleaned = value.replace(',', '').replace('$', '').strip()
        if cleaned == '':
            return 0.0
        try:
            return float(cleaned)
        except Exception:
            return 0.0
    return 0.0

def _split_full_name(full_name):
    parts = [p for p in str(full_name or '').strip().split() if p]
    if not parts:
        return '', ''
    if len(parts) == 1:
        return parts[0], ''
    return parts[0], ' '.join(parts[1:])

def _normalize_text(value):
    return str(value or '').strip().lower()

def _parse_location_text(text):
    raw = str(text or '').strip()
    if not raw:
        return '', '', '', ''
    parts = [p.strip() for p in raw.split(',') if p.strip()]
    city = parts[0] if len(parts) >= 1 else ''
    state = parts[1] if len(parts) >= 2 else ''
    country = parts[2] if len(parts) >= 3 else ''
    return city, state, country, raw

def ensure_default_automation_rules(conn):
    defaults = [
        {
            "id": "auto-graveyard-route",
            "name": "Route Graveyard Investors",
            "trigger_type": "flag_change",
            "condition_json": {"field": "graveyard", "equals": True},
            "action_json": {"set_list": "graveyard"},
            "enabled": 1
        },
        {
            "id": "auto-followup-route",
            "name": "Route Follow-up Investors",
            "trigger_type": "flag_change",
            "condition_json": {"field": "follow_up", "equals": True},
            "action_json": {"set_list": "follow_up"},
            "enabled": 1
        }
    ]
    for r in defaults:
        conn.execute("""
            INSERT INTO fundraising_automation_rules (id, name, trigger_type, condition_json, action_json, enabled, updated_at)
            VALUES (?, ?, ?, ?, ?, ?, ?)
            ON CONFLICT(id) DO UPDATE SET
                name = excluded.name,
                trigger_type = excluded.trigger_type,
                condition_json = excluded.condition_json,
                action_json = excluded.action_json,
                updated_at = excluded.updated_at
        """, (r["id"], r["name"], r["trigger_type"], json.dumps(r["condition_json"]), json.dumps(r["action_json"]), r["enabled"], now()))

def run_fundraising_automations(conn):
    ensure_default_automation_rules(conn)
    rules = rows_to_list(conn.execute("SELECT * FROM fundraising_automation_rules WHERE enabled = 1 ORDER BY id").fetchall())
    investors = rows_to_list(conn.execute("SELECT * FROM fundraising_investors").fetchall())
    previous = {}
    for m in rows_to_list(conn.execute("SELECT investor_id, list_key FROM fundraising_list_memberships").fetchall()):
        previous.setdefault(m['investor_id'], set()).add(m['list_key'])

    desired = {}
    for inv in investors:
        inv_id = inv['id']
        lists = {'all'}
        if _to_bool(inv.get('graveyard')):
            lists.add('graveyard')
        else:
            lists.add('main')
        if _to_bool(inv.get('follow_up')):
            lists.add('follow_up')
        desired[inv_id] = lists

    conn.execute("DELETE FROM fundraising_list_memberships")
    for inv_id, lists in desired.items():
        for key in sorted(lists):
            conn.execute("""
                INSERT INTO fundraising_list_memberships (id, investor_id, list_key, source, updated_at)
                VALUES (?, ?, ?, 'automation', ?)
            """, (generate_id(), inv_id, key, now()))

    for inv in investors:
        inv_id = inv['id']
        before = previous.get(inv_id, set())
        after = desired.get(inv_id, set())
        if before == after:
            continue
        added = sorted(list(after - before))
        removed = sorted(list(before - after))
        if added or removed:
            conn.execute("""
                INSERT INTO fundraising_automation_runs (id, rule_id, investor_id, status, result_json, created_at)
                VALUES (?, ?, ?, 'applied', ?, ?)
            """, (
                generate_id(),
                None,
                inv_id,
                json.dumps({"lists_added": added, "lists_removed": removed}),
                now()
            ))

def _ensure_organization_by_name(conn, org_name, actor_user_id=None):
    name = str(org_name or '').strip()
    if not name:
        return None
    existing = conn.execute("SELECT id FROM organizations WHERE lower(name) = lower(?) LIMIT 1", (name,)).fetchone()
    if existing:
        return existing['id']
    org_id = generate_id()
    conn.execute(
        "INSERT INTO organizations (id, name, type, created_by, updated_at) VALUES (?, ?, 'other', ?, ?)",
        (org_id, name, actor_user_id, now())
    )
    return org_id

def _upsert_contact_from_fundraising(conn, investor_name, contact, actor_user_id=None):
    if not isinstance(contact, dict):
        return None
    full_name = str(contact.get('name') or '').strip()
    email = str(contact.get('email') or '').strip()
    title = str(contact.get('title') or '').strip()
    source = str(contact.get('source') or '').strip()
    city = str(contact.get('city') or '').strip()
    state = str(contact.get('state') or '').strip()
    country = str(contact.get('country') or '').strip()
    location_query = str(contact.get('location_query') or '').strip()
    if not full_name and not email:
        return None
    first_name, last_name = _split_full_name(full_name)
    if not first_name and email:
        first_name = email.split('@')[0]
    org_id = _ensure_organization_by_name(conn, investor_name, actor_user_id)

    existing = None
    if email:
        existing = conn.execute(
            "SELECT * FROM contacts WHERE lower(email) = lower(?) ORDER BY updated_at DESC LIMIT 1",
            (email,)
        ).fetchone()
    if not existing and first_name:
        if org_id:
            existing = conn.execute(
                """
                SELECT * FROM contacts
                WHERE lower(first_name) = lower(?) AND lower(last_name) = lower(?) AND organization_id = ?
                ORDER BY updated_at DESC LIMIT 1
                """,
                (first_name, last_name, org_id)
            ).fetchone()
        else:
            existing = conn.execute(
                """
                SELECT * FROM contacts
                WHERE lower(first_name) = lower(?) AND lower(last_name) = lower(?) AND organization_id IS NULL
                ORDER BY updated_at DESC LIMIT 1
                """,
                (first_name, last_name)
            ).fetchone()

    if existing:
        next_first = first_name or str(existing['first_name'] or '')
        next_last = last_name if (last_name or full_name) else str(existing['last_name'] or '')
        next_email = email or str(existing['email'] or '')
        next_title = title or str(existing['title'] or '')
        next_source = source or str(existing['source'] or '')
        next_city = city or str(existing['city'] or '')
        next_state = state or str(existing['state'] or '')
        next_country = country or str(existing['country'] or '')
        next_location_query = location_query or str(existing['location_query'] or '')
        next_org = org_id or existing['organization_id']
        conn.execute("""
            UPDATE contacts
            SET first_name = ?, last_name = ?, email = ?, title = ?,
                organization_id = ?, source = ?, contact_type = 'investor', city = ?, state = ?, country = ?, location_query = ?, updated_at = ?
            WHERE id = ?
        """, (next_first, next_last, next_email, next_title, next_org, next_source, next_city, next_state, next_country, next_location_query, now(), existing['id']))
        return existing['id']

    contact_id = generate_id()
    conn.execute("""
        INSERT INTO contacts (
            id, first_name, last_name, email, title, organization_id, source, contact_type, status, city, state, country, location_query, created_by, updated_at
        ) VALUES (?, ?, ?, ?, ?, ?, ?, 'investor', 'active', ?, ?, ?, ?, ?, ?)
    """, (
        contact_id,
        first_name or 'Unknown',
        last_name or '',
        email,
        title,
        org_id,
        source,
        city,
        state,
        country,
        location_query,
        actor_user_id,
        now()
    ))
    return contact_id

def _sync_contact_to_fundraising_state(conn, contact_row, actor_user_id=None, remove=False):
    if not isinstance(contact_row, dict):
        return
    row = conn.execute("SELECT * FROM fundraising_state WHERE id = 'main'").fetchone()
    if not row:
        return
    try:
        grid = json.loads(row['grid_json']) if row['grid_json'] else {}
    except Exception:
        grid = {}
    grid = sanitize_fundraising_grid(grid)
    columns = grid.get('columns', [])
    rows = grid.get('rows', [])
    if not isinstance(columns, list) or not isinstance(rows, list):
        return
    if not any(isinstance(c, dict) and c.get('id') == 'contacts' for c in columns):
        return

    org_name = str(contact_row.get('organization_name') or '').strip()
    email = str(contact_row.get('email') or '').strip()
    full_name = ' '.join([str(contact_row.get('first_name') or '').strip(), str(contact_row.get('last_name') or '').strip()]).strip()
    title = str(contact_row.get('title') or '').strip()
    source = str(contact_row.get('source') or '').strip()
    city = str(contact_row.get('city') or '').strip()
    state = str(contact_row.get('state') or '').strip()
    country = str(contact_row.get('country') or '').strip()
    location_query = str(contact_row.get('location_query') or '').strip()
    if not full_name and not email:
        return

    target_row_indexes = []
    org_norm = _normalize_text(org_name)
    for idx, inv in enumerate(rows):
        if not isinstance(inv, dict):
            continue
        if org_norm and _normalize_text(inv.get('investor_name')) == org_norm:
            target_row_indexes.append(idx)
    if not target_row_indexes:
        for idx, inv in enumerate(rows):
            if not isinstance(inv, dict):
                continue
            contacts = inv.get('contacts')
            if not isinstance(contacts, list):
                continue
            for c in contacts:
                if not isinstance(c, dict):
                    continue
                if email and _normalize_text(c.get('email')) == _normalize_text(email):
                    target_row_indexes.append(idx)
                    break
                if full_name and _normalize_text(c.get('name')) == _normalize_text(full_name):
                    target_row_indexes.append(idx)
                    break

    if not target_row_indexes:
        return

    changed = False
    email_norm = _normalize_text(email)
    name_norm = _normalize_text(full_name)
    for idx in target_row_indexes:
        inv = rows[idx]
        contacts = inv.get('contacts')
        if not isinstance(contacts, list):
            contacts = []
        next_contacts = list(contacts)
        match_index = -1
        for c_idx, c in enumerate(next_contacts):
            if not isinstance(c, dict):
                continue
            c_email_norm = _normalize_text(c.get('email'))
            c_name_norm = _normalize_text(c.get('name'))
            if email_norm and c_email_norm and c_email_norm == email_norm:
                match_index = c_idx
                break
            if name_norm and c_name_norm == name_norm:
                match_index = c_idx
                break

        if remove:
            if match_index >= 0:
                next_contacts.pop(match_index)
                inv['contacts'] = next_contacts
                changed = True
            continue

        if match_index >= 0:
            existing = next_contacts[match_index] if isinstance(next_contacts[match_index], dict) else {}
            next_contacts[match_index] = {
                **existing,
                "name": full_name or existing.get('name') or '',
                "email": email or existing.get('email') or '',
                "title": title or existing.get('title') or '',
                "city": city or str(existing.get('city') or ''),
                "state": state or str(existing.get('state') or ''),
                "country": country or str(existing.get('country') or ''),
                "location_query": location_query or str(existing.get('location_query') or '')
            }
        else:
            next_contacts.append({
                "name": full_name,
                "email": email,
                "title": title,
                "city": city,
                "state": state,
                "country": country,
                "location_query": location_query
            })
        inv['contacts'] = next_contacts
        if source and not str(inv.get('lead_source') or '').strip():
            inv['lead_source'] = source
        changed = True

    if not changed:
        return

    next_views = []
    try:
        next_views = json.loads(row['views_json']) if row['views_json'] else []
    except Exception:
        next_views = []
    next_views = sanitize_grid_views(next_views)
    next_version = int(row['version'] or 1) + 1
    conn.execute("""
        UPDATE fundraising_state
        SET grid_json = ?, views_json = ?, version = ?, updated_by = ?, updated_at = ?
        WHERE id = 'main'
    """, (json.dumps(grid), json.dumps(next_views), next_version, actor_user_id, now()))
    sync_fundraising_relational(conn, grid, next_views, actor_user_id=actor_user_id)

def sync_fundraising_relational(conn, grid, views, actor_user_id=None):
    columns = grid.get('columns', []) if isinstance(grid, dict) else []
    rows = grid.get('rows', []) if isinstance(grid, dict) else []
    views = views if isinstance(views, list) else []

    fund_columns = []
    for idx, col in enumerate(columns):
        if not isinstance(col, dict):
            continue
        col_id = str(col.get('id') or '').strip()
        if not col_id:
            continue
        is_fund = bool(col.get('isFund')) or col.get('type') == 'currency'
        if is_fund:
            fund_columns.append((idx, col))

    seen_fund_col_ids = set()
    fund_id_by_col = {}
    for idx, col in fund_columns:
        col_id = str(col.get('id'))
        label = str(col.get('label') or col_id).strip()
        seen_fund_col_ids.add(col_id)
        existing = conn.execute("SELECT id FROM fundraising_funds WHERE column_id = ?", (col_id,)).fetchone()
        fund_id = existing['id'] if existing else generate_id()
        conn.execute("""
            INSERT INTO fundraising_funds (id, column_id, fund_name, display_order, active, updated_at)
            VALUES (?, ?, ?, ?, 1, ?)
            ON CONFLICT(column_id) DO UPDATE SET
                fund_name = excluded.fund_name,
                display_order = excluded.display_order,
                active = 1,
                updated_at = excluded.updated_at
        """, (fund_id, col_id, label, idx, now()))
        fund_id_by_col[col_id] = fund_id

    if seen_fund_col_ids:
        placeholders = ','.join(['?'] * len(seen_fund_col_ids))
        conn.execute(f"UPDATE fundraising_funds SET active = 0, updated_at = ? WHERE column_id NOT IN ({placeholders})", [now(), *list(seen_fund_col_ids)])
    else:
        conn.execute("UPDATE fundraising_funds SET active = 0, updated_at = ?", (now(),))

    seen_source_row_ids = set()
    for row in rows:
        if not isinstance(row, dict):
            continue
        source_row_id = str(row.get('id') or '').strip()
        if not source_row_id:
            continue
        seen_source_row_ids.add(source_row_id)
        investor_name = str(row.get('investor_name') or '').strip() or 'Untitled Investor'
        notes = str(row.get('notes') or '')
        lead = str(row.get('lead') or '')
        lead_source = str(row.get('lead_source') or row.get('combined_lead_source') or '').strip()
        total_invested = 0.0
        for _, col in fund_columns:
            total_invested += _to_number(row.get(str(col.get('id'))))

        existing = conn.execute("SELECT id FROM fundraising_investors WHERE source_row_id = ?", (source_row_id,)).fetchone()
        investor_id = existing['id'] if existing else generate_id()
        conn.execute("""
            INSERT INTO fundraising_investors (
                id, investor_name, notes, lead, lead_source, priority, follow_up, graveyard,
                source_row_id, total_invested, updated_at
            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
            ON CONFLICT(source_row_id) DO UPDATE SET
                investor_name = excluded.investor_name,
                notes = excluded.notes,
                lead = excluded.lead,
                lead_source = CASE
                    WHEN COALESCE(trim(fundraising_investors.lead_source), '') = '' THEN excluded.lead_source
                    ELSE fundraising_investors.lead_source
                END,
                priority = excluded.priority,
                follow_up = excluded.follow_up,
                graveyard = excluded.graveyard,
                total_invested = excluded.total_invested,
                updated_at = excluded.updated_at
        """, (
            investor_id,
            investor_name,
            notes,
            lead,
            lead_source,
            1 if _to_bool(row.get('priority')) else 0,
            1 if _to_bool(row.get('follow_up')) else 0,
            1 if _to_bool(row.get('graveyard')) else 0,
            source_row_id,
            total_invested,
            now()
        ))

        fresh = conn.execute("SELECT id FROM fundraising_investors WHERE source_row_id = ?", (source_row_id,)).fetchone()
        investor_id = fresh['id']

        conn.execute("DELETE FROM fundraising_contacts WHERE investor_id = ?", (investor_id,))
        contacts = row.get('contacts')
        if isinstance(contacts, list):
            for i, c in enumerate(contacts):
                if not isinstance(c, dict):
                    continue
                full_name = str(c.get('name') or '').strip()
                email = str(c.get('email') or '').strip()
                if not full_name and not email:
                    continue
                contact_payload = dict(c)
                if lead_source and not str(contact_payload.get('source') or '').strip():
                    contact_payload['source'] = lead_source
                _upsert_contact_from_fundraising(conn, investor_name, contact_payload, actor_user_id=actor_user_id)
                conn.execute("""
                    INSERT INTO fundraising_contacts (
                        id, investor_id, full_name, email, title, city, state, country, location_query, sort_order, updated_at
                    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
                """, (
                    generate_id(), investor_id, full_name, email, str(c.get('title') or ''),
                    str(c.get('city') or ''), str(c.get('state') or ''), str(c.get('country') or ''),
                    str(c.get('location_query') or ''), i, now()
                ))
        elif isinstance(contacts, str) and contacts.strip():
            _upsert_contact_from_fundraising(conn, investor_name, {"name": contacts.strip(), "email": "", "title": "", "source": lead_source}, actor_user_id=actor_user_id)
            conn.execute("""
                INSERT INTO fundraising_contacts (
                    id, investor_id, full_name, email, title, city, state, country, location_query, sort_order, updated_at
                ) VALUES (?, ?, ?, '', '', '', '', '', '', 0, ?)
            """, (generate_id(), investor_id, contacts.strip(), now()))

        conn.execute("DELETE FROM fundraising_commitments WHERE investor_id = ?", (investor_id,))
        for _, col in fund_columns:
            col_id = str(col.get('id'))
            fund_id = fund_id_by_col.get(col_id)
            if not fund_id:
                continue
            amount = _to_number(row.get(col_id))
            if abs(amount) < 1e-9:
                continue
            conn.execute("""
                INSERT INTO fundraising_commitments (id, investor_id, fund_id, amount, updated_at)
                VALUES (?, ?, ?, ?, ?)
            """, (generate_id(), investor_id, fund_id, amount, now()))

    if seen_source_row_ids:
        placeholders = ','.join(['?'] * len(seen_source_row_ids))
        conn.execute(f"DELETE FROM fundraising_investors WHERE source_row_id NOT IN ({placeholders})", list(seen_source_row_ids))
    else:
        conn.execute("DELETE FROM fundraising_investors")

    conn.execute("DELETE FROM fundraising_views")
    for v in views:
        if not isinstance(v, dict):
            continue
        view_id = str(v.get('id') or '').strip()
        if not view_id:
            continue
        conn.execute("""
            INSERT INTO fundraising_views (
                id, name, filters_json, quick_search, hidden_columns_json, column_filters_json, updated_at
            ) VALUES (?, ?, ?, ?, ?, ?, ?)
        """, (
            view_id,
            str(v.get('name') or view_id),
            json.dumps(v.get('filters') if isinstance(v.get('filters'), dict) else {}),
            str(v.get('quickSearch') or ''),
            json.dumps(v.get('hiddenColumns') if isinstance(v.get('hiddenColumns'), list) else []),
            json.dumps(v.get('columnFilters') if isinstance(v.get('columnFilters'), list) else []),
            now()
        ))
    run_fundraising_automations(conn)

def ensure_fundraising_state_row(conn):
    existing = conn.execute("SELECT * FROM fundraising_state WHERE id = 'main'").fetchone()
    if not existing:
        default_grid = {
            "columns": deep_copy_json(DEFAULT_FUNDRAISING_COLUMNS),
            "rows": deep_copy_json(DEFAULT_FUNDRAISING_ROWS)
        }
        default_views = sanitize_grid_views(deep_copy_json(DEFAULT_GRID_VIEWS))
        conn.execute("""
            INSERT INTO fundraising_state (id, grid_json, views_json, version, updated_by)
            VALUES ('main', ?, ?, 1, NULL)
        """, (json.dumps(default_grid), json.dumps(default_views)))
        sync_fundraising_relational(conn, default_grid, default_views)
        conn.commit()
        return

    try:
        grid = json.loads(existing['grid_json']) if existing['grid_json'] else {}
    except Exception:
        grid = {}
    grid = sanitize_fundraising_grid(grid)
    try:
        views = json.loads(existing['views_json']) if existing['views_json'] else []
    except Exception:
        views = []
    rel_count = conn.execute("SELECT COUNT(*) AS c FROM fundraising_investors").fetchone()['c']
    if rel_count == 0:
        sync_fundraising_relational(conn, grid, sanitize_grid_views(views if isinstance(views, list) else deep_copy_json(DEFAULT_GRID_VIEWS)))
        conn.commit()

def list_backups():
    backup_dir = os.path.join(DATA_DIR, "backups")
    os.makedirs(backup_dir, exist_ok=True)
    entries = []
    for name in os.listdir(backup_dir):
        if not name.endswith('.json'):
            continue
        path = os.path.join(backup_dir, name)
        if not os.path.isfile(path):
            continue
        st = os.stat(path)
        if "pre_restore" in name:
            kind = "pre_restore"
        elif "_auto_" in name:
            kind = "auto"
        else:
            kind = "backup"
        entries.append({
            "filename": name,
            "path": path,
            "size_bytes": st.st_size,
            "modified_at": datetime.utcfromtimestamp(st.st_mtime).isoformat() + "Z",
            "kind": kind
        })
    entries.sort(key=lambda x: x["modified_at"], reverse=True)
    return entries

def create_fundraising_backup_file(state_row, kind="backup"):
    backup_dir = os.path.join(DATA_DIR, "backups")
    os.makedirs(backup_dir, exist_ok=True)
    ts = datetime.utcnow().strftime('%Y%m%dT%H%M%SZ')
    if kind == "pre_restore":
        filename = f"fundraising_state_pre_restore_{ts}.json"
    elif kind == "auto":
        filename = f"fundraising_state_auto_{ts}.json"
    else:
        filename = f"fundraising_state_{ts}.json"
    path = os.path.join(backup_dir, filename)
    try:
        grid = json.loads(state_row['grid_json']) if state_row and state_row['grid_json'] else {}
    except Exception:
        grid = {}
    try:
        views = json.loads(state_row['views_json']) if state_row and state_row['views_json'] else []
    except Exception:
        views = []
    payload = {
        "backup_at": now(),
        "version": state_row['version'] if state_row else 1,
        "updated_at": state_row['updated_at'] if state_row else None,
        "grid": grid,
        "views": views
    }
    with open(path, 'w', encoding='utf-8') as f:
        json.dump(payload, f, ensure_ascii=True, indent=2)
    return {
        "filename": filename,
        "path": path,
        "version": payload["version"],
        "kind": kind
    }

def apply_backup_retention(policy):
    entries = [b for b in list_backups() if b.get('kind') in ('backup', 'auto')]
    keep = max(1, int(policy.get('max_backups') or 60))
    cutoff = datetime.utcnow() - timedelta(days=max(1, int(policy.get('retention_days') or 30)))

    for b in entries:
        ts = parse_iso_utc(b.get('modified_at'))
        if ts and ts.replace(tzinfo=None) < cutoff and b.get('kind') == 'auto':
            try:
                os.remove(b['path'])
            except Exception:
                pass

    entries = [b for b in list_backups() if b.get('kind') in ('backup', 'auto')]
    for stale in entries[keep:]:
        if stale.get('kind') != 'auto':
            continue
        try:
            os.remove(stale['path'])
        except Exception:
            pass

def compute_restore_diff(current_grid, current_views, next_grid, next_views):
    current_columns = current_grid.get('columns', []) if isinstance(current_grid, dict) else []
    next_columns = next_grid.get('columns', []) if isinstance(next_grid, dict) else []
    current_rows = current_grid.get('rows', []) if isinstance(current_grid, dict) else []
    next_rows = next_grid.get('rows', []) if isinstance(next_grid, dict) else []

    curr_col_map = {str(c.get('id')): c for c in current_columns if isinstance(c, dict) and c.get('id')}
    next_col_map = {str(c.get('id')): c for c in next_columns if isinstance(c, dict) and c.get('id')}
    curr_row_map = {str(r.get('id')): r for r in current_rows if isinstance(r, dict) and r.get('id')}
    next_row_map = {str(r.get('id')): r for r in next_rows if isinstance(r, dict) and r.get('id')}

    added_columns = [next_col_map[k] for k in next_col_map.keys() - curr_col_map.keys()]
    removed_columns = [curr_col_map[k] for k in curr_col_map.keys() - next_col_map.keys()]

    changed_column_types = []
    for key in curr_col_map.keys() & next_col_map.keys():
        before_type = curr_col_map[key].get('type')
        after_type = next_col_map[key].get('type')
        if before_type != after_type:
            changed_column_types.append({
                "id": key,
                "label": next_col_map[key].get('label') or curr_col_map[key].get('label') or key,
                "before_type": before_type,
                "after_type": after_type
            })

    common_cols = list(curr_col_map.keys() & next_col_map.keys())
    cell_changes_count = 0
    changed_rows = set()
    for rid in curr_row_map.keys() & next_row_map.keys():
        before = curr_row_map[rid]
        after = next_row_map[rid]
        row_changed = False
        for cid in common_cols:
            if before.get(cid) != after.get(cid):
                cell_changes_count += 1
                row_changed = True
        if row_changed:
            changed_rows.add(rid)

    curr_view_ids = {str(v.get('id')) for v in current_views if isinstance(v, dict) and v.get('id')}
    next_view_ids = {str(v.get('id')) for v in next_views if isinstance(v, dict) and v.get('id')}
    curr_view_names = {str(v.get('id')): v.get('name') for v in current_views if isinstance(v, dict) and v.get('id')}
    next_view_names = {str(v.get('id')): v.get('name') for v in next_views if isinstance(v, dict) and v.get('id')}

    return {
        "columns_added": [{"id": c.get('id'), "label": c.get('label')} for c in added_columns[:25]],
        "columns_removed": [{"id": c.get('id'), "label": c.get('label')} for c in removed_columns[:25]],
        "columns_type_changed": changed_column_types[:25],
        "rows_added_count": len(next_row_map.keys() - curr_row_map.keys()),
        "rows_removed_count": len(curr_row_map.keys() - next_row_map.keys()),
        "rows_changed_count": len(changed_rows),
        "cell_changes_count": cell_changes_count,
        "views_added": [{"id": i, "name": next_view_names.get(i)} for i in list(next_view_ids - curr_view_ids)[:25]],
        "views_removed": [{"id": i, "name": curr_view_names.get(i)} for i in list(curr_view_ids - next_view_ids)[:25]]
    }

def sanitize_grid_views(views):
    if not isinstance(views, list):
        return deep_copy_json(DEFAULT_GRID_VIEWS)
    out = []
    for view in views:
        if not isinstance(view, dict) or str(view.get('id') or '') == 'view-longshot':
            continue
        next_view = deep_copy_json(view)
        filters = next_view.get('filters')
        if isinstance(filters, dict) and 'longshotOnly' in filters:
            filters.pop('longshotOnly', None)
        if not isinstance(filters, dict):
            filters = {}
            next_view['filters'] = filters
        view_id = str(next_view.get('id') or '')
        if view_id == 'view-graveyard':
            filters['graveyardOnly'] = True
            filters['includeGraveyard'] = True
        else:
            filters['graveyardOnly'] = bool(filters.get('graveyardOnly', False))
        if view_id == 'view-followup':
            filters['followUpOnly'] = True
        else:
            filters['followUpOnly'] = bool(filters.get('followUpOnly', False))
        out.append(next_view)
    return out if out else deep_copy_json(DEFAULT_GRID_VIEWS)

def sanitize_fundraising_grid(grid):
    if not isinstance(grid, dict):
        return {
            "columns": deep_copy_json(DEFAULT_FUNDRAISING_COLUMNS),
            "rows": deep_copy_json(DEFAULT_FUNDRAISING_ROWS)
        }
    columns = grid.get('columns')
    rows = grid.get('rows')
    if not isinstance(columns, list):
        columns = deep_copy_json(DEFAULT_FUNDRAISING_COLUMNS)
    if not isinstance(rows, list):
        rows = deep_copy_json(DEFAULT_FUNDRAISING_ROWS)

    clean_columns = []
    seen = set()
    for col in columns:
        if not isinstance(col, dict):
            continue
        col_id = str(col.get('id') or '').strip()
        if not col_id or col_id == 'longshot_followup' or col_id in seen:
            continue
        seen.add(col_id)
        clean_columns.append(col)

    clean_rows = []
    for row in rows:
        if not isinstance(row, dict):
            continue
        next_row = dict(row)
        next_row.pop('longshot_followup', None)
        clean_rows.append(next_row)

    return {"columns": clean_columns, "rows": clean_rows}

def maybe_run_scheduled_backup():
    conn = get_db()
    try:
        ensure_fundraising_state_row(conn)
        policy = load_backup_policy(conn)
        if not policy.get('enabled'):
            return
        interval = timedelta(hours=int(policy.get('interval_hours') or 24))
        last_run = parse_iso_utc(policy.get('last_run_at'))
        current = datetime.utcnow()
        if last_run and (current - last_run.replace(tzinfo=None)) < interval:
            return
        row = conn.execute("SELECT * FROM fundraising_state WHERE id = 'main'").fetchone()
        if not row:
            return
        create_fundraising_backup_file(row, kind="auto")
        policy['last_run_at'] = now()
        save_backup_policy(conn, policy)
        apply_backup_retention(policy)
        conn.commit()
    finally:
        conn.close()

def start_backup_scheduler():
    def _loop():
        while True:
            try:
                maybe_run_scheduled_backup()
            except Exception as exc:
                print(f"[scheduler] backup check failed: {exc}")
            time.sleep(60)
    thread = threading.Thread(target=_loop, daemon=True)
    thread.start()

# ─── Request Handler ──────────────────────────────────────────────────────────

PIPELINE_STAGES = ['lead', 'outreach', 'meeting', 'due_diligence', 'committed', 'funded']
CONTACT_TYPES = ['investor', 'prospect', 'advisor', 'other']
COMM_TYPES = ['email', 'call', 'meeting', 'note', 'text']

DEFAULT_GRID_VIEWS = [
    {"id": "view-main", "name": "Main Fundraising", "filters": {"includeGraveyard": False, "graveyardOnly": False, "followUpOnly": False, "lead": ""}, "quickSearch": "", "hiddenColumns": [], "columnFilters": []},
    {"id": "view-followup", "name": "Follow-up List", "filters": {"includeGraveyard": False, "graveyardOnly": False, "followUpOnly": True, "lead": ""}, "quickSearch": "", "hiddenColumns": [], "columnFilters": []},
    {"id": "view-graveyard", "name": "Graveyard", "filters": {"includeGraveyard": True, "graveyardOnly": True, "followUpOnly": False, "lead": ""}, "quickSearch": "", "hiddenColumns": [], "columnFilters": []},
    {"id": "view-all", "name": "All Investors", "filters": {"includeGraveyard": True, "graveyardOnly": False, "followUpOnly": False, "lead": ""}, "quickSearch": "", "hiddenColumns": [], "columnFilters": []}
]

DEFAULT_FUNDRAISING_COLUMNS = [
    {"id": "investor_name", "label": "Investor Name", "type": "text", "width": 220},
    {"id": "contacts", "label": "Contacts", "type": "contacts", "width": 260},
    {"id": "log_action", "label": "Log", "type": "action", "readOnly": True, "width": 90},
    {"id": "notes", "label": "Notes / Communication / Outreach", "type": "longtext", "width": 420},
    {"id": "lead_source", "label": "Lead Source", "type": "text", "width": 180},
    {"id": "notes_last_modified", "label": "Notes Last Modified", "type": "date", "readOnly": True, "width": 180},
    {"id": "last_communication_date", "label": "Last Communication Date", "type": "date", "readOnly": True, "width": 195},
    {"id": "priority", "label": "Priority", "type": "checkbox", "width": 110},
    {"id": "follow_up", "label": "Follow up", "type": "checkbox", "width": 110},
    {"id": "lead", "label": "Lead", "type": "select", "options": ["JK", "Grant", "MB", "Parker", "Other"], "width": 130},
    {"id": "graveyard", "label": "Graveyard", "type": "checkbox", "width": 115},
    {"id": "fund_i", "label": "Fund I", "type": "currency", "isFund": True, "width": 130},
    {"id": "fund_ii", "label": "Fund II", "type": "currency", "isFund": True, "width": 130},
    {"id": "fund_iii", "label": "Fund III", "type": "currency", "isFund": True, "width": 130},
    {"id": "tactical_fund", "label": "Tactical Fund", "type": "currency", "isFund": True, "width": 140},
    {"id": "pawn_to_e4", "label": "Pawn to E4", "type": "currency", "isFund": True, "width": 130},
    {"id": "ten31_terahash", "label": "Ten31 Terahash", "type": "currency", "isFund": True, "width": 150},
    {"id": "sats_and_stats", "label": "Sats and Stats", "type": "currency", "isFund": True, "width": 140},
    {"id": "pawn_to_f4", "label": "Pawn to f4", "type": "currency", "isFund": True, "width": 130},
    {"id": "join_the_fold", "label": "Join the Fold", "type": "currency", "isFund": True, "width": 130},
    {"id": "total_invested", "label": "Total invested", "type": "rollup", "readOnly": True, "width": 150},
    {"id": "tactical_fund_commit_date", "label": "Tactical Fund Commit Date", "type": "date", "width": 180}
]

DEFAULT_FUNDRAISING_ROWS = []

class CRMHandler(BaseHTTPRequestHandler):
    """Main HTTP request handler for the CRM API."""
    # Class-level state shared across all handler threads. Protected by
    # _abuse_lock; see rate_limited() and record_404() for usage.
    _rate_limit_buckets = {}   # (scope, ip) -> [timestamps]
    _404_buckets = {}          # ip -> [timestamps] of recent 404 responses
    _banned_ips = {}           # ip -> ban_until_epoch
    _abuse_lock = threading.Lock()

    def log_message(self, format, *args):
        """Override to use cleaner logging."""
        sys.stderr.write(f"[{datetime.now().strftime('%H:%M:%S')}] {args[0]}\n")

    # ── Request Parsing ──

    def get_body(self):
        # Cache parsed body on the request handler instance so repeated
        # calls don't try to re-read an already-consumed stream. Handler
        # instances are one-per-request in ThreadingHTTPServer, so the
        # cache is naturally request-scoped.
        if hasattr(self, '_cached_body'):
            return self._cached_body
        content_length = int(self.headers.get('Content-Length', 0))
        if content_length == 0:
            self._cached_body = {}
            return self._cached_body
        body = self.rfile.read(content_length)
        try:
            self._cached_body = json.loads(body.decode('utf-8'))
        except json.JSONDecodeError:
            self._cached_body = {}
        return self._cached_body

    def get_query_params(self):
        parsed = urlparse(self.path)
        return {k: v[0] if len(v) == 1 else v for k, v in parse_qs(parsed.query).items()}

    def get_path(self):
        return urlparse(self.path).path

    def get_user(self):
        auth = self.headers.get('Authorization', '')
        if not auth.startswith('Bearer '):
            return None
        token = auth[7:]
        payload = decode_token(token)
        if not payload:
            return None
        user_id = str(payload.get('user_id') or '').strip()
        if not user_id:
            return None
        conn = get_db()
        row = conn.execute(
            "SELECT id as user_id, username, role, is_active FROM users WHERE id = ?",
            (user_id,)
        ).fetchone()
        conn.close()
        if not row or int(row['is_active'] or 0) != 1:
            return None
        return {
            "user_id": row['user_id'],
            "username": row['username'],
            "role": row['role']
        }

    def get_client_ip(self):
        fwd = self.headers.get('X-Forwarded-For', '')
        if fwd:
            return fwd.split(',')[0].strip()
        return str(self.client_address[0])

    def rate_limited(self, scope, limit_per_minute):
        now_ts = time.time()
        bucket_key = f"{scope}:{self.get_client_ip()}"
        with self._abuse_lock:
            bucket = self._rate_limit_buckets.get(bucket_key, [])
            cutoff = now_ts - 60.0
            bucket = [t for t in bucket if t >= cutoff]
            if len(bucket) >= max(1, int(limit_per_minute)):
                self._rate_limit_buckets[bucket_key] = bucket
                return True
            bucket.append(now_ts)
            self._rate_limit_buckets[bucket_key] = bucket
            return False

    def is_banned(self):
        """Return True if the client IP is currently in the abuse blacklist.

        Uses a coarse class-level dict — fine for a handful of scanners hitting
        a small team CRM. Auto-expires entries when their ban window passes.
        """
        ip = self.get_client_ip()
        now_ts = time.time()
        with self._abuse_lock:
            until = self._banned_ips.get(ip)
            if until is None:
                return False
            if now_ts >= until:
                self._banned_ips.pop(ip, None)
                return False
            return True

    def record_404(self):
        """Track 404s per IP and auto-ban IPs that exceed the burst threshold.

        Called from send_error_json whenever we send a 404. A scanner probing
        /.env, /.git/config, /swagger, /actuator/env etc. will trip this fast
        and get parked on the blacklist for ABUSE_BAN_SEC seconds.
        """
        ip = self.get_client_ip()
        now_ts = time.time()
        with self._abuse_lock:
            bucket = self._404_buckets.get(ip, [])
            cutoff = now_ts - ABUSE_404_WINDOW_SEC
            bucket = [t for t in bucket if t >= cutoff]
            bucket.append(now_ts)
            if len(bucket) >= ABUSE_404_THRESHOLD:
                self._banned_ips[ip] = now_ts + ABUSE_BAN_SEC
                self._404_buckets.pop(ip, None)
                sys.stderr.write(
                    f"[abuse] Banning {ip} for {ABUSE_BAN_SEC}s after "
                    f"{len(bucket)} 404s in {ABUSE_404_WINDOW_SEC}s\n"
                )
            else:
                self._404_buckets[ip] = bucket

    # ── Response Helpers ──

    def send_json(self, data, status=200):
        self.send_response(status)
        self.send_header('Content-Type', 'application/json')
        self.send_header('Access-Control-Allow-Origin', CORS_ORIGIN)
        self.send_header('Access-Control-Allow-Methods', 'GET, POST, PUT, PATCH, DELETE, OPTIONS')
        self.send_header('Access-Control-Allow-Headers', 'Content-Type, Authorization')
        self.end_headers()
        self.wfile.write(json.dumps(data, default=str).encode('utf-8'))

    def send_error_json(self, message, status=400):
        # Record 404s for the abuser-detection blacklist before responding.
        # We do this on send (not on routing) so that any code path that 404s
        # contributes to the burst counter, including unknown POST paths.
        if status == 404:
            try:
                self.record_404()
            except Exception:
                pass
        self.send_json({"error": message}, status)

    def send_file(self, filepath, content_type='text/html'):
        try:
            with open(filepath, 'rb') as f:
                content = f.read()
            self.send_response(200)
            self.send_header('Content-Type', content_type)
            self.send_header('Content-Length', str(len(content)))
            self.end_headers()
            self.wfile.write(content)
        except FileNotFoundError:
            self.send_error_json("File not found", 404)

    # ── Routing ──

    def do_OPTIONS(self):
        if self.is_banned():
            return self.send_error_json("Too many requests", 429)
        self.send_response(200)
        self.send_header('Access-Control-Allow-Origin', CORS_ORIGIN)
        self.send_header('Access-Control-Allow-Methods', 'GET, POST, PUT, PATCH, DELETE, OPTIONS')
        self.send_header('Access-Control-Allow-Headers', 'Content-Type, Authorization')
        self.end_headers()

    def do_GET(self):
        # Short-circuit known abusers before any work, including file reads.
        if self.is_banned():
            return self.send_error_json("Too many requests", 429)
        # Generic per-IP GET rate limit. Generous enough for a normal user
        # (page load fans out ~15 GETs + heartbeats every ~6s) but blocks a
        # scanner blasting hundreds of paths per second.
        if self.rate_limited('get', GET_RATE_LIMIT_PER_MIN):
            return self.send_error_json("Too many requests", 429)

        path = self.get_path()

        # ─── Gmail integration routes (feature-flag-guarded) ─────────
        try:
            from email_integration.routes import try_handle as _email_try_handle
            if _email_try_handle(self):
                return
        except ImportError:
            pass

        # Serve frontend
        if path == '/' or path == '/index.html':
            return self.send_file(os.path.join(FRONTEND_DIR, 'index.html'))
        if path.startswith('/assets/'):
            filepath = os.path.join(FRONTEND_DIR, path.lstrip('/'))
            ext = os.path.splitext(path)[1].lower()
            content_types = {
                '.css': 'text/css',
                '.js': 'application/javascript',
                '.svg': 'image/svg+xml',
                '.png': 'image/png',
                '.jpg': 'image/jpeg',
                '.jpeg': 'image/jpeg',
                '.gif': 'image/gif',
                '.webp': 'image/webp',
                '.ico': 'image/x-icon',
            }
            ct = content_types.get(ext, 'application/octet-stream')
            return self.send_file(filepath, ct)

        # API routes
        if path == '/api/health':
            return self.send_json({"status": "ok", "timestamp": now()})
        if path == '/api/bootstrap/status':
            return self.handle_bootstrap_status()

        # Auth required from here
        user = self.get_user()
        if not user and path.startswith('/api/'):
            return self.send_error_json("Authentication required", 401)

        params = self.get_query_params()

        # Contacts
        if path == '/api/contacts':
            return self.handle_list_contacts(user, params)
        if re.match(r'^/api/contacts/[^/]+$', path):
            contact_id = path.split('/')[-1]
            return self.handle_get_contact(user, contact_id)
        if re.match(r'^/api/contacts/[^/]+/communications$', path):
            contact_id = path.split('/')[-2]
            return self.handle_list_contact_communications(user, contact_id, params)

        # Organizations
        if path == '/api/organizations':
            return self.handle_list_organizations(user, params)
        if re.match(r'^/api/organizations/[^/]+$', path):
            org_id = path.split('/')[-1]
            return self.handle_get_organization(user, org_id)

        # Opportunities
        if path == '/api/opportunities':
            return self.handle_list_opportunities(user, params)
        if re.match(r'^/api/opportunities/[^/]+$', path):
            opp_id = path.split('/')[-1]
            return self.handle_get_opportunity(user, opp_id)

        # Communications
        if path == '/api/communications':
            return self.handle_list_communications(user, params)

        # LP Profiles
        if path == '/api/lp-profiles':
            return self.handle_list_lp_profiles(user, params)
        if re.match(r'^/api/lp-profiles/[^/]+$', path):
            lp_id = path.split('/')[-1]
            return self.handle_get_lp_profile(user, lp_id)

        # Reports
        if path == '/api/reports/dashboard':
            return self.handle_dashboard_report(user)
        if path == '/api/reports/pipeline':
            return self.handle_pipeline_report(user)
        if path == '/api/reports/lp-breakdown':
            return self.handle_lp_breakdown_report(user)
        if path == '/api/reports/activity':
            return self.handle_activity_report(user, params)

        # Export
        if path == '/api/export/contacts':
            return self.handle_export_contacts(user, params)

        # Feature requests
        if path == '/api/feature-requests':
            return self.handle_list_feature_requests(user, params)

        # Fundraising grid state
        if path == '/api/fundraising/state':
            return self.handle_get_fundraising_state(user)
        if path == '/api/fundraising/collab/state':
            return self.handle_get_fundraising_collab_state(user)
        if path == '/api/fundraising/export':
            return self.handle_export_fundraising_state(user)
        if path == '/api/fundraising/backups':
            return self.handle_list_fundraising_backups(user)
        if path == '/api/fundraising/backup-policy':
            return self.handle_get_backup_policy(user)
        if path == '/api/fundraising/relational-summary':
            return self.handle_get_fundraising_relational_summary(user)
        if path == '/api/fundraising/automations':
            return self.handle_list_fundraising_automations(user)
        if path == '/api/fundraising/automation-runs':
            return self.handle_list_fundraising_automation_runs(user, params)
        if path == '/api/fundraising/activity':
            return self.handle_get_fundraising_activity(user, params)
        if path == '/api/security/status':
            return self.handle_security_status(user)
        if path == '/api/system/status':
            return self.handle_system_status(user)

        # Users
        if path == '/api/users':
            return self.handle_list_users(user)

        # Audit log
        if path == '/api/audit-log':
            return self.handle_list_audit_log(user, params)

        # ─── Architect thesis (Phase 1) ───
        if path == '/api/thesis/lines':
            return self.handle_list_thesis_lines(user)
        if path == '/api/thesis/versions':
            return self.handle_list_thesis_review_queue(user)
        if re.match(r'^/api/thesis/versions/[^/]+$', path):
            return self.handle_get_thesis_version(user, path.split('/')[-1])
        if re.match(r'^/api/thesis/[^/]+/canonical$', path):
            return self.handle_get_canonical_thesis(user, path.split('/')[-2])

        # ─── Entity-merge review queue ───
        if path == '/api/entities/merge-candidates':
            return self.handle_list_merge_candidates(user, params)

        self.send_error_json("Not found", 404)

    def do_POST(self):
        if self.is_banned():
            return self.send_error_json("Too many requests", 429)

        path = self.get_path()
        body = self.get_body()

        if self.rate_limited('write', WRITE_RATE_LIMIT_PER_MIN):
            return self.send_error_json("Too many requests", 429)

        # ─── Gmail integration routes (feature-flag-guarded) ─────────
        try:
            from email_integration.routes import try_handle as _email_try_handle
            if _email_try_handle(self):
                return
        except ImportError:
            pass

        # Auth (no token needed)
        if path == '/api/auth/login':
            if self.rate_limited('login', LOGIN_RATE_LIMIT_PER_MIN):
                return self.send_error_json("Too many login attempts", 429)
            return self.handle_login(body)
        if path == '/api/auth/register':
            return self.handle_register(body)

        # Auth required
        user = self.get_user()
        if not user:
            return self.send_error_json("Authentication required", 401)

        if path == '/api/contacts':
            return self.handle_create_contact(user, body)
        if path == '/api/organizations':
            return self.handle_create_organization(user, body)
        if path == '/api/opportunities':
            return self.handle_create_opportunity(user, body)
        if path == '/api/communications':
            return self.handle_create_communication(user, body)
        if path == '/api/lp-profiles':
            return self.handle_create_lp_profile(user, body)
        if path == '/api/import/csv':
            return self.handle_import_csv(user, body)
        if path == '/api/feature-requests':
            return self.handle_create_feature_request(user, body)
        if path == '/api/fundraising/log-communication':
            return self.handle_log_fundraising_communication(user, body)
        if path == '/api/fundraising/collab/heartbeat':
            return self.handle_fundraising_collab_heartbeat(user, body)
        if path == '/api/admin/users':
            return self.handle_admin_create_user(user, body)
        if path == '/api/admin/reset-all-data':
            return self.handle_admin_reset_all_data(user, body)
        if path == '/api/fundraising/backup':
            return self.handle_backup_fundraising_state(user)
        if path == '/api/fundraising/restore-preview':
            return self.handle_preview_fundraising_restore(user, body)
        if path == '/api/fundraising/restore':
            return self.handle_restore_fundraising_state(user, body)
        if path == '/api/fundraising/backup-verify':
            return self.handle_verify_fundraising_backups(user)

        # ─── Architect thesis review (Phase 1, human approval gate) ───
        if re.match(r'^/api/thesis/versions/[^/]+/review$', path):
            return self.handle_thesis_review(user, path.split('/')[-2], body)

        # ─── UI-triggered index jobs + entity-merge decisions (Phase 1) ───
        if path == '/api/index/rebuild':
            return self.handle_index_job(user, 'rebuild_index')
        if path == '/api/index/update':
            return self.handle_index_job(user, 'update_index')
        if path == '/api/entities/find-duplicates':
            return self.handle_index_job(user, 'find_duplicates')
        if re.match(r'^/api/entities/merge-candidates/[^/]+$', path):
            return self.handle_decide_merge_candidate(user, path.split('/')[-1], body)

        self.send_error_json("Not found", 404)

    def do_PUT(self):
        if self.is_banned():
            return self.send_error_json("Too many requests", 429)
        path = self.get_path()
        body = self.get_body()
        if self.rate_limited('write', WRITE_RATE_LIMIT_PER_MIN):
            return self.send_error_json("Too many requests", 429)
        user = self.get_user()
        if not user:
            return self.send_error_json("Authentication required", 401)

        if re.match(r'^/api/contacts/[^/]+$', path):
            return self.handle_update_contact(user, path.split('/')[-1], body)
        if re.match(r'^/api/organizations/[^/]+$', path):
            return self.handle_update_organization(user, path.split('/')[-1], body)
        if re.match(r'^/api/opportunities/[^/]+$', path):
            return self.handle_update_opportunity(user, path.split('/')[-1], body)
        if re.match(r'^/api/communications/[^/]+$', path):
            return self.handle_update_communication(user, path.split('/')[-1], body)
        if re.match(r'^/api/lp-profiles/[^/]+$', path):
            return self.handle_update_lp_profile(user, path.split('/')[-1], body)
        if path == '/api/fundraising/state':
            return self.handle_update_fundraising_state(user, body)

        self.send_error_json("Not found", 404)

    def do_PATCH(self):
        if self.is_banned():
            return self.send_error_json("Too many requests", 429)
        path = self.get_path()
        body = self.get_body()
        if self.rate_limited('write', WRITE_RATE_LIMIT_PER_MIN):
            return self.send_error_json("Too many requests", 429)
        user = self.get_user()
        if not user:
            return self.send_error_json("Authentication required", 401)

        if re.match(r'^/api/opportunities/[^/]+/stage$', path):
            opp_id = path.split('/')[-2]
            return self.handle_update_stage(user, opp_id, body)
        if re.match(r'^/api/feature-requests/[^/]+$', path):
            fr_id = path.split('/')[-1]
            return self.handle_update_feature_request(user, fr_id, body)
        if re.match(r'^/api/admin/users/[^/]+$', path):
            target_user_id = path.split('/')[-1]
            return self.handle_admin_update_user(user, target_user_id, body)
        if path == '/api/fundraising/backup-policy':
            return self.handle_update_backup_policy(user, body)
        if re.match(r'^/api/fundraising/automations/[^/]+$', path):
            rule_id = path.split('/')[-1]
            return self.handle_update_fundraising_automation_rule(user, rule_id, body)

        self.send_error_json("Not found", 404)

    def do_DELETE(self):
        if self.is_banned():
            return self.send_error_json("Too many requests", 429)
        path = self.get_path()
        if self.rate_limited('write', WRITE_RATE_LIMIT_PER_MIN):
            return self.send_error_json("Too many requests", 429)
        user = self.get_user()
        if not user:
            return self.send_error_json("Authentication required", 401)

        if re.match(r'^/api/contacts/[^/]+$', path):
            return self.handle_delete_contact(user, path.split('/')[-1])
        if re.match(r'^/api/organizations/[^/]+$', path):
            return self.handle_delete_organization(user, path.split('/')[-1])
        if re.match(r'^/api/opportunities/[^/]+$', path):
            return self.handle_delete_opportunity(user, path.split('/')[-1])
        if re.match(r'^/api/communications/[^/]+$', path):
            return self.handle_delete_communication(user, path.split('/')[-1])
        self.send_error_json("Not found", 404)

    # ═══════════════════════════════════════════════════════════════════════════
    # AUTH HANDLERS
    # ═══════════════════════════════════════════════════════════════════════════

    def handle_bootstrap_status(self):
        conn = get_db()
        count = int(conn.execute("SELECT COUNT(*) as c FROM users").fetchone()['c'])
        conn.close()
        return self.send_json({
            "data": {
                "user_count": count,
                "setup_required": count == 0
            }
        })

    def handle_login(self, body):
        username = body.get('username', '').strip()
        password = body.get('password', '')
        if not username or not password:
            return self.send_error_json("Username and password required")

        conn = get_db()
        user = conn.execute("SELECT * FROM users WHERE username = ? AND is_active = 1", (username,)).fetchone()
        conn.close()

        if not user or not verify_password(password, user['password_hash']):
            return self.send_error_json("Invalid credentials", 401)

        token = create_token(user['id'], user['username'], user['role'])
        return self.send_json({
            "token": token,
            "user": {
                "id": user['id'],
                "username": user['username'],
                "email": user['email'],
                "full_name": user['full_name'],
                "role": user['role']
            }
        })

    def handle_register(self, body):
        required = ['username', 'password', 'email', 'full_name']
        for field in required:
            if not body.get(field, '').strip():
                return self.send_error_json(f"{field} is required")
        if len(str(body.get('password') or '').strip()) < 8:
            return self.send_error_json("password must be at least 8 characters")

        conn = get_db()
        user_count = conn.execute("SELECT COUNT(*) as c FROM users").fetchone()['c']
        if user_count > 0:
            conn.close()
            return self.send_error_json("Registration is disabled. Ask an admin for an invite.", 403)

        existing = conn.execute("SELECT id FROM users WHERE username = ? OR email = ?",
                                (body['username'], body['email'])).fetchone()
        if existing:
            conn.close()
            return self.send_error_json("Username or email already exists")

        user_id = generate_id()
        # Bootstrap first user as admin
        role = 'admin' if user_count == 0 else body.get('role', 'member')

        conn.execute(
            "INSERT INTO users (id, username, email, password_hash, full_name, role) VALUES (?, ?, ?, ?, ?, ?)",
            (user_id, body['username'], body['email'], hash_password(body['password']), body['full_name'], role)
        )
        conn.commit()
        conn.close()

        token = create_token(user_id, body['username'], role)
        return self.send_json({
            "token": token,
            "user": {
                "id": user_id,
                "username": body['username'],
                "email": body['email'],
                "full_name": body['full_name'],
                "role": role
            }
        }, 201)

    # ═══════════════════════════════════════════════════════════════════════════
    # CONTACT HANDLERS
    # ═══════════════════════════════════════════════════════════════════════════

    def handle_list_contacts(self, user, params):
        conn = get_db()
        query = """
            SELECT c.*, o.name as organization_name,
                   (SELECT COUNT(*) FROM communications WHERE contact_id = c.id) as comm_count,
                   (SELECT MAX(communication_date) FROM communications WHERE contact_id = c.id) as last_contact_date
            FROM contacts c
            LEFT JOIN organizations o ON c.organization_id = o.id
            WHERE 1=1 AND c.deleted_at IS NULL
        """
        args = []

        if params.get('type'):
            query += " AND c.contact_type = ?"
            args.append(params['type'])
        if params.get('status'):
            query += " AND c.status = ?"
            args.append(params['status'])
        if params.get('search'):
            search = f"%{params['search']}%"
            query += " AND (c.first_name LIKE ? OR c.last_name LIKE ? OR c.email LIKE ? OR o.name LIKE ? OR c.source LIKE ?)"
            args.extend([search, search, search, search, search])
        if params.get('organization_id'):
            query += " AND c.organization_id = ?"
            args.append(params['organization_id'])
        if params.get('tag'):
            query += " AND c.tags LIKE ?"
            args.append(f'%"{params["tag"]}"%')

        sort = params.get('sort', 'updated_at')
        order = 'DESC' if params.get('order', 'desc').lower() == 'desc' else 'ASC'
        allowed_sorts = ['first_name', 'last_name', 'email', 'created_at', 'updated_at', 'contact_type', 'source']
        if sort in allowed_sorts:
            query += f" ORDER BY c.{sort} {order}"
        else:
            query += f" ORDER BY c.updated_at DESC"

        # Pagination
        limit = min(int(params.get('limit', 50)), 500)
        offset = int(params.get('offset', 0))
        count_query = f"SELECT COUNT(*) as total FROM ({query})"
        total = conn.execute(count_query, args).fetchone()['total']

        query += " LIMIT ? OFFSET ?"
        args.extend([limit, offset])

        contacts = rows_to_list(conn.execute(query, args).fetchall())
        conn.close()

        return self.send_json({
            "data": contacts,
            "total": total,
            "limit": limit,
            "offset": offset
        })

    def handle_get_contact(self, user, contact_id):
        conn = get_db()
        contact = conn.execute("""
            SELECT c.*, o.name as organization_name
            FROM contacts c
            LEFT JOIN organizations o ON c.organization_id = o.id
            WHERE c.id = ?
        """, (contact_id,)).fetchone()

        if not contact:
            conn.close()
            return self.send_error_json("Contact not found", 404)

        result = row_to_dict(contact)

        # Get related data
        result['communications'] = rows_to_list(conn.execute(
            """SELECT cm.*, u.full_name as created_by_name
               FROM communications cm LEFT JOIN users u ON cm.created_by = u.id
               WHERE cm.contact_id = ? ORDER BY cm.communication_date DESC LIMIT 20""",
            (contact_id,)
        ).fetchall())

        result['opportunities'] = rows_to_list(conn.execute(
            "SELECT * FROM opportunities WHERE contact_id = ? ORDER BY updated_at DESC",
            (contact_id,)
        ).fetchall())

        lp = conn.execute("SELECT * FROM lp_profiles WHERE contact_id = ?", (contact_id,)).fetchone()
        result['lp_profile'] = row_to_dict(lp) if lp else None

        conn.close()
        return self.send_json({"data": result})

    def handle_create_contact(self, user, body):
        if not body.get('first_name') or not body.get('last_name'):
            return self.send_error_json("first_name and last_name are required")

        contact_id = generate_id()
        conn = get_db()
        organization_id = body.get('organization_id')
        if not organization_id and body.get('organization'):
            organization_id = _ensure_organization_by_name(conn, body.get('organization'), user['user_id'])

        tags = json.dumps(body.get('tags', []))
        conn.execute("""
            INSERT INTO contacts (id, first_name, last_name, email, phone, mobile, title,
                organization_id, contact_type, status, source, tags, notes, linkedin_url,
                city, state, country, location_query, preferred_contact, created_by)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
        """, (
            contact_id, body['first_name'], body['last_name'],
            body.get('email'), body.get('phone'), body.get('mobile'),
            body.get('title'), organization_id,
            body.get('contact_type', 'prospect'), body.get('status', 'active'),
            body.get('source'), tags, body.get('notes'),
            body.get('linkedin_url'), body.get('city'), body.get('state'),
            body.get('country'), body.get('location_query'),
            body.get('preferred_contact', 'email'),
            user['user_id']
        ))

        contact = row_to_dict(conn.execute("""
            SELECT c.*, o.name as organization_name
            FROM contacts c LEFT JOIN organizations o ON c.organization_id = o.id
            WHERE c.id = ?
        """, (contact_id,)).fetchone())
        _sync_contact_to_fundraising_state(conn, contact, actor_user_id=user['user_id'], remove=False)
        log_audit(conn, user['user_id'], 'contact', contact_id, 'create')
        conn.commit()
        conn.close()
        return self.send_json({"data": contact}, 201)

    def handle_update_contact(self, user, contact_id, body):
        conn = get_db()
        existing = conn.execute("""
            SELECT c.*, o.name as organization_name
            FROM contacts c LEFT JOIN organizations o ON c.organization_id = o.id
            WHERE c.id = ?
        """, (contact_id,)).fetchone()
        if not existing:
            conn.close()
            return self.send_error_json("Contact not found", 404)
        previous_contact = row_to_dict(existing)

        updatable = ['first_name', 'last_name', 'email', 'phone', 'mobile', 'title',
                     'organization_id', 'contact_type', 'status', 'source', 'notes',
                     'linkedin_url', 'city', 'state', 'country', 'location_query', 'preferred_contact']
        sets = []
        args = []
        for field in updatable:
            if field in body:
                sets.append(f"{field} = ?")
                args.append(body[field])
        if 'organization' in body and 'organization_id' not in body:
            org_id = _ensure_organization_by_name(conn, body.get('organization'), user['user_id'])
            sets.append("organization_id = ?")
            args.append(org_id)

        if 'tags' in body:
            sets.append("tags = ?")
            args.append(json.dumps(body['tags']))

        if not sets:
            conn.close()
            return self.send_error_json("No fields to update")

        sets.append("updated_at = ?")
        args.append(now())
        args.append(contact_id)

        conn.execute(f"UPDATE contacts SET {', '.join(sets)} WHERE id = ?", args)

        contact = row_to_dict(conn.execute("""
            SELECT c.*, o.name as organization_name
            FROM contacts c LEFT JOIN organizations o ON c.organization_id = o.id
            WHERE c.id = ?
        """, (contact_id,)).fetchone())
        _sync_contact_to_fundraising_state(conn, previous_contact, actor_user_id=user['user_id'], remove=True)
        _sync_contact_to_fundraising_state(conn, contact, actor_user_id=user['user_id'], remove=False)
        log_audit(conn, user['user_id'], 'contact', contact_id, 'update', body)
        conn.commit()
        conn.close()
        return self.send_json({"data": contact})

    def handle_delete_contact(self, user, contact_id):
        conn = get_db()
        existing = conn.execute("""
            SELECT c.*, o.name as organization_name
            FROM contacts c LEFT JOIN organizations o ON c.organization_id = o.id
            WHERE c.id = ?
        """, (contact_id,)).fetchone()
        if not existing:
            conn.close()
            return self.send_error_json("Contact not found", 404)

        _sync_contact_to_fundraising_state(conn, row_to_dict(existing), actor_user_id=user['user_id'], remove=True)
        # Soft-delete (guardrail #3 — never hard-delete): mark deleted_at and
        # cascade to the contact's opportunities, communications, and lp_profile.
        _ts = now()
        conn.execute("UPDATE contacts SET deleted_at = ?, updated_at = ? WHERE id = ?", (_ts, _ts, contact_id))
        conn.execute("UPDATE opportunities SET deleted_at = ? WHERE contact_id = ? AND deleted_at IS NULL", (_ts, contact_id))
        conn.execute("UPDATE communications SET deleted_at = ? WHERE contact_id = ? AND deleted_at IS NULL", (_ts, contact_id))
        conn.execute("UPDATE lp_profiles SET deleted_at = ? WHERE contact_id = ? AND deleted_at IS NULL", (_ts, contact_id))
        log_audit(conn, user['user_id'], 'contact', contact_id, 'delete')
        conn.commit()
        conn.close()
        return self.send_json({"message": "Contact deleted"})

    # ═══════════════════════════════════════════════════════════════════════════
    # ORGANIZATION HANDLERS
    # ═══════════════════════════════════════════════════════════════════════════

    def handle_list_organizations(self, user, params):
        conn = get_db()
        query = """
            SELECT o.*,
                   (SELECT COUNT(*) FROM contacts WHERE organization_id = o.id) as contact_count,
                   (SELECT COALESCE(SUM(commitment_amount), 0) FROM opportunities WHERE organization_id = o.id AND stage = 'funded') as total_funded
            FROM organizations o WHERE 1=1 AND o.deleted_at IS NULL
        """
        args = []
        if params.get('search'):
            search = f"%{params['search']}%"
            query += " AND (o.name LIKE ? OR o.industry LIKE ?)"
            args.extend([search, search])
        if params.get('type'):
            query += " AND o.type = ?"
            args.append(params['type'])

        query += " ORDER BY o.name ASC"
        limit = min(int(params.get('limit', 50)), 500)
        offset = int(params.get('offset', 0))
        total = conn.execute(f"SELECT COUNT(*) as total FROM ({query})", args).fetchone()['total']
        query += " LIMIT ? OFFSET ?"
        args.extend([limit, offset])

        orgs = rows_to_list(conn.execute(query, args).fetchall())
        conn.close()
        return self.send_json({"data": orgs, "total": total, "limit": limit, "offset": offset})

    def handle_get_organization(self, user, org_id):
        conn = get_db()
        org = conn.execute("SELECT * FROM organizations WHERE id = ?", (org_id,)).fetchone()
        if not org:
            conn.close()
            return self.send_error_json("Organization not found", 404)

        result = row_to_dict(org)
        result['contacts'] = rows_to_list(conn.execute(
            "SELECT * FROM contacts WHERE organization_id = ? ORDER BY last_name", (org_id,)
        ).fetchall())
        result['opportunities'] = rows_to_list(conn.execute(
            "SELECT * FROM opportunities WHERE organization_id = ? ORDER BY updated_at DESC", (org_id,)
        ).fetchall())
        conn.close()
        return self.send_json({"data": result})

    def handle_create_organization(self, user, body):
        if not body.get('name'):
            return self.send_error_json("name is required")

        org_id = generate_id()
        conn = get_db()
        tags = json.dumps(body.get('tags', []))
        conn.execute("""
            INSERT INTO organizations (id, name, type, industry, website, phone, email,
                address, city, state, country, description, tags, created_by)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
        """, (
            org_id, body['name'], body.get('type', 'other'), body.get('industry'),
            body.get('website'), body.get('phone'), body.get('email'),
            body.get('address'), body.get('city'), body.get('state'),
            body.get('country'), body.get('description'), tags, user['user_id']
        ))
        log_audit(conn, user['user_id'], 'organization', org_id, 'create')
        conn.commit()
        org = row_to_dict(conn.execute("SELECT * FROM organizations WHERE id = ?", (org_id,)).fetchone())
        conn.close()
        return self.send_json({"data": org}, 201)

    def handle_update_organization(self, user, org_id, body):
        conn = get_db()
        existing = conn.execute("SELECT id FROM organizations WHERE id = ?", (org_id,)).fetchone()
        if not existing:
            conn.close()
            return self.send_error_json("Organization not found", 404)

        updatable = ['name', 'type', 'industry', 'website', 'phone', 'email',
                     'address', 'city', 'state', 'country', 'description']
        sets = []
        args = []
        for field in updatable:
            if field in body:
                sets.append(f"{field} = ?")
                args.append(body[field])
        if 'tags' in body:
            sets.append("tags = ?")
            args.append(json.dumps(body['tags']))

        if sets:
            sets.append("updated_at = ?")
            args.append(now())
            args.append(org_id)
            conn.execute(f"UPDATE organizations SET {', '.join(sets)} WHERE id = ?", args)
            log_audit(conn, user['user_id'], 'organization', org_id, 'update', body)
            conn.commit()

        org = row_to_dict(conn.execute("SELECT * FROM organizations WHERE id = ?", (org_id,)).fetchone())
        conn.close()
        return self.send_json({"data": org})

    def handle_delete_organization(self, user, org_id):
        conn = get_db()
        existing = conn.execute("SELECT id FROM organizations WHERE id = ?", (org_id,)).fetchone()
        if not existing:
            conn.close()
            return self.send_error_json("Organization not found", 404)

        conn.execute("UPDATE contacts SET organization_id = NULL WHERE organization_id = ?", (org_id,))
        conn.execute("UPDATE organizations SET deleted_at = ?, updated_at = ? WHERE id = ?", (now(), now(), org_id))
        log_audit(conn, user['user_id'], 'organization', org_id, 'delete')
        conn.commit()
        conn.close()
        return self.send_json({"message": "Organization deleted"})

    # ═══════════════════════════════════════════════════════════════════════════
    # OPPORTUNITY (PIPELINE) HANDLERS
    # ═══════════════════════════════════════════════════════════════════════════

    def handle_list_opportunities(self, user, params):
        conn = get_db()
        query = """
            SELECT op.*, c.first_name, c.last_name, c.email as contact_email,
                   o.name as organization_name, u.full_name as owner_name
            FROM opportunities op
            LEFT JOIN contacts c ON op.contact_id = c.id
            LEFT JOIN organizations o ON op.organization_id = o.id
            LEFT JOIN users u ON op.owner_id = u.id
            WHERE 1=1 AND op.deleted_at IS NULL
        """
        args = []
        if params.get('stage'):
            query += " AND op.stage = ?"
            args.append(params['stage'])
        if params.get('owner_id'):
            query += " AND op.owner_id = ?"
            args.append(params['owner_id'])
        if params.get('search'):
            search = f"%{params['search']}%"
            query += " AND (op.name LIKE ? OR c.first_name LIKE ? OR c.last_name LIKE ?)"
            args.extend([search, search, search])
        if params.get('priority'):
            query += " AND op.priority = ?"
            args.append(params['priority'])
        if params.get('fund_name'):
            query += " AND op.fund_name = ?"
            args.append(params['fund_name'])

        query += " ORDER BY op.updated_at DESC"
        limit = min(int(params.get('limit', 100)), 500)
        offset = int(params.get('offset', 0))
        total = conn.execute(f"SELECT COUNT(*) as total FROM ({query})", args).fetchone()['total']
        query += " LIMIT ? OFFSET ?"
        args.extend([limit, offset])

        opps = rows_to_list(conn.execute(query, args).fetchall())
        conn.close()
        return self.send_json({"data": opps, "total": total, "limit": limit, "offset": offset})

    def handle_get_opportunity(self, user, opp_id):
        conn = get_db()
        opp = conn.execute("""
            SELECT op.*, c.first_name, c.last_name, c.email as contact_email,
                   o.name as organization_name, u.full_name as owner_name
            FROM opportunities op
            LEFT JOIN contacts c ON op.contact_id = c.id
            LEFT JOIN organizations o ON op.organization_id = o.id
            LEFT JOIN users u ON op.owner_id = u.id
            WHERE op.id = ?
        """, (opp_id,)).fetchone()

        if not opp:
            conn.close()
            return self.send_error_json("Opportunity not found", 404)

        result = row_to_dict(opp)
        result['communications'] = rows_to_list(conn.execute(
            """SELECT cm.*, u.full_name as created_by_name
               FROM communications cm LEFT JOIN users u ON cm.created_by = u.id
               WHERE cm.opportunity_id = ? ORDER BY cm.communication_date DESC""",
            (opp_id,)
        ).fetchall())

        # Stage history from audit log
        result['stage_history'] = rows_to_list(conn.execute(
            """SELECT al.*, u.full_name as user_name FROM audit_log al
               LEFT JOIN users u ON al.user_id = u.id
               WHERE al.entity_type = 'opportunity' AND al.entity_id = ?
               AND al.action = 'stage_change'
               ORDER BY al.created_at DESC""",
            (opp_id,)
        ).fetchall())

        conn.close()
        return self.send_json({"data": result})

    def handle_create_opportunity(self, user, body):
        if not body.get('name') or not body.get('contact_id'):
            return self.send_error_json("name and contact_id are required")

        opp_id = generate_id()
        conn = get_db()

        # Verify contact exists
        contact = conn.execute("SELECT id, organization_id FROM contacts WHERE id = ?",
                               (body['contact_id'],)).fetchone()
        if not contact:
            conn.close()
            return self.send_error_json("Contact not found", 404)

        org_id = body.get('organization_id', contact['organization_id'])

        conn.execute("""
            INSERT INTO opportunities (id, name, contact_id, organization_id, stage,
                commitment_amount, expected_amount, probability, expected_close_date,
                fund_name, description, next_step, owner_id, priority)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
        """, (
            opp_id, body['name'], body['contact_id'], org_id,
            body.get('stage', 'lead'),
            body.get('commitment_amount', 0), body.get('expected_amount', 0),
            body.get('probability', 10), body.get('expected_close_date'),
            body.get('fund_name'), body.get('description'), body.get('next_step'),
            body.get('owner_id', user['user_id']), body.get('priority', 'medium')
        ))
        log_audit(conn, user['user_id'], 'opportunity', opp_id, 'create')
        conn.commit()

        opp = row_to_dict(conn.execute("""
            SELECT op.*, c.first_name, c.last_name, o.name as organization_name
            FROM opportunities op
            LEFT JOIN contacts c ON op.contact_id = c.id
            LEFT JOIN organizations o ON op.organization_id = o.id
            WHERE op.id = ?
        """, (opp_id,)).fetchone())
        conn.close()
        return self.send_json({"data": opp}, 201)

    def handle_update_opportunity(self, user, opp_id, body):
        conn = get_db()
        existing = conn.execute("SELECT * FROM opportunities WHERE id = ?", (opp_id,)).fetchone()
        if not existing:
            conn.close()
            return self.send_error_json("Opportunity not found", 404)

        updatable = ['name', 'contact_id', 'organization_id', 'stage', 'commitment_amount',
                     'expected_amount', 'probability', 'expected_close_date', 'fund_name',
                     'description', 'next_step', 'owner_id', 'priority', 'lost_reason']
        sets = []
        args = []

        old_stage = existing['stage']
        new_stage = body.get('stage', old_stage)

        for field in updatable:
            if field in body:
                sets.append(f"{field} = ?")
                args.append(body[field])

        if sets:
            sets.append("updated_at = ?")
            args.append(now())
            args.append(opp_id)
            conn.execute(f"UPDATE opportunities SET {', '.join(sets)} WHERE id = ?", args)

            if new_stage != old_stage:
                log_audit(conn, user['user_id'], 'opportunity', opp_id, 'stage_change',
                          {"from": old_stage, "to": new_stage})
            else:
                log_audit(conn, user['user_id'], 'opportunity', opp_id, 'update', body)
            conn.commit()

        opp = row_to_dict(conn.execute("""
            SELECT op.*, c.first_name, c.last_name, o.name as organization_name, u.full_name as owner_name
            FROM opportunities op
            LEFT JOIN contacts c ON op.contact_id = c.id
            LEFT JOIN organizations o ON op.organization_id = o.id
            LEFT JOIN users u ON op.owner_id = u.id
            WHERE op.id = ?
        """, (opp_id,)).fetchone())
        conn.close()
        return self.send_json({"data": opp})

    def handle_update_stage(self, user, opp_id, body):
        new_stage = body.get('stage', '')
        if new_stage not in PIPELINE_STAGES:
            return self.send_error_json(f"Invalid stage. Must be one of: {', '.join(PIPELINE_STAGES)}")

        conn = get_db()
        existing = conn.execute("SELECT stage FROM opportunities WHERE id = ?", (opp_id,)).fetchone()
        if not existing:
            conn.close()
            return self.send_error_json("Opportunity not found", 404)

        old_stage = existing['stage']
        conn.execute("UPDATE opportunities SET stage = ?, updated_at = ? WHERE id = ?",
                     (new_stage, now(), opp_id))
        log_audit(conn, user['user_id'], 'opportunity', opp_id, 'stage_change',
                  {"from": old_stage, "to": new_stage})
        conn.commit()

        opp = row_to_dict(conn.execute("""
            SELECT op.*, c.first_name, c.last_name, o.name as organization_name, u.full_name as owner_name
            FROM opportunities op
            LEFT JOIN contacts c ON op.contact_id = c.id
            LEFT JOIN organizations o ON op.organization_id = o.id
            LEFT JOIN users u ON op.owner_id = u.id
            WHERE op.id = ?
        """, (opp_id,)).fetchone())
        conn.close()
        return self.send_json({"data": opp})

    def handle_delete_opportunity(self, user, opp_id):
        conn = get_db()
        existing = conn.execute("SELECT id FROM opportunities WHERE id = ?", (opp_id,)).fetchone()
        if not existing:
            conn.close()
            return self.send_error_json("Opportunity not found", 404)

        conn.execute("UPDATE opportunities SET deleted_at = ?, updated_at = ? WHERE id = ?", (now(), now(), opp_id))
        log_audit(conn, user['user_id'], 'opportunity', opp_id, 'delete')
        conn.commit()
        conn.close()
        return self.send_json({"message": "Opportunity deleted"})

    # ═══════════════════════════════════════════════════════════════════════════
    # COMMUNICATION HANDLERS
    # ═══════════════════════════════════════════════════════════════════════════

    def handle_list_communications(self, user, params):
        conn = get_db()
        query = """
            SELECT cm.*, c.first_name, c.last_name, u.full_name as created_by_name
            FROM communications cm
            LEFT JOIN contacts c ON cm.contact_id = c.id
            LEFT JOIN users u ON cm.created_by = u.id
            WHERE 1=1 AND cm.deleted_at IS NULL
        """
        args = []
        if params.get('contact_id'):
            query += " AND cm.contact_id = ?"
            args.append(params['contact_id'])
        if params.get('type'):
            query += " AND cm.type = ?"
            args.append(params['type'])
        if params.get('search'):
            search = f"%{params['search']}%"
            query += " AND (cm.subject LIKE ? OR cm.body LIKE ?)"
            args.extend([search, search])

        query += " ORDER BY cm.communication_date DESC"
        limit = min(int(params.get('limit', 50)), 500)
        offset = int(params.get('offset', 0))
        total = conn.execute(f"SELECT COUNT(*) as total FROM ({query})", args).fetchone()['total']
        query += " LIMIT ? OFFSET ?"
        args.extend([limit, offset])

        comms = rows_to_list(conn.execute(query, args).fetchall())
        conn.close()
        return self.send_json({"data": comms, "total": total, "limit": limit, "offset": offset})

    def handle_list_contact_communications(self, user, contact_id, params):
        params['contact_id'] = contact_id
        return self.handle_list_communications(user, params)

    def handle_create_communication(self, user, body):
        if not body.get('contact_id'):
            return self.send_error_json("contact_id is required")

        comm_id = generate_id()
        conn = get_db()

        contact = conn.execute("SELECT id FROM contacts WHERE id = ?", (body['contact_id'],)).fetchone()
        if not contact:
            conn.close()
            return self.send_error_json("Contact not found", 404)

        attendees = json.dumps(body.get('attendees', []))
        conn.execute("""
            INSERT INTO communications (id, contact_id, opportunity_id, type, subject, body,
                communication_date, duration_minutes, outcome, next_action, next_action_date,
                attendees, created_by)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
        """, (
            comm_id, body['contact_id'], body.get('opportunity_id'),
            body.get('type', 'note'), body.get('subject'), body.get('body'),
            body.get('communication_date', now()),
            body.get('duration_minutes'), body.get('outcome'),
            body.get('next_action'), body.get('next_action_date'),
            attendees, user['user_id']
        ))

        # Update contact's updated_at
        conn.execute("UPDATE contacts SET updated_at = ? WHERE id = ?", (now(), body['contact_id']))

        log_audit(conn, user['user_id'], 'communication', comm_id, 'create')
        conn.commit()

        comm = row_to_dict(conn.execute("""
            SELECT cm.*, c.first_name, c.last_name, u.full_name as created_by_name
            FROM communications cm
            LEFT JOIN contacts c ON cm.contact_id = c.id
            LEFT JOIN users u ON cm.created_by = u.id
            WHERE cm.id = ?
        """, (comm_id,)).fetchone())
        conn.close()
        return self.send_json({"data": comm}, 201)

    def handle_log_fundraising_communication(self, user, body):
        row_id = str(body.get('row_id') or '').strip()
        investor_name_in = str(body.get('investor_name') or '').strip()
        contact_in = body.get('contact')
        append_note = bool(body.get('append_note', True))
        create_investor_if_missing = bool(body.get('create_investor_if_missing', False))

        if not row_id and not investor_name_in:
            return self.send_error_json("row_id or investor_name is required")

        conn = get_db()
        ensure_fundraising_state_row(conn)
        state = conn.execute("SELECT * FROM fundraising_state WHERE id = 'main'").fetchone()
        if not state:
            conn.close()
            return self.send_error_json("Fundraising state not found", 404)

        try:
            grid = json.loads(state['grid_json']) if state['grid_json'] else {}
        except Exception:
            grid = {}
        grid = sanitize_fundraising_grid(grid)
        rows = grid.get('rows', [])
        if not isinstance(rows, list):
            rows = []

        target_index = -1
        target_row = None
        for idx, row in enumerate(rows):
            if not isinstance(row, dict):
                continue
            if row_id and str(row.get('id') or '').strip() == row_id:
                target_index = idx
                target_row = row
                break
        if target_row is None and investor_name_in:
            wanted = _normalize_text(investor_name_in)
            for idx, row in enumerate(rows):
                if not isinstance(row, dict):
                    continue
                if _normalize_text(row.get('investor_name')) == wanted:
                    target_index = idx
                    target_row = row
                    break
        if target_row is None and investor_name_in and create_investor_if_missing:
            new_row = {
                "id": f"inv-{int(time.time() * 1000)}-{uuid.uuid4().hex[:6]}",
                "investor_name": investor_name_in,
                "contacts": [contact_in] if isinstance(contact_in, dict) else [],
                "notes": "",
                "notes_last_modified": "",
                "last_communication_date": "",
                "lead": "",
                "priority": False,
                "follow_up": False,
                "graveyard": False
            }
            columns = grid.get('columns') if isinstance(grid.get('columns'), list) else []
            for col in columns:
                if not isinstance(col, dict):
                    continue
                col_id = str(col.get('id') or '').strip()
                if not col_id or col_id == 'longshot_followup' or col_id in new_row:
                    continue
                col_type = str(col.get('type') or 'text').strip().lower()
                if col_type == 'checkbox':
                    new_row[col_id] = False
                elif col_type in ('currency', 'number'):
                    new_row[col_id] = 0
                elif col_type == 'contacts':
                    new_row[col_id] = []
                else:
                    new_row[col_id] = ''
            rows.append(new_row)
            target_index = len(rows) - 1
            target_row = new_row
        if target_row is None:
            conn.close()
            return self.send_error_json("Fundraising row not found", 404)

        investor_name = str(target_row.get('investor_name') or investor_name_in or '').strip()
        contacts = target_row.get('contacts')
        if not isinstance(contact_in, dict):
            contact_in = None
        if contact_in is None and isinstance(contacts, list) and contacts:
            first = contacts[0]
            contact_in = first if isinstance(first, dict) else None
        if not isinstance(contact_in, dict):
            conn.close()
            return self.send_error_json("contact is required")

        contact_id = _upsert_contact_from_fundraising(conn, investor_name, contact_in, actor_user_id=user['user_id'])
        if not contact_id:
            conn.close()
            return self.send_error_json("Could not resolve contact for communication")

        comm_id = generate_id()
        comm_type = str(body.get('type') or 'note').strip() or 'note'
        comm_subject = str(body.get('subject') or '').strip()
        comm_body = str(body.get('body') or '').strip()
        comm_outcome = str(body.get('outcome') or '').strip()
        next_action = str(body.get('next_action') or '').strip()
        next_action_date = str(body.get('next_action_date') or '').strip()
        comm_date = str(body.get('communication_date') or now()).strip() or now()
        conn.execute("""
            INSERT INTO communications (id, contact_id, opportunity_id, type, subject, body,
                communication_date, duration_minutes, outcome, next_action, next_action_date,
                attendees, created_by)
            VALUES (?, ?, NULL, ?, ?, ?, ?, NULL, ?, ?, ?, '[]', ?)
        """, (
            comm_id,
            contact_id,
            comm_type,
            comm_subject,
            comm_body,
            comm_date,
            comm_outcome or None,
            next_action or None,
            next_action_date or None,
            user['user_id']
        ))
        conn.execute("UPDATE contacts SET updated_at = ? WHERE id = ?", (now(), contact_id))
        log_audit(conn, user['user_id'], 'communication', comm_id, 'create', {"source": "fundraising_grid"})

        iso_day = now()[:10]
        target_row['last_communication_date'] = iso_day
        if append_note:
            contact_name = str(contact_in.get('name') or '').strip()
            summary = comm_subject or comm_body[:120]
            note_line = f"{iso_day} [{comm_type}] {contact_name}: {summary}".strip()
            existing_notes = str(target_row.get('notes') or '')
            target_row['notes'] = f"{existing_notes}\n{note_line}".strip() if existing_notes.strip() else note_line
            target_row['notes_last_modified'] = iso_day
        rows[target_index] = target_row

        try:
            views = json.loads(state['views_json']) if state['views_json'] else []
        except Exception:
            views = []
        views = sanitize_grid_views(views)
        next_version = int(state['version'] or 1) + 1
        conn.execute("""
            UPDATE fundraising_state
            SET grid_json = ?, views_json = ?, version = ?, updated_by = ?, updated_at = ?
            WHERE id = 'main'
        """, (json.dumps(grid), json.dumps(views), next_version, user['user_id'], now()))
        sync_fundraising_relational(conn, grid, views, actor_user_id=user['user_id'])
        conn.commit()

        comm = row_to_dict(conn.execute("""
            SELECT cm.*, c.first_name, c.last_name, u.full_name as created_by_name
            FROM communications cm
            LEFT JOIN contacts c ON cm.contact_id = c.id
            LEFT JOIN users u ON cm.created_by = u.id
            WHERE cm.id = ?
        """, (comm_id,)).fetchone())
        conn.close()
        return self.send_json({"data": {"communication": comm, "row": target_row, "version": next_version}}, 201)

    def handle_update_communication(self, user, comm_id, body):
        conn = get_db()
        existing = conn.execute("SELECT id FROM communications WHERE id = ?", (comm_id,)).fetchone()
        if not existing:
            conn.close()
            return self.send_error_json("Communication not found", 404)

        updatable = ['type', 'subject', 'body', 'communication_date', 'duration_minutes',
                     'outcome', 'next_action', 'next_action_date']
        sets = []
        args = []
        for field in updatable:
            if field in body:
                sets.append(f"{field} = ?")
                args.append(body[field])
        if 'attendees' in body:
            sets.append("attendees = ?")
            args.append(json.dumps(body['attendees']))

        if sets:
            sets.append("updated_at = ?")
            args.append(now())
            args.append(comm_id)
            conn.execute(f"UPDATE communications SET {', '.join(sets)} WHERE id = ?", args)
            log_audit(conn, user['user_id'], 'communication', comm_id, 'update')
            conn.commit()

        comm = row_to_dict(conn.execute("SELECT * FROM communications WHERE id = ?", (comm_id,)).fetchone())
        conn.close()
        return self.send_json({"data": comm})

    def handle_delete_communication(self, user, comm_id):
        conn = get_db()
        existing = conn.execute("SELECT id FROM communications WHERE id = ?", (comm_id,)).fetchone()
        if not existing:
            conn.close()
            return self.send_error_json("Communication not found", 404)

        conn.execute("UPDATE communications SET deleted_at = ?, updated_at = ? WHERE id = ?", (now(), now(), comm_id))
        log_audit(conn, user['user_id'], 'communication', comm_id, 'delete')
        conn.commit()
        conn.close()
        return self.send_json({"message": "Communication deleted"})

    # ═══════════════════════════════════════════════════════════════════════════
    # LP PROFILE HANDLERS
    # ═══════════════════════════════════════════════════════════════════════════

    def handle_list_lp_profiles(self, user, params):
        conn = get_db()
        query = """
            SELECT lp.*, c.first_name, c.last_name, c.email, o.name as organization_name
            FROM lp_profiles lp
            LEFT JOIN contacts c ON lp.contact_id = c.id
            LEFT JOIN organizations o ON c.organization_id = o.id
            WHERE 1=1
        """
        args = []
        if params.get('fund_name'):
            query += " AND lp.fund_name = ?"
            args.append(params['fund_name'])
        if params.get('search'):
            search = f"%{params['search']}%"
            query += " AND (c.first_name LIKE ? OR c.last_name LIKE ? OR o.name LIKE ?)"
            args.extend([search, search, search])

        query += " ORDER BY lp.commitment_amount DESC"
        profiles = rows_to_list(conn.execute(query, args).fetchall())
        conn.close()
        return self.send_json({"data": profiles, "total": len(profiles)})

    def handle_get_lp_profile(self, user, lp_id):
        conn = get_db()
        lp = conn.execute("""
            SELECT lp.*, c.first_name, c.last_name, c.email, c.phone, o.name as organization_name
            FROM lp_profiles lp
            LEFT JOIN contacts c ON lp.contact_id = c.id
            LEFT JOIN organizations o ON c.organization_id = o.id
            WHERE lp.id = ?
        """, (lp_id,)).fetchone()
        if not lp:
            conn.close()
            return self.send_error_json("LP profile not found", 404)
        conn.close()
        return self.send_json({"data": row_to_dict(lp)})

    def handle_create_lp_profile(self, user, body):
        if not body.get('contact_id'):
            return self.send_error_json("contact_id is required")

        conn = get_db()
        existing = conn.execute("SELECT id FROM lp_profiles WHERE contact_id = ?",
                                (body['contact_id'],)).fetchone()
        if existing:
            conn.close()
            return self.send_error_json("LP profile already exists for this contact")

        lp_id = generate_id()
        conn.execute("""
            INSERT INTO lp_profiles (id, contact_id, commitment_amount, funded_amount,
                commitment_date, fund_name, investor_type, accredited, legal_docs_signed,
                signed_date, wire_received, wire_date, k1_sent, preferred_communication, notes)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
        """, (
            lp_id, body['contact_id'], body.get('commitment_amount', 0),
            body.get('funded_amount', 0), body.get('commitment_date'),
            body.get('fund_name'), body.get('investor_type'),
            body.get('accredited', 0), body.get('legal_docs_signed', 0),
            body.get('signed_date'), body.get('wire_received', 0),
            body.get('wire_date'), body.get('k1_sent', 0),
            body.get('preferred_communication', 'email'), body.get('notes')
        ))

        # Update contact type to investor
        conn.execute("UPDATE contacts SET contact_type = 'investor', updated_at = ? WHERE id = ?",
                     (now(), body['contact_id']))

        log_audit(conn, user['user_id'], 'lp_profile', lp_id, 'create')
        conn.commit()

        lp = row_to_dict(conn.execute("SELECT * FROM lp_profiles WHERE id = ?", (lp_id,)).fetchone())
        conn.close()
        return self.send_json({"data": lp}, 201)

    def handle_update_lp_profile(self, user, lp_id, body):
        conn = get_db()
        existing = conn.execute("SELECT id FROM lp_profiles WHERE id = ?", (lp_id,)).fetchone()
        if not existing:
            conn.close()
            return self.send_error_json("LP profile not found", 404)

        updatable = ['commitment_amount', 'funded_amount', 'commitment_date', 'fund_name',
                     'investor_type', 'accredited', 'legal_docs_signed', 'signed_date',
                     'wire_received', 'wire_date', 'k1_sent', 'preferred_communication', 'notes']
        sets = []
        args = []
        for field in updatable:
            if field in body:
                sets.append(f"{field} = ?")
                args.append(body[field])

        if sets:
            sets.append("updated_at = ?")
            args.append(now())
            args.append(lp_id)
            conn.execute(f"UPDATE lp_profiles SET {', '.join(sets)} WHERE id = ?", args)
            log_audit(conn, user['user_id'], 'lp_profile', lp_id, 'update', body)
            conn.commit()

        lp = row_to_dict(conn.execute("SELECT * FROM lp_profiles WHERE id = ?", (lp_id,)).fetchone())
        conn.close()
        return self.send_json({"data": lp})

    # ═══════════════════════════════════════════════════════════════════════════
    # REPORT HANDLERS
    # ═══════════════════════════════════════════════════════════════════════════

    def handle_dashboard_report(self, user):
        conn = get_db()

        # Key metrics
        total_lps = conn.execute("SELECT COUNT(*) as c FROM contacts WHERE contact_type = 'investor'").fetchone()['c']
        total_prospects = conn.execute("SELECT COUNT(*) as c FROM contacts WHERE contact_type = 'prospect'").fetchone()['c']
        total_contacts = conn.execute("SELECT COUNT(*) as c FROM contacts").fetchone()['c']

        total_committed = conn.execute(
            "SELECT COALESCE(SUM(commitment_amount), 0) as total FROM lp_profiles"
        ).fetchone()['total']
        total_funded = conn.execute(
            "SELECT COALESCE(SUM(funded_amount), 0) as total FROM lp_profiles"
        ).fetchone()['total']

        pipeline_value = conn.execute(
            "SELECT COALESCE(SUM(expected_amount), 0) as total FROM opportunities WHERE stage NOT IN ('funded', 'lost')"
        ).fetchone()['total']

        active_opportunities = conn.execute(
            "SELECT COUNT(*) as c FROM opportunities WHERE stage NOT IN ('funded', 'lost')"
        ).fetchone()['c']

        # Pipeline by stage
        pipeline_stages = rows_to_list(conn.execute("""
            SELECT stage, COUNT(*) as count, COALESCE(SUM(expected_amount), 0) as total_value,
                   COALESCE(SUM(commitment_amount), 0) as committed_value
            FROM opportunities
            WHERE stage != 'lost'
            GROUP BY stage
            ORDER BY CASE stage
                WHEN 'lead' THEN 1 WHEN 'outreach' THEN 2 WHEN 'meeting' THEN 3
                WHEN 'due_diligence' THEN 4 WHEN 'committed' THEN 5 WHEN 'funded' THEN 6
            END
        """).fetchall())

        # Recent communications
        recent_comms = rows_to_list(conn.execute("""
            SELECT cm.*, c.first_name, c.last_name, u.full_name as created_by_name
            FROM communications cm
            LEFT JOIN contacts c ON cm.contact_id = c.id
            LEFT JOIN users u ON cm.created_by = u.id
            ORDER BY cm.communication_date DESC LIMIT 10
        """).fetchall())

        # Upcoming actions
        upcoming = rows_to_list(conn.execute("""
            SELECT cm.*, c.first_name, c.last_name
            FROM communications cm
            LEFT JOIN contacts c ON cm.contact_id = c.id
            WHERE cm.next_action IS NOT NULL AND cm.next_action != ''
            AND cm.next_action_date >= date('now')
            ORDER BY cm.next_action_date ASC LIMIT 10
        """).fetchall())

        # Recent stage changes
        recent_stage_changes = rows_to_list(conn.execute("""
            SELECT al.*, u.full_name as user_name, op.name as opportunity_name
            FROM audit_log al
            LEFT JOIN users u ON al.user_id = u.id
            LEFT JOIN opportunities op ON al.entity_id = op.id
            WHERE al.entity_type = 'opportunity' AND al.action = 'stage_change'
            ORDER BY al.created_at DESC LIMIT 10
        """).fetchall())

        # Comms this month
        comms_this_month = conn.execute("""
            SELECT COUNT(*) as c FROM communications
            WHERE communication_date >= date('now', 'start of month')
        """).fetchone()['c']

        # Meetings this month
        meetings_this_month = conn.execute("""
            SELECT COUNT(*) as c FROM communications
            WHERE type = 'meeting' AND communication_date >= date('now', 'start of month')
        """).fetchone()['c']

        conn.close()
        return self.send_json({
            "data": {
                "metrics": {
                    "total_lps": total_lps,
                    "total_prospects": total_prospects,
                    "total_contacts": total_contacts,
                    "total_committed": total_committed,
                    "total_funded": total_funded,
                    "pipeline_value": pipeline_value,
                    "active_opportunities": active_opportunities,
                    "comms_this_month": comms_this_month,
                    "meetings_this_month": meetings_this_month
                },
                "pipeline_stages": pipeline_stages,
                "recent_communications": recent_comms,
                "upcoming_actions": upcoming,
                "recent_stage_changes": recent_stage_changes
            }
        })

    def handle_pipeline_report(self, user):
        conn = get_db()
        stages = rows_to_list(conn.execute("""
            SELECT stage, COUNT(*) as count,
                   COALESCE(SUM(expected_amount), 0) as total_expected,
                   COALESCE(SUM(commitment_amount), 0) as total_committed,
                   COALESCE(AVG(probability), 0) as avg_probability
            FROM opportunities
            GROUP BY stage
            ORDER BY CASE stage
                WHEN 'lead' THEN 1 WHEN 'outreach' THEN 2 WHEN 'meeting' THEN 3
                WHEN 'due_diligence' THEN 4 WHEN 'committed' THEN 5 WHEN 'funded' THEN 6
            END
        """).fetchall())

        by_owner = rows_to_list(conn.execute("""
            SELECT u.full_name as owner, op.stage, COUNT(*) as count,
                   COALESCE(SUM(op.expected_amount), 0) as total_expected
            FROM opportunities op
            LEFT JOIN users u ON op.owner_id = u.id
            GROUP BY op.owner_id, op.stage
            ORDER BY u.full_name, op.stage
        """).fetchall())

        by_priority = rows_to_list(conn.execute("""
            SELECT priority, COUNT(*) as count,
                   COALESCE(SUM(expected_amount), 0) as total_expected
            FROM opportunities
            WHERE stage NOT IN ('funded', 'lost')
            GROUP BY priority
        """).fetchall())

        conn.close()
        return self.send_json({
            "data": {
                "by_stage": stages,
                "by_owner": by_owner,
                "by_priority": by_priority
            }
        })

    def handle_lp_breakdown_report(self, user):
        conn = get_db()
        lps = rows_to_list(conn.execute("""
            SELECT lp.*, c.first_name, c.last_name, c.email, o.name as organization_name,
                   (SELECT MAX(communication_date) FROM communications WHERE contact_id = c.id) as last_contact_date,
                   (SELECT COUNT(*) FROM communications WHERE contact_id = c.id) as total_communications
            FROM lp_profiles lp
            LEFT JOIN contacts c ON lp.contact_id = c.id
            LEFT JOIN organizations o ON c.organization_id = o.id
            ORDER BY lp.commitment_amount DESC
        """).fetchall())

        summary = conn.execute("""
            SELECT COUNT(*) as total_lps,
                   COALESCE(SUM(commitment_amount), 0) as total_committed,
                   COALESCE(SUM(funded_amount), 0) as total_funded,
                   COALESCE(AVG(commitment_amount), 0) as avg_commitment,
                   MAX(commitment_amount) as largest_commitment,
                   MIN(CASE WHEN commitment_amount > 0 THEN commitment_amount END) as smallest_commitment
            FROM lp_profiles
        """).fetchone()

        by_type = rows_to_list(conn.execute("""
            SELECT COALESCE(investor_type, 'Unknown') as investor_type,
                   COUNT(*) as count,
                   COALESCE(SUM(commitment_amount), 0) as total_committed
            FROM lp_profiles
            GROUP BY investor_type
            ORDER BY total_committed DESC
        """).fetchall())

        conn.close()
        return self.send_json({
            "data": {
                "lps": lps,
                "summary": row_to_dict(summary),
                "by_type": by_type
            }
        })

    def handle_activity_report(self, user, params):
        conn = get_db()
        days = int(params.get('days', 30))

        by_user = rows_to_list(conn.execute("""
            SELECT u.full_name, cm.type, COUNT(*) as count
            FROM communications cm
            LEFT JOIN users u ON cm.created_by = u.id
            WHERE cm.communication_date >= date('now', ?)
            GROUP BY cm.created_by, cm.type
            ORDER BY u.full_name, cm.type
        """, (f'-{days} days',)).fetchall())

        by_day = rows_to_list(conn.execute("""
            SELECT date(communication_date) as date, type, COUNT(*) as count
            FROM communications
            WHERE communication_date >= date('now', ?)
            GROUP BY date(communication_date), type
            ORDER BY date DESC
        """, (f'-{days} days',)).fetchall())

        conn.close()
        return self.send_json({
            "data": {
                "by_user": by_user,
                "by_day": by_day,
                "period_days": days
            }
        })

    # ═══════════════════════════════════════════════════════════════════════════
    # IMPORT / EXPORT HANDLERS
    # ═══════════════════════════════════════════════════════════════════════════

    def handle_import_csv(self, user, body):
        """Import contacts from CSV data (sent as JSON array or raw CSV text)."""
        csv_data = body.get('data', [])
        entity_type = body.get('entity_type', 'contacts')
        mapping = body.get('mapping', {})
        dry_run = body.get('dry_run', False)
        update_existing = bool(body.get('update_existing', True))
        action_overrides_raw = body.get('action_overrides', {}) or {}

        if not csv_data:
            return self.send_error_json("No data provided. Send 'data' as array of objects.")

        conn = get_db()
        results = {"created": 0, "updated": 0, "skipped": 0, "errors": [], "matches": []}
        # Keep in-memory email matches so dry-run mirrors real behavior for
        # duplicate emails appearing multiple times in the same CSV batch.
        batch_email_matches = {}

        try:
            for i, row in enumerate(csv_data):
                try:
                    # Apply field mapping
                    mapped = {}
                    for csv_col, crm_field in mapping.items():
                        if csv_col in row:
                            mapped[crm_field] = row[csv_col]

                    # Use mapped data, fall back to raw row
                    data = mapped if mapping else row

                    if entity_type == 'contacts':
                        first_name = data.get('first_name', '').strip()
                        last_name = data.get('last_name', '').strip()

                        # Try splitting a 'name' field
                        if not first_name and not last_name and data.get('name'):
                            parts = data['name'].strip().split(' ', 1)
                            first_name = parts[0]
                            last_name = parts[1] if len(parts) > 1 else ''

                        if not first_name:
                            results['errors'].append(f"Row {i+1}: Missing first_name")
                            results['skipped'] += 1
                            continue

                        email = data.get('email', '').strip()
                        email_key = email.lower()
                        linkedin_url = data.get('linkedin_url', data.get('linkedin', '')).strip()
                        city = data.get('city', '').strip()
                        state = data.get('state', '').strip()
                        country = data.get('country', '').strip()
                        location_query = data.get('location_query', '').strip()
                        raw_location = data.get('location', data.get('city_location', data.get('city/location', ''))).strip()
                        if raw_location:
                            p_city, p_state, p_country, p_query = _parse_location_text(raw_location)
                            city = city or p_city
                            state = state or p_state
                            country = country or p_country
                            location_query = location_query or p_query

                        # Check for existing contact by email
                        existing = None
                        existing_summary = None
                        if email:
                            if email_key in batch_email_matches:
                                existing_summary = batch_email_matches[email_key]
                                existing = {"id": existing_summary.get('id')}
                            else:
                                existing = conn.execute("""
                                    SELECT c.id, c.first_name, c.last_name, c.email, o.name as organization_name
                                    FROM contacts c
                                    LEFT JOIN organizations o ON c.organization_id = o.id
                                    WHERE lower(c.email) = lower(?)
                                    ORDER BY c.updated_at DESC
                                    LIMIT 1
                                """, (email,)).fetchone()
                                if existing:
                                    existing_summary = {
                                        "id": existing['id'],
                                        "name": f"{str(existing['first_name'] or '').strip()} {str(existing['last_name'] or '').strip()}".strip(),
                                        "email": str(existing['email'] or ''),
                                        "organization": str(existing['organization_name'] or '')
                                    }
                                    batch_email_matches[email_key] = existing_summary

                        # Handle organization
                        org_id = None
                        org_name = data.get('organization', data.get('company', '')).strip()
                        if org_name:
                            org = conn.execute("SELECT id FROM organizations WHERE name = ?", (org_name,)).fetchone()
                            if org:
                                org_id = org['id']
                            elif not dry_run:
                                org_id = generate_id()
                                conn.execute(
                                    "INSERT INTO organizations (id, name, created_by) VALUES (?, ?, ?)",
                                    (org_id, org_name, user['user_id'])
                                )

                        action_override = None
                        if isinstance(action_overrides_raw, dict):
                            action_override = action_overrides_raw.get(str(i + 1)) or action_overrides_raw.get(i + 1)
                        default_action = 'update' if update_existing else 'skip'
                        action = action_override if action_override in ('update', 'skip', 'create_duplicate') else default_action
                        if existing:
                            incoming_name = f"{first_name} {last_name}".strip()
                            results['matches'].append({
                                "row": i + 1,
                                "incoming_name": incoming_name,
                                "incoming_email": email,
                                "incoming_organization": org_name,
                                "existing_id": existing_summary.get('id') if isinstance(existing_summary, dict) else existing['id'],
                                "existing_name": existing_summary.get('name') if isinstance(existing_summary, dict) else '',
                                "existing_email": existing_summary.get('email') if isinstance(existing_summary, dict) else email,
                                "existing_organization": existing_summary.get('organization') if isinstance(existing_summary, dict) else '',
                                "default_action": default_action,
                                "action": action
                            })

                        if not dry_run:
                            if existing:
                                if action == 'update':
                                    conn.execute("""
                                        UPDATE contacts SET first_name=?, last_name=?, phone=?, title=?,
                                        organization_id=COALESCE(?, organization_id),
                                        contact_type=COALESCE(?, contact_type),
                                        linkedin_url=COALESCE(?, linkedin_url),
                                        city=COALESCE(?, city),
                                        state=COALESCE(?, state),
                                        country=COALESCE(?, country),
                                        location_query=COALESCE(?, location_query),
                                        updated_at=?
                                        WHERE id=?
                                    """, (first_name, last_name, data.get('phone'),
                                          data.get('title'), org_id,
                                          data.get('contact_type'),
                                          linkedin_url if linkedin_url else None,
                                          city if city else None,
                                          state if state else None,
                                          country if country else None,
                                          location_query if location_query else None,
                                      now(), existing['id']))
                                    if email:
                                        batch_email_matches[email_key] = {
                                            "id": existing['id'],
                                            "name": f"{first_name} {last_name}".strip(),
                                            "email": email,
                                            "organization": org_name
                                        }
                                    updated_contact = row_to_dict(conn.execute("""
                                        SELECT c.*, o.name as organization_name
                                        FROM contacts c LEFT JOIN organizations o ON c.organization_id = o.id
                                        WHERE c.id = ?
                                    """, (existing['id'],)).fetchone())
                                    _sync_contact_to_fundraising_state(conn, updated_contact, actor_user_id=user['user_id'], remove=False)
                                    results['updated'] += 1
                                elif action == 'create_duplicate':
                                    contact_id = generate_id()
                                    conn.execute("""
                                        INSERT INTO contacts (id, first_name, last_name, email, phone,
                                            title, organization_id, contact_type, status, source,
                                            linkedin_url, city, state, country, location_query, created_by)
                                        VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'active', 'import', ?, ?, ?, ?, ?, ?)
                                    """, (contact_id, first_name, last_name, email,
                                          data.get('phone'), data.get('title'), org_id,
                                          data.get('contact_type', 'prospect'), linkedin_url,
                                          city, state, country, location_query, user['user_id']))
                                    if email:
                                        batch_email_matches[email_key] = {
                                            "id": contact_id,
                                            "name": f"{first_name} {last_name}".strip(),
                                            "email": email,
                                            "organization": org_name
                                        }
                                    created_contact = row_to_dict(conn.execute("""
                                        SELECT c.*, o.name as organization_name
                                        FROM contacts c LEFT JOIN organizations o ON c.organization_id = o.id
                                        WHERE c.id = ?
                                    """, (contact_id,)).fetchone())
                                    _sync_contact_to_fundraising_state(conn, created_contact, actor_user_id=user['user_id'], remove=False)
                                    results['created'] += 1
                                else:
                                    results['skipped'] += 1
                                    results['errors'].append(f"Row {i+1}: Existing contact matched by email; skipped")
                            else:
                                contact_id = generate_id()
                                conn.execute("""
                                    INSERT INTO contacts (id, first_name, last_name, email, phone,
                                        title, organization_id, contact_type, status, source,
                                        linkedin_url, city, state, country, location_query, created_by)
                                    VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'active', 'import', ?, ?, ?, ?, ?, ?)
                                """, (contact_id, first_name, last_name, email,
                                      data.get('phone'), data.get('title'), org_id,
                                      data.get('contact_type', 'prospect'), linkedin_url,
                                      city, state, country, location_query, user['user_id']))
                                if email:
                                    batch_email_matches[email_key] = {
                                        "id": contact_id,
                                        "name": f"{first_name} {last_name}".strip(),
                                        "email": email,
                                        "organization": org_name
                                    }
                                created_contact = row_to_dict(conn.execute("""
                                    SELECT c.*, o.name as organization_name
                                    FROM contacts c LEFT JOIN organizations o ON c.organization_id = o.id
                                    WHERE c.id = ?
                                """, (contact_id,)).fetchone())
                                _sync_contact_to_fundraising_state(conn, created_contact, actor_user_id=user['user_id'], remove=False)
                                results['created'] += 1
                        else:
                            if existing:
                                if action == 'update':
                                    results['updated'] += 1
                                    if email:
                                        batch_email_matches[email_key] = {
                                            "id": existing['id'],
                                            "name": f"{first_name} {last_name}".strip(),
                                            "email": email,
                                            "organization": org_name
                                        }
                                elif action == 'create_duplicate':
                                    results['created'] += 1
                                    if email:
                                        batch_email_matches[email_key] = {
                                            "id": f"dryrun-{i+1}",
                                            "name": f"{first_name} {last_name}".strip(),
                                            "email": email,
                                            "organization": org_name
                                        }
                                else:
                                    results['skipped'] += 1
                                    results['errors'].append(f"Row {i+1}: Existing contact matched by email; would be skipped")
                            else:
                                results['created'] += 1
                                if email:
                                    # Simulate that the row now exists for subsequent duplicate-email rows.
                                    batch_email_matches[email_key] = {
                                        "id": f"dryrun-{i+1}",
                                        "name": f"{first_name} {last_name}".strip(),
                                        "email": email,
                                        "organization": org_name
                                    }

                except Exception as e:
                    results['errors'].append(f"Row {i+1}: {str(e)}")
                    results['skipped'] += 1

            if not dry_run:
                conn.commit()

        except Exception as e:
            conn.rollback()
            conn.close()
            return self.send_error_json(f"Import failed: {str(e)}", 500)

        conn.close()
        return self.send_json({
            "data": results,
            "dry_run": dry_run
        })

    def handle_export_contacts(self, user, params):
        conn = get_db()
        contacts = rows_to_list(conn.execute("""
            SELECT c.*, o.name as organization_name
            FROM contacts c
            LEFT JOIN organizations o ON c.organization_id = o.id
            ORDER BY c.last_name, c.first_name
        """).fetchall())
        conn.close()
        return self.send_json({"data": contacts})

    # ═══════════════════════════════════════════════════════════════════════════
    # TAGS / USERS / AUDIT
    # ═══════════════════════════════════════════════════════════════════════════

    def handle_list_tags(self, user):
        conn = get_db()
        tags = rows_to_list(conn.execute("SELECT * FROM tags ORDER BY name").fetchall())
        conn.close()
        return self.send_json({"data": tags})

    def handle_create_tag(self, user, body):
        if not body.get('name'):
            return self.send_error_json("name is required")
        conn = get_db()
        tag_id = generate_id()
        try:
            conn.execute("INSERT INTO tags (id, name, color) VALUES (?, ?, ?)",
                         (tag_id, body['name'], body.get('color', '#6366f1')))
            conn.commit()
        except sqlite3.IntegrityError:
            conn.close()
            return self.send_error_json("Tag already exists")
        tag = row_to_dict(conn.execute("SELECT * FROM tags WHERE id = ?", (tag_id,)).fetchone())
        conn.close()
        return self.send_json({"data": tag}, 201)

    def handle_delete_tag(self, user, tag_id):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)
        conn = get_db()
        existing = conn.execute("SELECT id FROM tags WHERE id = ?", (tag_id,)).fetchone()
        if not existing:
            conn.close()
            return self.send_error_json("Tag not found", 404)
        conn.execute("DELETE FROM tags WHERE id = ?", (tag_id,))
        log_audit(conn, user['user_id'], 'tag', tag_id, 'delete')
        conn.commit()
        conn.close()
        return self.send_json({"message": "Tag deleted"})

    def handle_system_status(self, user):
        """System / search-index health for the in-app status view (DB-derived)."""
        conn = get_db()
        out = {}
        try:
            live = "deleted_at IS NULL"
            out['canonical_entities'] = {
                'lp': conn.execute(f"SELECT COUNT(*) FROM canonical_entities WHERE entity_kind='lp' AND {live}").fetchone()[0],
                'organization': conn.execute(f"SELECT COUNT(*) FROM canonical_entities WHERE entity_kind='organization' AND {live}").fetchone()[0],
                'person': conn.execute(f"SELECT COUNT(*) FROM canonical_entities WHERE entity_kind='person' AND {live}").fetchone()[0],
            }
            out['entity_links'] = conn.execute("SELECT COUNT(*) FROM entity_links").fetchone()[0]
        except Exception:
            out['canonical_entities'] = None
        try:
            r = conn.execute("SELECT ts, payload FROM interaction_log WHERE action='ingest.sync' ORDER BY ts DESC LIMIT 1").fetchone()
            out['last_index_sync'] = ({'ts': r['ts'], **json.loads(r['payload'] or '{}')} if r else None)
        except Exception:
            out['last_index_sync'] = None
        try:
            out['thesis'] = {
                'lines': conn.execute("SELECT COUNT(*) FROM thesis_lines WHERE deleted_at IS NULL").fetchone()[0],
                'canonical_versions': conn.execute("SELECT COUNT(*) FROM thesis_versions WHERE status='canonical'").fetchone()[0],
                'in_review': conn.execute("SELECT COUNT(*) FROM thesis_versions WHERE status='in_review'").fetchone()[0],
            }
        except Exception:
            out['thesis'] = None
        try:
            out['recent_activity'] = [dict(r) for r in conn.execute(
                "SELECT ts, actor_type, actor_id, action FROM interaction_log ORDER BY ts DESC LIMIT 12")]
        except Exception:
            out['recent_activity'] = []
        try:
            out['pending_merge_candidates'] = conn.execute(
                "SELECT COUNT(*) FROM entity_merge_candidates WHERE status='pending'").fetchone()[0]
        except Exception:
            out['pending_merge_candidates'] = None
        out['index_job'] = entity_jobs.get_status() if entity_jobs else None
        # Raw source-record counts, so the resolved canonical numbers can be
        # sanity-checked against what's actually in the CRM.
        try:
            out['source_counts'] = {
                'contacts': conn.execute("SELECT COUNT(*) FROM contacts WHERE deleted_at IS NULL").fetchone()[0],
                'organizations': conn.execute("SELECT COUNT(*) FROM organizations WHERE deleted_at IS NULL").fetchone()[0],
                'fundraising_investors': conn.execute("SELECT COUNT(*) FROM fundraising_investors").fetchone()[0],
                'fundraising_contacts': conn.execute("SELECT COUNT(*) FROM fundraising_contacts").fetchone()[0],
            }
        except Exception:
            out['source_counts'] = None
        conn.close()
        self.send_json({"data": out})

    # ─── UI-triggered index jobs + entity-merge review (Phase 1) ───
    def handle_index_job(self, user, kind):
        if not require_admin(user):
            return self.send_error_json("Admin required", 403)
        if entity_jobs is None:
            return self.send_error_json("Jobs unavailable", 503)
        res = entity_jobs.start(kind, DB_PATH)
        if res.get('error'):
            return self.send_error_json(res['error'], 409)
        return self.send_json({"data": res})

    def handle_list_merge_candidates(self, user, params):
        if not require_admin(user):
            return self.send_error_json("Admin required", 403)
        if entity_merge is None:
            return self.send_error_json("Unavailable", 503)
        return self.send_json(entity_merge.list_candidates(DB_PATH, params.get('status', 'pending')))

    def handle_decide_merge_candidate(self, user, candidate_id, body):
        if not require_admin(user):
            return self.send_error_json("Admin required", 403)
        if entity_merge is None:
            return self.send_error_json("Unavailable", 503)
        res = entity_merge.decide(DB_PATH, candidate_id, (body or {}).get('decision'), user['user_id'])
        if res.get('error'):
            return self.send_error_json(res['error'], 400)
        return self.send_json({"data": res})

    # ─── Architect thesis (Phase 1) ───
    def handle_list_thesis_lines(self, user):
        if thesis_review is None:
            return self.send_error_json("Thesis module unavailable", 503)
        return self.send_json(thesis_review.list_lines(DB_PATH))

    def handle_list_thesis_review_queue(self, user):
        if thesis_review is None:
            return self.send_error_json("Thesis module unavailable", 503)
        return self.send_json(thesis_review.list_versions_for_review(DB_PATH))

    def handle_get_thesis_version(self, user, version_id):
        if thesis_review is None:
            return self.send_error_json("Thesis module unavailable", 503)
        return self.send_json(thesis_review.get_version(DB_PATH, version_id))

    def handle_get_canonical_thesis(self, user, line_key):
        if thesis_review is None:
            return self.send_error_json("Thesis module unavailable", 503)
        return self.send_json(thesis_review.get_canonical(DB_PATH, line_key))

    def handle_thesis_review(self, user, version_id, body):
        # Promotion to canonical is a human partner action (guardrail #4).
        if not require_admin(user):
            return self.send_error_json("Admin required", 403)
        if thesis_review is None:
            return self.send_error_json("Thesis module unavailable", 503)
        body = body or {}
        res = thesis_review.record_review(DB_PATH, version_id, user['user_id'],
                                          body.get('decision'), body.get('feedback'),
                                          body.get('target_node_id'))
        if res.get('error'):
            return self.send_error_json(res['error'], 400)
        return self.send_json({"data": res})

    def handle_list_users(self, user):
        conn = get_db()
        users = rows_to_list(conn.execute(
            "SELECT id, username, email, full_name, role, is_active, created_at FROM users ORDER BY full_name"
        ).fetchall())
        conn.close()
        return self.send_json({"data": users})

    def handle_admin_create_user(self, user, body):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)

        required = ['username', 'password', 'email', 'full_name']
        for field in required:
            if not str(body.get(field, '')).strip():
                return self.send_error_json(f"{field} is required")
        if len(str(body.get('password') or '').strip()) < 8:
            return self.send_error_json("password must be at least 8 characters")

        role = body.get('role', 'member')
        if role not in ('admin', 'member'):
            return self.send_error_json("role must be admin or member")

        conn = get_db()
        existing = conn.execute(
            "SELECT id FROM users WHERE username = ? OR email = ?",
            (body['username'].strip(), body['email'].strip())
        ).fetchone()
        if existing:
            conn.close()
            return self.send_error_json("Username or email already exists")

        user_id = generate_id()
        conn.execute(
            "INSERT INTO users (id, username, email, password_hash, full_name, role) VALUES (?, ?, ?, ?, ?, ?)",
            (
                user_id,
                body['username'].strip(),
                body['email'].strip(),
                hash_password(body['password']),
                body['full_name'].strip(),
                role
            )
        )
        log_audit(conn, user['user_id'], 'user', user_id, 'create', {"username": body['username'].strip(), "role": role})
        conn.commit()
        created = row_to_dict(conn.execute(
            "SELECT id, username, email, full_name, role, is_active, created_at FROM users WHERE id = ?",
            (user_id,)
        ).fetchone())
        conn.close()
        return self.send_json({"data": created}, 201)

    def handle_admin_update_user(self, user, target_user_id, body):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)

        conn = get_db()
        existing = conn.execute(
            "SELECT id, username, full_name, role, is_active FROM users WHERE id = ?",
            (target_user_id,)
        ).fetchone()
        if not existing:
            conn.close()
            return self.send_error_json("User not found", 404)

        sets = []
        args = []

        if 'is_active' in body:
            next_active = 1 if bool(body.get('is_active')) else 0
            # prevent locking out the currently authenticated admin accidentally
            if existing['id'] == user.get('user_id') and next_active == 0:
                conn.close()
                return self.send_error_json("You cannot deactivate your own account", 400)
            sets.append("is_active = ?")
            args.append(next_active)

        if 'role' in body:
            role = str(body.get('role'))
            if role not in ('admin', 'member'):
                conn.close()
                return self.send_error_json("role must be admin or member")
            sets.append("role = ?")
            args.append(role)

        if 'password' in body:
            password = str(body.get('password') or '')
            if len(password.strip()) < 8:
                conn.close()
                return self.send_error_json("password must be at least 8 characters")
            sets.append("password_hash = ?")
            args.append(hash_password(password))

        if not sets:
            conn.close()
            return self.send_error_json("No fields to update")

        sets.append("updated_at = ?")
        args.append(now())
        args.append(target_user_id)
        conn.execute(f"UPDATE users SET {', '.join(sets)} WHERE id = ?", args)
        audit_payload = dict(body)
        if 'password' in audit_payload:
            audit_payload['password'] = '[REDACTED]'
        log_audit(conn, user['user_id'], 'user', target_user_id, 'update', audit_payload)
        conn.commit()
        updated = row_to_dict(conn.execute(
            "SELECT id, username, email, full_name, role, is_active, created_at FROM users WHERE id = ?",
            (target_user_id,)
        ).fetchone())
        conn.close()
        return self.send_json({"data": updated})

    def handle_admin_reset_all_data(self, user, body):
        if not require_admin(user):
            return self.send_error_json("Admin only", 403)

        confirm_phrase = str(body.get('confirm_phrase') or '').strip()
        if confirm_phrase != 'RESET ALL DATA':
            return self.send_error_json("Confirmation phrase must be exactly: RESET ALL DATA", 400)

        conn = get_db()
        try:
            ensure_fundraising_state_row(conn)
            state = conn.execute("SELECT * FROM fundraising_state WHERE id = 'main'").fetchone()
            pre_backup = create_fundraising_backup_file(state, kind="pre_restore") if state else None

            conn.execute("DELETE FROM communications")
            conn.execute("DELETE FROM opportunities")
            conn.execute("DELETE FROM lp_profiles")
            conn.execute("DELETE FROM custom_field_values")
            conn.execute("DELETE FROM custom_fields")
            conn.execute("DELETE FROM feature_requests")
            conn.execute("DELETE FROM contacts")
            conn.execute("DELETE FROM organizations")

            default_grid = {
                "columns": deep_copy_json(DEFAULT_FUNDRAISING_COLUMNS),
                "rows": deep_copy_json(DEFAULT_FUNDRAISING_ROWS)
            }
            default_views = sanitize_grid_views(deep_copy_json(DEFAULT_GRID_VIEWS))
            conn.execute("""
                UPDATE fundraising_state
                SET grid_json = ?, views_json = ?, version = COALESCE(version, 1) + 1, updated_by = ?, updated_at = ?
                WHERE id = 'main'
            """, (json.dumps(default_grid), json.dumps(default_views), user['user_id'], now()))
            sync_fundraising_relational(conn, default_grid, default_views, actor_user_id=user['user_id'])

            log_audit(conn, user['user_id'], 'system', 'all-data', 'reset', {
                "pre_backup": pre_backup['filename'] if pre_backup else None
            })
            conn.commit()
        except Exception as exc:
            conn.rollback()
            conn.close()
            return self.send_error_json(f"Failed to reset data: {str(exc)}", 500)

        conn.close()
        return self.send_json({
            "data": {
                "status": "ok",
                "pre_backup": pre_backup
            }
        })

    def handle_list_audit_log(self, user, params):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)
        conn = get_db()
        query = """
            SELECT al.*, u.full_name as user_name
            FROM audit_log al LEFT JOIN users u ON al.user_id = u.id
            WHERE 1=1
        """
        args = []
        if params.get('entity_type'):
            query += " AND al.entity_type = ?"
            args.append(params['entity_type'])
        if params.get('entity_id'):
            query += " AND al.entity_id = ?"
            args.append(params['entity_id'])
        if params.get('action'):
            query += " AND al.action = ?"
            args.append(params['action'])
        if params.get('user_id'):
            query += " AND al.user_id = ?"
            args.append(params['user_id'])
        if params.get('date_from'):
            query += " AND al.created_at >= ?"
            args.append(params['date_from'])
        if params.get('date_to'):
            query += " AND al.created_at <= ?"
            args.append(params['date_to'])
        if params.get('search'):
            search = f"%{str(params.get('search')).strip()}%"
            query += " AND (al.entity_id LIKE ? OR al.entity_type LIKE ? OR al.action LIKE ? OR al.changes LIKE ?)"
            args.extend([search, search, search, search])

        try:
            limit = max(1, min(500, int(params.get('limit') or 100)))
        except Exception:
            limit = 100
        query += " ORDER BY al.created_at DESC LIMIT ?"
        args.append(limit)
        logs = rows_to_list(conn.execute(query, args).fetchall())
        conn.close()
        return self.send_json({"data": logs})

    def handle_get_backup_policy(self, user):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)
        conn = get_db()
        policy = load_backup_policy(conn)
        conn.close()
        last_run_dt = parse_iso_utc(policy.get('last_run_at'))
        next_run_at = None
        if policy.get('enabled'):
            if last_run_dt:
                next_run_at = (last_run_dt.replace(tzinfo=None) + timedelta(hours=int(policy.get('interval_hours') or 24))).isoformat() + "Z"
            else:
                next_run_at = now()
        return self.send_json({"data": {**policy, "next_run_at": next_run_at}})

    def handle_update_backup_policy(self, user, body):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)
        conn = get_db()
        policy = load_backup_policy(conn)
        if 'enabled' in body:
            policy['enabled'] = bool(body.get('enabled'))
        if 'interval_hours' in body:
            try:
                policy['interval_hours'] = max(1, min(168, int(body.get('interval_hours'))))
            except Exception:
                conn.close()
                return self.send_error_json("interval_hours must be an integer from 1 to 168")
        if 'retention_days' in body:
            try:
                policy['retention_days'] = max(1, min(365, int(body.get('retention_days'))))
            except Exception:
                conn.close()
                return self.send_error_json("retention_days must be an integer from 1 to 365")
        if 'max_backups' in body:
            try:
                policy['max_backups'] = max(1, min(1000, int(body.get('max_backups'))))
            except Exception:
                conn.close()
                return self.send_error_json("max_backups must be an integer from 1 to 1000")
        updated = save_backup_policy(conn, policy)
        apply_backup_retention(updated)
        conn.commit()
        conn.close()
        return self.send_json({"data": updated})

    def handle_get_fundraising_relational_summary(self, user):
        conn = get_db()
        counts = {
            "investors": conn.execute("SELECT COUNT(*) AS c FROM fundraising_investors").fetchone()['c'],
            "contacts": conn.execute("SELECT COUNT(*) AS c FROM fundraising_contacts").fetchone()['c'],
            "funds": conn.execute("SELECT COUNT(*) AS c FROM fundraising_funds WHERE active = 1").fetchone()['c'],
            "commitments": conn.execute("SELECT COUNT(*) AS c FROM fundraising_commitments").fetchone()['c'],
            "views": conn.execute("SELECT COUNT(*) AS c FROM fundraising_views").fetchone()['c']
        }
        totals = conn.execute("""
            SELECT
                COALESCE(SUM(total_invested), 0) AS total_invested,
                SUM(CASE WHEN graveyard = 1 THEN 1 ELSE 0 END) AS graveyard_count,
                SUM(CASE WHEN follow_up = 1 THEN 1 ELSE 0 END) AS follow_up_count
            FROM fundraising_investors
        """).fetchone()
        top_funds = rows_to_list(conn.execute("""
            SELECT f.fund_name, f.column_id, COALESCE(SUM(c.amount), 0) AS total_commitment
            FROM fundraising_funds f
            LEFT JOIN fundraising_commitments c ON c.fund_id = f.id
            WHERE f.active = 1
            GROUP BY f.id
            ORDER BY total_commitment DESC
            LIMIT 10
        """).fetchall())
        conn.close()
        return self.send_json({
            "data": {
                "counts": counts,
                "totals": {
                    "total_invested": totals['total_invested'] if totals else 0,
                    "graveyard_count": totals['graveyard_count'] if totals and totals['graveyard_count'] is not None else 0,
                    "follow_up_count": totals['follow_up_count'] if totals and totals['follow_up_count'] is not None else 0
                },
                "top_funds": top_funds
            }
        })

    def handle_list_fundraising_automations(self, user):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)
        conn = get_db()
        ensure_default_automation_rules(conn)
        rules = rows_to_list(conn.execute("""
            SELECT * FROM fundraising_automation_rules
            ORDER BY name ASC
        """).fetchall())
        conn.commit()
        conn.close()
        for r in rules:
            try:
                r['condition_json'] = json.loads(r.get('condition_json') or '{}')
            except Exception:
                r['condition_json'] = {}
            try:
                r['action_json'] = json.loads(r.get('action_json') or '{}')
            except Exception:
                r['action_json'] = {}
        return self.send_json({"data": rules, "total": len(rules)})

    def handle_update_fundraising_automation_rule(self, user, rule_id, body):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)
        conn = get_db()
        existing = conn.execute("SELECT * FROM fundraising_automation_rules WHERE id = ?", (rule_id,)).fetchone()
        if not existing:
            conn.close()
            return self.send_error_json("Automation rule not found", 404)
        sets = []
        args = []
        if 'enabled' in body:
            sets.append("enabled = ?")
            args.append(1 if _to_bool(body.get('enabled')) else 0)
        if 'name' in body:
            sets.append("name = ?")
            args.append(str(body.get('name') or '').strip() or existing['name'])
        if 'condition_json' in body:
            sets.append("condition_json = ?")
            args.append(json.dumps(body.get('condition_json') if isinstance(body.get('condition_json'), dict) else {}))
        if 'action_json' in body:
            sets.append("action_json = ?")
            args.append(json.dumps(body.get('action_json') if isinstance(body.get('action_json'), dict) else {}))
        if not sets:
            conn.close()
            return self.send_error_json("No fields to update")
        sets.append("updated_at = ?")
        args.append(now())
        args.append(rule_id)
        conn.execute(f"UPDATE fundraising_automation_rules SET {', '.join(sets)} WHERE id = ?", args)
        run_fundraising_automations(conn)
        log_audit(conn, user['user_id'], 'fundraising_automation_rule', rule_id, 'update', body)
        conn.commit()
        updated = row_to_dict(conn.execute("SELECT * FROM fundraising_automation_rules WHERE id = ?", (rule_id,)).fetchone())
        conn.close()
        try:
            updated['condition_json'] = json.loads(updated.get('condition_json') or '{}')
        except Exception:
            updated['condition_json'] = {}
        try:
            updated['action_json'] = json.loads(updated.get('action_json') or '{}')
        except Exception:
            updated['action_json'] = {}
        return self.send_json({"data": updated})

    def handle_list_fundraising_automation_runs(self, user, params):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)
        limit = 100
        try:
            limit = max(1, min(500, int(params.get('limit') or 100)))
        except Exception:
            limit = 100
        conn = get_db()
        rows = rows_to_list(conn.execute("""
            SELECT r.*, i.investor_name, ar.name as rule_name
            FROM fundraising_automation_runs r
            LEFT JOIN fundraising_investors i ON r.investor_id = i.id
            LEFT JOIN fundraising_automation_rules ar ON r.rule_id = ar.id
            ORDER BY r.created_at DESC
            LIMIT ?
        """, (limit,)).fetchall())
        conn.close()
        for row in rows:
            try:
                row['result_json'] = json.loads(row.get('result_json') or '{}')
            except Exception:
                row['result_json'] = {}
        return self.send_json({"data": rows, "total": len(rows)})

    def handle_verify_fundraising_backups(self, user):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)
        checked = 0
        valid = 0
        invalid = []
        for b in list_backups():
            checked += 1
            try:
                with open(b['path'], 'r', encoding='utf-8') as f:
                    payload = json.load(f)
                if not isinstance(payload, dict):
                    raise ValueError("payload is not an object")
                if not isinstance(payload.get('grid'), dict):
                    raise ValueError("missing grid")
                if not isinstance(payload.get('views'), list):
                    raise ValueError("missing views")
                if not isinstance(payload['grid'].get('columns'), list) or not isinstance(payload['grid'].get('rows'), list):
                    raise ValueError("grid missing columns/rows")
                valid += 1
            except Exception as exc:
                invalid.append({"filename": b['filename'], "error": str(exc)})
        return self.send_json({
            "data": {
                "checked": checked,
                "valid": valid,
                "invalid": invalid,
                "invalid_count": len(invalid)
            }
        })

    def handle_get_fundraising_activity(self, user, params):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)
        limit = 120
        try:
            limit = max(20, min(500, int(params.get('limit') or 120)))
        except Exception:
            limit = 120
        conn = get_db()
        audit_rows = rows_to_list(conn.execute("""
            SELECT al.id, al.created_at, 'audit' AS source, al.entity_type, al.entity_id, al.action, u.full_name as actor_name
            FROM audit_log al
            LEFT JOIN users u ON al.user_id = u.id
            ORDER BY al.created_at DESC
            LIMIT ?
        """, (limit,)).fetchall())
        auto_rows = rows_to_list(conn.execute("""
            SELECT r.id, r.created_at, 'automation' AS source, 'fundraising_automation' AS entity_type, COALESCE(i.source_row_id, r.investor_id) AS entity_id,
                   r.status AS action, ar.name AS actor_name
            FROM fundraising_automation_runs r
            LEFT JOIN fundraising_automation_rules ar ON r.rule_id = ar.id
            LEFT JOIN fundraising_investors i ON r.investor_id = i.id
            ORDER BY r.created_at DESC
            LIMIT ?
        """, (limit,)).fetchall())
        conn.close()
        backup_rows = []
        for b in list_backups()[:limit]:
            backup_rows.append({
                "id": f"backup:{b['filename']}",
                "created_at": b['modified_at'],
                "source": "backup",
                "entity_type": "fundraising_backup",
                "entity_id": b['filename'],
                "action": b['kind'],
                "actor_name": "system"
            })
        combined = audit_rows + auto_rows + backup_rows
        combined.sort(key=lambda x: str(x.get('created_at') or ''), reverse=True)
        return self.send_json({"data": combined[:limit], "total": len(combined[:limit])})

    def handle_security_status(self, user):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)
        has_custom_secret = bool(os.environ.get("CRM_SECRET_KEY"))
        return self.send_json({
            "data": {
                "env": ENV,
                "auth_required": True,
                "cors_origin": CORS_ORIGIN,
                "has_custom_secret": has_custom_secret,
                "warnings": [] if has_custom_secret else ["CRM_SECRET_KEY is not explicitly set"],
                "rate_limits": {
                    "login_per_min": LOGIN_RATE_LIMIT_PER_MIN,
                    "write_per_min": WRITE_RATE_LIMIT_PER_MIN
                }
            }
        })

    # ═══════════════════════════════════════════════════════════════════════════
    # FEATURE REQUESTS
    # ═══════════════════════════════════════════════════════════════════════════

    def handle_list_feature_requests(self, user, params):
        conn = get_db()
        query = """
            SELECT fr.*, u.full_name as requested_by_name
            FROM feature_requests fr
            LEFT JOIN users u ON fr.requested_by_user_id = u.id
            WHERE 1=1
        """
        args = []

        if params.get('status'):
            query += " AND fr.status = ?"
            args.append(params['status'])

        if params.get('search'):
            search = f"%{params['search']}%"
            query += " AND (fr.title LIKE ? OR fr.description LIKE ? OR fr.requested_by LIKE ?)"
            args.extend([search, search, search])

        query += " ORDER BY fr.created_at DESC"
        rows = rows_to_list(conn.execute(query, args).fetchall())
        conn.close()
        return self.send_json({"data": rows, "total": len(rows)})

    def handle_create_feature_request(self, user, body):
        title = str(body.get('title', '')).strip()
        if not title:
            return self.send_error_json("title is required")

        req_id = generate_id()
        requested_by = str(body.get('requested_by') or user.get('username') or '').strip()
        conn = get_db()
        conn.execute("""
            INSERT INTO feature_requests (
                id, title, description, page, category, priority, status,
                requested_by, requested_by_user_id
            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
        """, (
            req_id,
            title,
            body.get('description'),
            body.get('page'),
            body.get('category', 'general'),
            body.get('priority', 'medium'),
            body.get('status', 'new'),
            requested_by,
            user['user_id']
        ))
        log_audit(conn, user['user_id'], 'feature_request', req_id, 'create', {"title": title})
        conn.commit()
        row = row_to_dict(conn.execute("SELECT * FROM feature_requests WHERE id = ?", (req_id,)).fetchone())
        conn.close()
        return self.send_json({"data": row}, 201)

    def handle_update_feature_request(self, user, req_id, body):
        conn = get_db()
        existing = conn.execute("SELECT * FROM feature_requests WHERE id = ?", (req_id,)).fetchone()
        if not existing:
            conn.close()
            return self.send_error_json("Feature request not found", 404)

        updatable = ['title', 'description', 'page', 'category', 'priority', 'status', 'requested_by']
        sets = []
        args = []
        for field in updatable:
            if field in body:
                sets.append(f"{field} = ?")
                args.append(body[field])

        if not sets:
            conn.close()
            return self.send_error_json("No fields to update")

        sets.append("updated_at = ?")
        args.append(now())
        args.append(req_id)
        conn.execute(f"UPDATE feature_requests SET {', '.join(sets)} WHERE id = ?", args)
        log_audit(conn, user['user_id'], 'feature_request', req_id, 'update', body)
        conn.commit()
        row = row_to_dict(conn.execute("SELECT * FROM feature_requests WHERE id = ?", (req_id,)).fetchone())
        conn.close()
        return self.send_json({"data": row})

    # ═══════════════════════════════════════════════════════════════════════════
    # FUNDRAISING STATE (AIRTABLE-LIKE GRID)
    # ═══════════════════════════════════════════════════════════════════════════

    def _ensure_fundraising_state_row(self, conn):
        ensure_fundraising_state_row(conn)

    def handle_get_fundraising_state(self, user):
        conn = get_db()
        self._ensure_fundraising_state_row(conn)
        row = conn.execute("SELECT * FROM fundraising_state WHERE id = 'main'").fetchone()
        conn.close()

        try:
            grid = json.loads(row['grid_json']) if row and row['grid_json'] else {}
        except json.JSONDecodeError:
            grid = {}
        try:
            views = json.loads(row['views_json']) if row and row['views_json'] else []
        except json.JSONDecodeError:
            views = []
        views = sanitize_grid_views(views)

        grid = sanitize_fundraising_grid(grid)
        if not isinstance(views, list):
            views = []
        columns = grid.get('columns', [])
        rows = grid.get('rows', [])

        return self.send_json({
            "data": {
                "grid": {"columns": columns, "rows": rows},
                "views": views if views else deep_copy_json(DEFAULT_GRID_VIEWS),
                "version": row['version'] if row else 1,
                "updated_at": row['updated_at'] if row else None
            }
        })

    def _cleanup_fundraising_collab(self, conn):
        now_epoch = int(time.time())
        conn.execute("DELETE FROM fundraising_presence WHERE expires_at_epoch <= ?", (now_epoch,))
        conn.execute("DELETE FROM fundraising_cell_locks WHERE expires_at_epoch <= ?", (now_epoch,))

    def _list_fundraising_collab_state(self, conn):
        presence_rows = rows_to_list(conn.execute("""
            SELECT user_id, username, full_name, active_view, row_id, col_id, is_editing, cell_key, last_seen_at
            FROM fundraising_presence
            ORDER BY last_seen_at DESC
        """).fetchall())
        lock_rows = rows_to_list(conn.execute("""
            SELECT cell_key, row_id, col_id, locked_by_user_id, locked_by_username, locked_by_full_name, last_seen_at
            FROM fundraising_cell_locks
            ORDER BY last_seen_at DESC
        """).fetchall())
        for row in presence_rows:
            row['is_editing'] = bool(row.get('is_editing'))
        return {"presence": presence_rows, "locks": lock_rows}

    def handle_get_fundraising_collab_state(self, user):
        conn = get_db()
        self._cleanup_fundraising_collab(conn)
        snapshot = self._list_fundraising_collab_state(conn)
        conn.commit()
        conn.close()
        return self.send_json({"data": snapshot})

    def handle_fundraising_collab_heartbeat(self, user, body):
        active_view = str(body.get('active_view') or '').strip()
        selected = body.get('selected') if isinstance(body.get('selected'), dict) else {}
        editing = body.get('editing') if isinstance(body.get('editing'), dict) else {}
        selected_row_id = str(selected.get('row_id') or '').strip()
        selected_col_id = str(selected.get('col_id') or '').strip()
        editing_row_id = str(editing.get('row_id') or '').strip()
        editing_col_id = str(editing.get('col_id') or '').strip()
        is_editing = bool(editing_row_id and editing_col_id)
        ttl_seconds = int(body.get('ttl_seconds') or 25)
        ttl_seconds = max(10, min(120, ttl_seconds))
        now_epoch = int(time.time())
        expires_at_epoch = now_epoch + ttl_seconds
        seen_at = now()
        lock_conflict = None

        conn = get_db()
        self._cleanup_fundraising_collab(conn)

        user_row = conn.execute("SELECT username, full_name FROM users WHERE id = ?", (user['user_id'],)).fetchone()
        username = str(user_row['username']) if user_row and user_row['username'] else str(user.get('username') or '')
        full_name = str(user_row['full_name']) if user_row and user_row['full_name'] else ''
        editing_cell_key = f"{editing_row_id}:{editing_col_id}" if is_editing else None

        if is_editing and editing_cell_key:
            existing_lock = conn.execute("""
                SELECT cell_key, row_id, col_id, locked_by_user_id, locked_by_username, locked_by_full_name, last_seen_at
                FROM fundraising_cell_locks
                WHERE cell_key = ? AND locked_by_user_id != ? AND expires_at_epoch > ?
                LIMIT 1
            """, (editing_cell_key, user['user_id'], now_epoch)).fetchone()
            if existing_lock:
                lock_conflict = row_to_dict(existing_lock)
                is_editing = False
                editing_cell_key = None
            else:
                conn.execute("""
                    INSERT INTO fundraising_cell_locks (
                        cell_key, row_id, col_id, locked_by_user_id, locked_by_username, locked_by_full_name, last_seen_at, expires_at_epoch
                    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
                    ON CONFLICT(cell_key) DO UPDATE SET
                        row_id = excluded.row_id,
                        col_id = excluded.col_id,
                        locked_by_user_id = excluded.locked_by_user_id,
                        locked_by_username = excluded.locked_by_username,
                        locked_by_full_name = excluded.locked_by_full_name,
                        last_seen_at = excluded.last_seen_at,
                        expires_at_epoch = excluded.expires_at_epoch
                """, (editing_cell_key, editing_row_id, editing_col_id, user['user_id'], username, full_name, seen_at, expires_at_epoch))
                conn.execute("""
                    DELETE FROM fundraising_cell_locks
                    WHERE locked_by_user_id = ? AND cell_key != ?
                """, (user['user_id'], editing_cell_key))
        else:
            conn.execute("DELETE FROM fundraising_cell_locks WHERE locked_by_user_id = ?", (user['user_id'],))

        conn.execute("""
            INSERT INTO fundraising_presence (
                user_id, username, full_name, active_view, row_id, col_id, is_editing, cell_key, last_seen_at, expires_at_epoch
            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
            ON CONFLICT(user_id) DO UPDATE SET
                username = excluded.username,
                full_name = excluded.full_name,
                active_view = excluded.active_view,
                row_id = excluded.row_id,
                col_id = excluded.col_id,
                is_editing = excluded.is_editing,
                cell_key = excluded.cell_key,
                last_seen_at = excluded.last_seen_at,
                expires_at_epoch = excluded.expires_at_epoch
        """, (
            user['user_id'],
            username,
            full_name,
            active_view,
            selected_row_id or editing_row_id,
            selected_col_id or editing_col_id,
            1 if is_editing else 0,
            editing_cell_key,
            seen_at,
            expires_at_epoch
        ))

        snapshot = self._list_fundraising_collab_state(conn)
        conn.commit()
        conn.close()

        return self.send_json({
            "data": {
                **snapshot,
                "lock_conflict": lock_conflict
            }
        })

    def handle_update_fundraising_state(self, user, body):
        grid = body.get('grid', {})
        views = body.get('views')
        expected_version = body.get('expected_version')

        if not isinstance(grid, dict):
            return self.send_error_json("grid must be an object")
        if 'columns' not in grid or 'rows' not in grid:
            return self.send_error_json("grid must include columns and rows")
        if not isinstance(grid.get('columns'), list) or not isinstance(grid.get('rows'), list):
            return self.send_error_json("grid.columns and grid.rows must be arrays")
        grid = sanitize_fundraising_grid(grid)
        if views is not None and not isinstance(views, list):
            return self.send_error_json("views must be an array when provided")

        conn = get_db()
        self._ensure_fundraising_state_row(conn)

        current = conn.execute("SELECT version FROM fundraising_state WHERE id = 'main'").fetchone()
        current_version = int(current['version']) if current else 1
        if expected_version is not None and int(expected_version) != current_version:
            snapshot = conn.execute("SELECT version, updated_at, updated_by FROM fundraising_state WHERE id = 'main'").fetchone()
            conn.close()
            return self.send_json({
                "error": "Version conflict",
                "current_version": current_version,
                "current_updated_at": snapshot['updated_at'] if snapshot else None,
                "current_updated_by": snapshot['updated_by'] if snapshot else None
            }, status=409)

        row = conn.execute("SELECT views_json FROM fundraising_state WHERE id = 'main'").fetchone()
        existing_views = []
        if row and row['views_json']:
            try:
                existing_views = json.loads(row['views_json'])
            except json.JSONDecodeError:
                existing_views = deep_copy_json(DEFAULT_GRID_VIEWS)
        existing_views = sanitize_grid_views(existing_views)

        next_views = sanitize_grid_views(views if views is not None else existing_views)
        next_version = current_version + 1
        conn.execute("""
            UPDATE fundraising_state
            SET grid_json = ?, views_json = ?, version = ?, updated_by = ?, updated_at = ?
            WHERE id = 'main'
        """, (json.dumps(grid), json.dumps(next_views), next_version, user['user_id'], now()))
        sync_fundraising_relational(conn, grid, next_views, actor_user_id=user['user_id'])
        log_audit(conn, user['user_id'], 'fundraising_state', 'main', 'update', {"version": next_version})
        conn.commit()
        conn.close()

        return self.send_json({
            "data": {
                "version": next_version,
                "updated_at": now()
            }
        })

    def handle_export_fundraising_state(self, user):
        conn = get_db()
        self._ensure_fundraising_state_row(conn)
        row = conn.execute("SELECT * FROM fundraising_state WHERE id = 'main'").fetchone()
        conn.close()

        if not row:
            return self.send_error_json("Fundraising state not found", 404)

        try:
            grid = json.loads(row['grid_json']) if row['grid_json'] else {}
        except json.JSONDecodeError:
            grid = {}
        try:
            views = json.loads(row['views_json']) if row['views_json'] else []
        except json.JSONDecodeError:
            views = []

        payload = {
            "exported_at": now(),
            "version": row['version'],
            "updated_at": row['updated_at'],
            "grid": grid,
            "views": views
        }
        return self.send_json({"data": payload})

    def handle_list_fundraising_backups(self, user):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)
        entries = list_backups()
        return self.send_json({"data": entries, "total": len(entries)})

    def _extract_fundraising_restore_payload(self, body):
        payload = body.get('payload')
        if payload is None:
            payload = body.get('data')

        if payload is None and body.get('filename'):
            filename = os.path.basename(str(body.get('filename')))
            if not filename.endswith('.json'):
                return None, "filename must point to a .json backup file"
            backup_dir = os.path.join(DATA_DIR, "backups")
            candidate = os.path.join(backup_dir, filename)
            if not os.path.isfile(candidate):
                return None, "backup file not found"
            try:
                with open(candidate, 'r', encoding='utf-8') as f:
                    payload = json.load(f)
            except Exception:
                return None, "backup file is not valid JSON"

        if payload is None:
            payload = body

        if not isinstance(payload, dict):
            return None, "payload must be a JSON object"
        return payload, None

    def _normalize_fundraising_payload(self, payload):
        grid = payload.get('grid')
        views = payload.get('views')

        if not isinstance(grid, dict):
            return None, None, "payload.grid is required"
        if not isinstance(grid.get('columns'), list) or not isinstance(grid.get('rows'), list):
            return None, None, "payload.grid.columns and payload.grid.rows must be arrays"
        if views is None:
            views = deep_copy_json(DEFAULT_GRID_VIEWS)
        if not isinstance(views, list):
            return None, None, "payload.views must be an array"
        return sanitize_fundraising_grid(grid), sanitize_grid_views(views), None

    def handle_preview_fundraising_restore(self, user, body):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)

        payload, err = self._extract_fundraising_restore_payload(body)
        if err:
            return self.send_error_json(err)
        grid, views, err = self._normalize_fundraising_payload(payload)
        if err:
            return self.send_error_json(err)

        conn = get_db()
        self._ensure_fundraising_state_row(conn)
        current = conn.execute("SELECT * FROM fundraising_state WHERE id = 'main'").fetchone()
        conn.close()
        try:
            current_grid = json.loads(current['grid_json']) if current and current['grid_json'] else {}
        except Exception:
            current_grid = {}
        try:
            current_views = json.loads(current['views_json']) if current and current['views_json'] else []
        except Exception:
            current_views = []
        diff = compute_restore_diff(current_grid, current_views, grid, views)

        preview = {
            "columns_count": len(grid.get('columns', [])),
            "rows_count": len(grid.get('rows', [])),
            "views_count": len(views),
            "source": "filename" if body.get('filename') else "payload",
            "diff": diff
        }
        return self.send_json({"data": preview})

    def handle_backup_fundraising_state(self, user):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)

        conn = get_db()
        self._ensure_fundraising_state_row(conn)
        row = conn.execute("SELECT * FROM fundraising_state WHERE id = 'main'").fetchone()
        conn.close()
        if not row:
            return self.send_error_json("Fundraising state not found", 404)

        info = create_fundraising_backup_file(row, kind="backup")
        conn = get_db()
        policy = load_backup_policy(conn)
        apply_backup_retention(policy)
        conn.close()
        return self.send_json({"data": info}, 201)

    def handle_restore_fundraising_state(self, user, body):
        if not require_admin(user):
            return self.send_error_json("Admin access required", 403)

        payload, err = self._extract_fundraising_restore_payload(body)
        if err:
            return self.send_error_json(err)
        grid, views, err = self._normalize_fundraising_payload(payload)
        if err:
            return self.send_error_json(err)

        conn = get_db()
        self._ensure_fundraising_state_row(conn)
        current = conn.execute("SELECT * FROM fundraising_state WHERE id = 'main'").fetchone()
        if not current:
            conn.close()
            return self.send_error_json("Fundraising state not found", 404)

        # Always create a rollback snapshot before restore
        pre_backup = create_fundraising_backup_file(current, kind="pre_restore")

        next_version = int(current['version']) + 1
        conn.execute("""
            UPDATE fundraising_state
            SET grid_json = ?, views_json = ?, version = ?, updated_by = ?, updated_at = ?
            WHERE id = 'main'
        """, (json.dumps(grid), json.dumps(views), next_version, user['user_id'], now()))
        sync_fundraising_relational(conn, grid, views, actor_user_id=user['user_id'])
        log_audit(conn, user['user_id'], 'fundraising_state', 'main', 'restore', {"version": next_version, "pre_backup": pre_backup['filename']})
        conn.commit()
        conn.close()

        return self.send_json({
            "data": {
                "version": next_version,
                "pre_restore_backup": {
                    "filename": pre_backup['filename'],
                    "path": pre_backup['path']
                }
            }
        }, 201)


# ─── Seed Data ────────────────────────────────────────────────────────────────

def seed_demo_data():
    """Create demo data for testing."""
    conn = get_db()

    # Check if already seeded
    if conn.execute("SELECT COUNT(*) as c FROM users").fetchone()['c'] > 0:
        conn.close()
        return

    print("Seeding demo data...")

    # Create admin user (password: admin123)
    admin_id = generate_id()
    conn.execute("""
        INSERT INTO users (id, username, email, password_hash, full_name, role)
        VALUES (?, 'admin', 'admin@fund.com', ?, 'Fund Admin', 'admin')
    """, (admin_id, hash_password('admin123')))

    # Create a second user
    user2_id = generate_id()
    conn.execute("""
        INSERT INTO users (id, username, email, password_hash, full_name, role)
        VALUES (?, 'grant', 'grant@ten31.xyz', ?, 'Grant', 'admin')
    """, (user2_id, hash_password('password')))

    # Create organizations
    orgs = [
        (generate_id(), "Sovereign Wealth Holdings", "institutional", "Sovereign Wealth", "https://example.com"),
        (generate_id(), "Pacific Capital Partners", "family_office", "Family Office", "https://example.com"),
        (generate_id(), "Northeast Pension Fund", "pension", "Pension Fund", "https://example.com"),
        (generate_id(), "Redwood Endowment", "endowment", "Endowment", "https://example.com"),
        (generate_id(), "Atlas Family Office", "family_office", "Family Office", "https://example.com"),
        (generate_id(), "Summit Insurance Group", "institutional", "Insurance", "https://example.com"),
        (generate_id(), "Cascade Wealth Management", "wealth_management", "Wealth Management", "https://example.com"),
        (generate_id(), "Blue Harbor Foundation", "foundation", "Foundation", "https://example.com"),
    ]
    for org in orgs:
        conn.execute("""
            INSERT INTO organizations (id, name, type, industry, website, created_by)
            VALUES (?, ?, ?, ?, ?, ?)
        """, (*org, admin_id))

    # Create contacts (mix of investors and prospects)
    contacts = [
        # Investors
        (generate_id(), "James", "Chen", "jchen@sovereign.com", "Managing Director", orgs[0][0], "investor"),
        (generate_id(), "Sarah", "Williams", "swilliams@pacificcap.com", "CIO", orgs[1][0], "investor"),
        (generate_id(), "Michael", "Davis", "mdavis@nepension.org", "Investment Director", orgs[2][0], "investor"),
        (generate_id(), "Elizabeth", "Thompson", "ethompson@redwood.edu", "Endowment Manager", orgs[3][0], "investor"),
        (generate_id(), "Robert", "Kim", "rkim@atlas.family", "Principal", orgs[4][0], "investor"),
        (generate_id(), "Patricia", "Anderson", "panderson@summit.com", "VP Investments", orgs[5][0], "investor"),
        # Prospects
        (generate_id(), "David", "Martinez", "dmartinez@cascade.com", "Partner", orgs[6][0], "prospect"),
        (generate_id(), "Jennifer", "Taylor", "jtaylor@blueharbor.org", "Executive Director", orgs[7][0], "prospect"),
        (generate_id(), "William", "Johnson", "wjohnson@gmail.com", "Independent Investor", None, "prospect"),
        (generate_id(), "Maria", "Garcia", "mgarcia@example.com", "Family Office Principal", None, "prospect"),
        (generate_id(), "Thomas", "Brown", "tbrown@example.com", "Wealth Manager", None, "prospect"),
        (generate_id(), "Linda", "Wilson", "lwilson@example.com", "Portfolio Manager", None, "prospect"),
    ]
    for c in contacts:
        conn.execute("""
            INSERT INTO contacts (id, first_name, last_name, email, title, organization_id, contact_type, created_by)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?)
        """, (*c, admin_id))

    # Create LP profiles for investors
    lp_data = [
        (contacts[0][0], 25000000, 25000000, "2024-03-15", "Fund I", "institutional"),
        (contacts[1][0], 15000000, 15000000, "2024-04-01", "Fund I", "family_office"),
        (contacts[2][0], 20000000, 20000000, "2024-05-15", "Fund I", "pension"),
        (contacts[3][0], 10000000, 10000000, "2024-06-01", "Fund I", "endowment"),
        (contacts[4][0], 5000000, 5000000, "2024-07-15", "Fund I", "family_office"),
        (contacts[5][0], 8000000, 8000000, "2024-08-01", "Fund I", "institutional"),
    ]
    for lp in lp_data:
        conn.execute("""
            INSERT INTO lp_profiles (id, contact_id, commitment_amount, funded_amount,
                commitment_date, fund_name, investor_type, accredited, legal_docs_signed, wire_received)
            VALUES (?, ?, ?, ?, ?, ?, ?, 1, 1, 1)
        """, (generate_id(), *lp))

    # Create opportunities
    opp_data = [
        (contacts[6][0], orgs[6][0], "Cascade Wealth - Fund II", "meeting", 10000000, 10000000, 40, user2_id),
        (contacts[7][0], orgs[7][0], "Blue Harbor - Fund II", "due_diligence", 5000000, 5000000, 60, user2_id),
        (contacts[8][0], None, "William Johnson - Direct", "outreach", 0, 2000000, 20, admin_id),
        (contacts[9][0], None, "Garcia Family Office - Fund II", "lead", 0, 15000000, 10, admin_id),
        (contacts[10][0], None, "Thomas Brown - WM Referral", "meeting", 0, 3000000, 30, user2_id),
        (contacts[11][0], None, "Linda Wilson - PM Intro", "outreach", 0, 5000000, 15, admin_id),
    ]
    for opp in opp_data:
        conn.execute("""
            INSERT INTO opportunities (id, contact_id, organization_id, name, stage,
                commitment_amount, expected_amount, probability, owner_id, fund_name)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'Fund II')
        """, (generate_id(), *opp))

    # Create communications
    comm_data = [
        (contacts[0][0], "meeting", "Q4 Review Meeting", "Discussed fund performance and upcoming distributions.", "2025-12-15", 60, "positive", admin_id),
        (contacts[1][0], "email", "Capital Call Notice", "Sent Q1 capital call documentation.", "2026-01-10", None, "neutral", admin_id),
        (contacts[2][0], "call", "Annual Check-in", "Reviewed portfolio allocation and discussed Fund II.", "2026-01-20", 30, "positive", user2_id),
        (contacts[6][0], "meeting", "Initial Presentation", "Presented Fund II thesis and track record.", "2026-02-01", 90, "positive", user2_id),
        (contacts[7][0], "email", "Due Diligence Materials", "Sent DDQ, audited financials, and PPM.", "2026-02-05", None, "neutral", user2_id),
        (contacts[8][0], "call", "Intro Call", "Initial outreach, discussed investment interests.", "2026-02-08", 20, "positive", admin_id),
        (contacts[9][0], "email", "Introduction Email", "Sent overview deck and one-pager.", "2026-02-10", None, "neutral", admin_id),
        (contacts[6][0], "call", "Follow-up on Presentation", "Cascade team asked about risk management approach.", "2026-02-10", 25, "positive", user2_id),
    ]
    for cm in comm_data:
        conn.execute("""
            INSERT INTO communications (id, contact_id, type, subject, body,
                communication_date, duration_minutes, outcome, created_by)
            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
        """, (generate_id(), *cm))

    # Create some tags
    tag_data = [
        ("High Priority", "#ef4444"),
        ("Fund I LP", "#3b82f6"),
        ("Fund II Prospect", "#8b5cf6"),
        ("Family Office", "#f59e0b"),
        ("Institutional", "#10b981"),
        ("Re-up Target", "#ec4899"),
    ]
    for tag in tag_data:
        conn.execute("INSERT INTO tags (id, name, color) VALUES (?, ?, ?)",
                     (generate_id(), *tag))

    conn.commit()
    conn.close()
    print("Demo data seeded successfully.")


# ─── Main Entry Point ────────────────────────────────────────────────────────

def main():
    if ENV == "production" and not os.environ.get("CRM_SECRET_KEY"):
        print("ERROR: CRM_SECRET_KEY must be set in production mode (CRM_ENV=production).")
        sys.exit(1)
    init_db()
    if SEED_DEMO_DATA:
        seed_demo_data()
    else:
        print("Demo data seeding disabled (set CRM_SEED_DEMO_DATA=1 to enable).")
    start_backup_scheduler()

    # ─── Gmail sync scheduler (feature-flag-guarded) ─────────────────
    if os.environ.get("CRM_GMAIL_INTEGRATION_ENABLED", "").lower() in ("1", "true", "yes", "on"):
        try:
            from email_integration.scheduler import start_sync_scheduler
            start_sync_scheduler()
            print("[email_integration] Gmail sync scheduler started")
        except Exception as _e:
            print(f"[email_integration] failed to start scheduler: {_e}")

    # ThreadingHTTPServer lets one slow request (or a wave of scanner probes)
    # not block legit users. SQLite is opened per-request via get_db(), and
    # WAL mode allows concurrent readers + a single writer, so this is safe.
    server = ThreadingHTTPServer((HOST, PORT), CRMHandler)
    server.daemon_threads = True
    print(f"\n{'='*60}")
    print(f"  Venture Fund CRM Server")
    print(f"  Running at http://{HOST}:{PORT}")
    print(f"  Database: {DB_PATH}")
    print(f"{'='*60}")
    if SEED_DEMO_DATA:
        print(f"\n  Demo login: admin / admin123")
        print(f"  Or: grant / password")
    print(f"\n  Press Ctrl+C to stop\n")

    try:
        server.serve_forever()
    except KeyboardInterrupt:
        print("\nShutting down server...")
        server.shutdown()

if __name__ == "__main__":
    main()