email: live backfill progress on Email Capture panel — v0.1.0:61

The first Gmail backfill leaves the account at "pending · never synced" until it
fully completes (the sync_runs row only finalizes at the end), so there was no
feedback. /api/email/status now also returns captured_emails (total, which climbs
page-by-page during backfill), the latest sync run, and a backfilling flag. The
panel shows a "Backfilling… N captured so far" banner + an Emails Captured count
and auto-refreshes every 5s while a backfill is in progress. Verified live in
preview with seeded data (count auto-climbed 37 -> 50 without manual refresh).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

backend/email_integration/routes.py

@@ -131,6 +131,18 @@ def _h_status(handler):
         counts = dict(cur.fetchone() or {})
         cur.execute("SELECT COUNT(*) AS n FROM emails WHERE match_status = 'matched'")
         snap["matched_emails"] = cur.fetchone()["n"]
+        # Total captured climbs page-by-page during backfill (the sync_runs row only
+        # finalizes at the end), so it is the live progress signal.
+        cur.execute("SELECT COUNT(*) AS n FROM emails")
+        snap["captured_emails"] = cur.fetchone()["n"]
+        # Latest sync run (status running|ok|error|partial) for a human-readable line.
+        cur.execute("SELECT kind, status, started_at, finished_at, messages_seen, messages_stored "
+                    "FROM email_sync_runs ORDER BY started_at DESC LIMIT 1")
+        lr = cur.fetchone()
+        snap["last_run"] = dict(lr) if lr else None
+        # An enrolled account whose backfill has not completed is still pulling history.
+        cur.execute("SELECT COUNT(*) AS n FROM email_accounts WHERE sync_enabled = 1 AND backfill_complete = 0")
+        snap["backfilling"] = (cur.fetchone()["n"] or 0) > 0
     finally:
         conn.close()
     snap["accounts_summary"] = counts

frontend/index.html

@@ -9964,7 +9964,7 @@
                 } catch (err) {
                     // Integration off on the server -> render the informative disabled state, not an error.
                     if (err?.status === 503 || /disabl/i.test(err?.payload?.error || '')) {
-                        setStatus({ enabled: false, accounts_summary: {}, matched_emails: 0 });
+                        setStatus({ enabled: false, accounts_summary: {}, matched_emails: 0, captured_emails: 0, backfilling: false, last_run: null });
                         setAccounts([]);
                         return;
                     }
@@ -9992,6 +9992,16 @@
                 return () => { cancelled = true; };
             }, [load]);
 
+            // While a backfill is in progress the captured-email count climbs page by page,
+            // so poll the status to show live progress without a manual refresh.
+            const backfilling = !!status && (status.backfilling || status.last_run?.status === 'running' || status.running);
+            useEffect(() => {
+                if (!backfilling) return undefined;
+                let cancelled = false;
+                const iv = setInterval(async () => { try { if (!cancelled) await load(); } catch (_) { /* transient */ } }, 5000);
+                return () => { cancelled = true; clearInterval(iv); };
+            }, [backfilling, load]);
+
             const runAction = async (key, endpoint, successMsg, confirmMsg, body) => {
                 if (busy) return;
                 if (confirmMsg && !window.confirm(confirmMsg)) return;
@@ -10018,8 +10028,11 @@
             const nAccounts = sum.n_accounts || 0;
             const nError = sum.n_error || 0;
             const capturing = enabled && nAccounts > 0;
-            const lastSync = status.last_run_unix ? formatDate(new Date(status.last_run_unix * 1000).toISOString()) : '—';
+            const captured = status.captured_emails ?? 0;
-            const statusWord = capturing ? 'Capturing' : enabled ? 'Not enrolled' : 'Disabled';
+            const lastRun = status.last_run;
+            const lastSync = lastRun ? formatDate(lastRun.finished_at || lastRun.started_at)
+                : (status.last_run_unix ? formatDate(new Date(status.last_run_unix * 1000).toISOString()) : '—');
+            const statusWord = backfilling ? 'Backfilling' : capturing ? 'Capturing' : enabled ? 'Not enrolled' : 'Disabled';
 
             return (
                 <div className="page-container">
@@ -10031,10 +10044,12 @@
                                 <span className="index-job-pill idle"><span className="index-job-dot" /> Integration disabled — no Gmail service-account key on the server.</span>
                             ) : nAccounts === 0 ? (
                                 <span className="index-job-pill idle"><span className="index-job-dot" /> Enabled, but no mailbox is enrolled yet, so nothing is being captured.</span>
+                            ) : backfilling ? (
+                                <span className="index-job-pill running"><span className="index-job-dot" /> Backfilling history… {captured} emails captured so far ({status.matched_emails || 0} matched to investors). Updates live; first backfill can take a while.</span>
                             ) : nError ? (
                                 <span className="index-job-pill running"><span className="index-job-dot" /> {nAccounts} mailbox(es), {nError} in error — see below.</span>
                             ) : (
-                                <span className="index-job-pill"><span className="index-job-dot" /> Capturing: {nAccounts} mailbox(es), {status.matched_emails || 0} emails matched to investors.</span>
+                                <span className="index-job-pill"><span className="index-job-dot" /> Capturing: {nAccounts} mailbox(es), {captured} emails captured, {status.matched_emails || 0} matched to investors.</span>
                             )}
                         </div>
                     </div>
@@ -10050,13 +10065,14 @@
                             {nError ? <div className="kpi-subtitle">{nError} error</div> : null}
                         </div>
                         <div className="kpi-card">
-                            <div className="kpi-label">Emails matched to investors</div>
+                            <div className="kpi-label">Emails captured</div>
-                            <div className="kpi-value">{status.matched_emails ?? 0}</div>
+                            <div className="kpi-value">{captured}{backfilling ? '…' : ''}</div>
+                            <div className="kpi-subtitle">{status.matched_emails ?? 0} matched to investors</div>
                         </div>
                         <div className="kpi-card">
                             <div className="kpi-label">Last sync</div>
-                            <div className="kpi-value" style={{ fontSize: '16px' }}>{lastSync}</div>
+                            <div className="kpi-value" style={{ fontSize: '16px' }}>{backfilling ? 'in progress' : lastSync}</div>
-                            <div className="kpi-subtitle">{status.interval_sec ? `every ${Math.round(status.interval_sec / 60)} min` : ''}</div>
+                            <div className="kpi-subtitle">{lastRun?.status ? `last run: ${lastRun.status}` : (status.interval_sec ? `every ${Math.round(status.interval_sec / 60)} min` : '')}</div>
                         </div>
                     </div>
 

start9/0.4/startos/utils.ts

@@ -25,8 +25,9 @@ export const PACKAGE_TITLE = 'Ten31 Database'
 //   * 0.1.0:57 (redaction fix: magnitude regex no longer eats the word after an amount)
 //   * 0.1.0:58 (seed 5 Architect positioning framings into the Workshop as candidate options)
 //   * 0.1.0:59 (Email Capture admin panel + matched email into the grounding corpus)
-//   * Current:        0.1.0:60 (Email Capture: single-mailbox enroll field for testing)
+//   * 0.1.0:60 (Email Capture: single-mailbox enroll field for testing)
-export const PACKAGE_VERSION = '0.1.0:60'
+//   * Current:        0.1.0:61 (Email Capture: live backfill progress + auto-refresh)
+export const PACKAGE_VERSION = '0.1.0:61'
 
 export const DATA_MOUNT_PATH = '/data'
 export const WEB_PORT = 8080

start9/0.4/startos/versions/index.ts

@@ -21,8 +21,9 @@ import { v_0_1_0_57 } from './v0.1.0.57'
 import { v_0_1_0_58 } from './v0.1.0.58'
 import { v_0_1_0_59 } from './v0.1.0.59'
 import { v_0_1_0_60 } from './v0.1.0.60'
+import { v_0_1_0_61 } from './v0.1.0.61'
 
 export const versionGraph = VersionGraph.of({
-  current: v_0_1_0_60,
+  current: v_0_1_0_61,
-  other: [v_0_1_0_39, v_0_1_0_40, v_0_1_0_41, v_0_1_0_42, v_0_1_0_43, v_0_1_0_44, v_0_1_0_45, v_0_1_0_46, v_0_1_0_47, v_0_1_0_48, v_0_1_0_49, v_0_1_0_50, v_0_1_0_51, v_0_1_0_52, v_0_1_0_53, v_0_1_0_54, v_0_1_0_55, v_0_1_0_56, v_0_1_0_57, v_0_1_0_58, v_0_1_0_59],
+  other: [v_0_1_0_39, v_0_1_0_40, v_0_1_0_41, v_0_1_0_42, v_0_1_0_43, v_0_1_0_44, v_0_1_0_45, v_0_1_0_46, v_0_1_0_47, v_0_1_0_48, v_0_1_0_49, v_0_1_0_50, v_0_1_0_51, v_0_1_0_52, v_0_1_0_53, v_0_1_0_54, v_0_1_0_55, v_0_1_0_56, v_0_1_0_57, v_0_1_0_58, v_0_1_0_59, v_0_1_0_60],
 })

start9/0.4/startos/versions/v0.1.0.61.ts

@@ -0,0 +1,18 @@
+import { VersionInfo } from '@start9labs/start-sdk'
+
+// Email Capture: show live backfill progress. The status endpoint now reports total
+// emails captured (climbs page-by-page during backfill, since the sync-run row only
+// finalizes at the end), the latest sync run, and whether a backfill is still in
+// progress. The panel shows a "Backfilling… N captured so far" banner + an Emails
+// Captured count, and auto-refreshes every 5s while a backfill runs. No schema migration.
+export const v_0_1_0_61 = VersionInfo.of({
+  version: '0.1.0:61',
+  releaseNotes: {
+    en_US: [
+      'Email Capture now shows live backfill progress: a count of emails captured so far',
+      'that ticks up while the first sync pulls your history, plus a clear backfilling banner',
+      'that refreshes on its own. No more staring at a static "pending" with no feedback.',
+    ].join(' '),
+  },
+  migrations: { up: async () => {}, down: async () => {} },
+})