grant/ten31-database
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

email: live backfill progress on Email Capture panel — v0.1.0:61

Browse Source 
The first Gmail backfill leaves the account at "pending · never synced" until it
fully completes (the sync_runs row only finalizes at the end), so there was no
feedback. /api/email/status now also returns captured_emails (total, which climbs
page-by-page during backfill), the latest sync run, and a backfilling flag. The
panel shows a "Backfilling… N captured so far" banner + an Emails Captured count
and auto-refreshes every 5s while a backfill is in progress. Verified live in
preview with seeded data (count auto-climbed 37 -> 50 without manual refresh).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Keysat
2026-06-06 12:29:01 -05:00
parent 1850bc4431
commit 2cb476e36b
5 changed files with 60 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/email_integration/routes.py
+12
View File
@@ -131,6 +131,18 @@ def _h_status(handler):
counts = dict(cur.fetchone() or {})
cur.execute("SELECT COUNT(*) AS n FROM emails WHERE match_status = 'matched'")
snap["matched_emails"] = cur.fetchone()["n"]
# Total captured climbs page-by-page during backfill (the sync_runs row only
# finalizes at the end), so it is the live progress signal.
cur.execute("SELECT COUNT(*) AS n FROM emails")
snap["captured_emails"] = cur.fetchone()["n"]
# Latest sync run (status running|ok|error|partial) for a human-readable line.
cur.execute("SELECT kind, status, started_at, finished_at, messages_seen, messages_stored "
"FROM email_sync_runs ORDER BY started_at DESC LIMIT 1")
lr = cur.fetchone()
snap["last_run"] = dict(lr) if lr else None
# An enrolled account whose backfill has not completed is still pulling history.
cur.execute("SELECT COUNT(*) AS n FROM email_accounts WHERE sync_enabled = 1 AND backfill_complete = 0")
snap["backfilling"] = (cur.fetchone()["n"] or 0) > 0
finally:
conn.close()
snap["accounts_summary"] = counts
frontend/index.html
+24 -8
View File
@@ -9964,7 +9964,7 @@
} catch (err) {
// Integration off on the server -> render the informative disabled state, not an error.
if (err?.status === 503 || /disabl/i.test(err?.payload?.error || '')) {
setStatus({ enabled: false, accounts_summary: {}, matched_emails: 0 });
setStatus({ enabled: false, accounts_summary: {}, matched_emails: 0, captured_emails: 0, backfilling: false, last_run: null });
setAccounts([]);
return;
}
@@ -9992,6 +9992,16 @@
return () => { cancelled = true; };
}, [load]);
// While a backfill is in progress the captured-email count climbs page by page,
// so poll the status to show live progress without a manual refresh.
const backfilling = !!status && (status.backfilling || status.last_run?.status === 'running' || status.running);
useEffect(() => {
if (!backfilling) return undefined;
let cancelled = false;
const iv = setInterval(async () => { try { if (!cancelled) await load(); } catch (_) { /* transient */ } }, 5000);
return () => { cancelled = true; clearInterval(iv); };
}, [backfilling, load]);
const runAction = async (key, endpoint, successMsg, confirmMsg, body) => {
if (busy) return;
if (confirmMsg && !window.confirm(confirmMsg)) return;
@@ -10018,8 +10028,11 @@
const nAccounts = sum.n_accounts || 0;
const nError = sum.n_error || 0;
const capturing = enabled && nAccounts > 0;
const lastSync = status.last_run_unix ? formatDate(new Date(status.last_run_unix * 1000).toISOString()) : '—';
const statusWord = capturing ? 'Capturing' : enabled ? 'Not enrolled' : 'Disabled';
const captured = status.captured_emails ?? 0;
const lastRun = status.last_run;
const lastSync = lastRun ? formatDate(lastRun.finished_at || lastRun.started_at)
: (status.last_run_unix ? formatDate(new Date(status.last_run_unix * 1000).toISOString()) : '—');
const statusWord = backfilling ? 'Backfilling' : capturing ? 'Capturing' : enabled ? 'Not enrolled' : 'Disabled';
return (
<div className="page-container">
@@ -10031,10 +10044,12 @@
<span className="index-job-pill idle"><span className="index-job-dot" /> Integration disabled — no Gmail service-account key on the server.</span>
) : nAccounts === 0 ? (
<span className="index-job-pill idle"><span className="index-job-dot" /> Enabled, but no mailbox is enrolled yet, so nothing is being captured.</span>
) : backfilling ? (
<span className="index-job-pill running"><span className="index-job-dot" /> Backfilling history… {captured} emails captured so far ({status.matched_emails || 0} matched to investors). Updates live; first backfill can take a while.</span>
) : nError ? (
<span className="index-job-pill running"><span className="index-job-dot" /> {nAccounts} mailbox(es), {nError} in error — see below.</span>
) : (
<span className="index-job-pill"><span className="index-job-dot" /> Capturing: {nAccounts} mailbox(es), {status.matched_emails || 0} emails matched to investors.</span>
<span className="index-job-pill"><span className="index-job-dot" /> Capturing: {nAccounts} mailbox(es), {captured} emails captured, {status.matched_emails || 0} matched to investors.</span>
)}
</div>
</div>
@@ -10050,13 +10065,14 @@
{nError ? <div className="kpi-subtitle">{nError} error</div> : null}
</div>
<div className="kpi-card">
<div className="kpi-label">Emails matched to investors</div>
<div className="kpi-value">{status.matched_emails ?? 0}</div>
<div className="kpi-label">Emails captured</div>
<div className="kpi-value">{captured}{backfilling ? '…' : ''}</div>
<div className="kpi-subtitle">{status.matched_emails ?? 0} matched to investors</div>
</div>
<div className="kpi-card">
<div className="kpi-label">Last sync</div>
<div className="kpi-value" style={{ fontSize: '16px' }}>{lastSync}</div>
<div className="kpi-subtitle">{status.interval_sec ? `every ${Math.round(status.interval_sec / 60)} min` : ''}</div>
<div className="kpi-value" style={{ fontSize: '16px' }}>{backfilling ? 'in progress' : lastSync}</div>
<div className="kpi-subtitle">{lastRun?.status ? `last run: ${lastRun.status}` : (status.interval_sec ? `every ${Math.round(status.interval_sec / 60)} min` : '')}</div>
</div>
</div>
start9/0.4/startos/utils.ts
+3 -2
View File
@@ -25,8 +25,9 @@ export const PACKAGE_TITLE = 'Ten31 Database'
// * 0.1.0:57 (redaction fix: magnitude regex no longer eats the word after an amount)
// * 0.1.0:58 (seed 5 Architect positioning framings into the Workshop as candidate options)
// * 0.1.0:59 (Email Capture admin panel + matched email into the grounding corpus)
// * Current: 0.1.0:60 (Email Capture: single-mailbox enroll field for testing)
export const PACKAGE_VERSION = '0.1.0:60'
// * 0.1.0:60 (Email Capture: single-mailbox enroll field for testing)
// * Current: 0.1.0:61 (Email Capture: live backfill progress + auto-refresh)
export const PACKAGE_VERSION = '0.1.0:61'
export const DATA_MOUNT_PATH = '/data'
export const WEB_PORT = 8080
start9/0.4/startos/versions/index.ts
+3 -2
View File
@@ -21,8 +21,9 @@ import { v_0_1_0_57 } from './v0.1.0.57'
import { v_0_1_0_58 } from './v0.1.0.58'
import { v_0_1_0_59 } from './v0.1.0.59'
import { v_0_1_0_60 } from './v0.1.0.60'
import { v_0_1_0_61 } from './v0.1.0.61'
export const versionGraph = VersionGraph.of({
current: v_0_1_0_60,
other: [v_0_1_0_39, v_0_1_0_40, v_0_1_0_41, v_0_1_0_42, v_0_1_0_43, v_0_1_0_44, v_0_1_0_45, v_0_1_0_46, v_0_1_0_47, v_0_1_0_48, v_0_1_0_49, v_0_1_0_50, v_0_1_0_51, v_0_1_0_52, v_0_1_0_53, v_0_1_0_54, v_0_1_0_55, v_0_1_0_56, v_0_1_0_57, v_0_1_0_58, v_0_1_0_59],
current: v_0_1_0_61,
other: [v_0_1_0_39, v_0_1_0_40, v_0_1_0_41, v_0_1_0_42, v_0_1_0_43, v_0_1_0_44, v_0_1_0_45, v_0_1_0_46, v_0_1_0_47, v_0_1_0_48, v_0_1_0_49, v_0_1_0_50, v_0_1_0_51, v_0_1_0_52, v_0_1_0_53, v_0_1_0_54, v_0_1_0_55, v_0_1_0_56, v_0_1_0_57, v_0_1_0_58, v_0_1_0_59, v_0_1_0_60],
})
start9/0.4/startos/versions/v0.1.0.61.ts
+18
View File
@@ -0,0 +1,18 @@
import { VersionInfo } from '@start9labs/start-sdk'
// Email Capture: show live backfill progress. The status endpoint now reports total
// emails captured (climbs page-by-page during backfill, since the sync-run row only
// finalizes at the end), the latest sync run, and whether a backfill is still in
// progress. The panel shows a "Backfilling… N captured so far" banner + an Emails
// Captured count, and auto-refreshes every 5s while a backfill runs. No schema migration.
export const v_0_1_0_61 = VersionInfo.of({
version: '0.1.0:61',
releaseNotes: {
en_US: [
'Email Capture now shows live backfill progress: a count of emails captured so far',
'that ticks up while the first sync pulls your history, plus a clear backfilling banner',
'that refreshes on its own. No more staring at a static "pending" with no feedback.',
].join(' '),
},
migrations: { up: async () => {}, down: async () => {} },
})