v0.6.0 - Service-level connectivity tracking + passive failure-report endpoint
connectivity.py:
- Generalized 'spark' subject to any string; renamed 'spark' field to 'subject'
- Legacy v0.5 events with the old 'spark' field are migrated transparently on read (kind defaults to 'transition')
- New record_report(subject, ok, source, detail, latency_ms): always appends an event with kind='report'; does NOT mutate the current state (only active polling is authoritative)
- summary() returns events normalized to the new schema
Wiring:
- /api/status now calls record_state for vllm/parakeet/magpie (dedup on no-change)
- /api/services calls record_state for each service after its http check
- Result: dashboard observes service-level transitions automatically with no extra polling
Passive endpoint:
- POST /api/health-event with {service, ok, source?, error?, ms?}
- Useful for external apps (e.g. Open WebUI) to surface sub-poll-interval failures the dashboard would otherwise miss
UI:
- Connectivity dialog groups events by subject (hosts ordered first, then services)
- Per-subject summary shows transition count, down count, report count, failed-report count
- Transitions and reports render inline with distinct styling; reports show source app + error + latency
- Legacy v0.5 events render unchanged
Docs:
- README documents /api/health-event with a curl example
Package: bump to 0.6.0:0
This commit is contained in:
Grant
+56
-15
@@ -146,28 +146,42 @@ function openConnectivityDialog() {
|
||||
const c = state.connectivity || {};
|
||||
const events = c.events || [];
|
||||
if (events.length === 0) {
|
||||
content.innerHTML = '<div class="muted small">No transitions recorded yet. Once a Spark goes down and comes back, you\'ll see entries here.</div>';
|
||||
content.innerHTML = '<div class="muted small">No events recorded yet. Once a Spark or service goes down and back up (or an external app reports a failure), entries appear here.</div>';
|
||||
dlg.showModal();
|
||||
return;
|
||||
}
|
||||
const bySpark = {};
|
||||
const bySubject = {};
|
||||
for (const e of events) {
|
||||
(bySpark[e.spark] = bySpark[e.spark] || []).push(e);
|
||||
const subj = e.subject || e.spark || 'unknown'; // legacy fallback
|
||||
(bySubject[subj] = bySubject[subj] || []).push(e);
|
||||
}
|
||||
const html = Object.entries(bySpark).map(([spark, evs]) => {
|
||||
const downs = evs.filter(e => e.transition === 'down').length;
|
||||
const mac = c.macs?.[spark];
|
||||
// Sort subjects: hosts first, then services, alphabetical
|
||||
const hostOrder = ['spark1', 'spark2'];
|
||||
const subjects = Object.keys(bySubject).sort((a, b) => {
|
||||
const ia = hostOrder.indexOf(a);
|
||||
const ib = hostOrder.indexOf(b);
|
||||
if (ia >= 0 && ib >= 0) return ia - ib;
|
||||
if (ia >= 0) return -1;
|
||||
if (ib >= 0) return 1;
|
||||
return a.localeCompare(b);
|
||||
});
|
||||
|
||||
const html = subjects.map((subj) => {
|
||||
const evs = bySubject[subj];
|
||||
const transitions = evs.filter(e => (e.kind || 'transition') === 'transition');
|
||||
const reports = evs.filter(e => e.kind === 'report');
|
||||
const downs = transitions.filter(e => e.transition === 'down').length;
|
||||
const failedReports = reports.filter(e => !e.ok).length;
|
||||
const mac = c.macs?.[subj];
|
||||
const summaryParts = [];
|
||||
if (transitions.length) summaryParts.push(`${transitions.length} probe transition${transitions.length===1?'':'s'} (${downs} down)`);
|
||||
if (reports.length) summaryParts.push(`${reports.length} app report${reports.length===1?'':'s'} (${failedReports} failed)`);
|
||||
const isHost = hostOrder.includes(subj);
|
||||
return `
|
||||
<div class="conn-spark">
|
||||
<h4>${escapeHtml(spark)}${mac ? ` <span class="muted small">${escapeHtml(mac)}</span>` : ''}</h4>
|
||||
<div class="conn-summary">${evs.length} transition${evs.length===1?'':'s'} · ${downs} down event${downs===1?'':'s'} in window</div>
|
||||
${evs.slice(-25).reverse().map(e => `
|
||||
<div class="conn-event ${e.transition}">
|
||||
<span class="when">${escapeHtml(e.at.replace('T', ' ').replace('Z', ''))}</span>
|
||||
<span class="what">${e.transition === 'up' ? '↑ came back online' : '↓ dropped offline'}</span>
|
||||
<span class="dur">${e.down_seconds != null ? `was down ${fmtDuration(e.down_seconds)}` : ''}${e.up_seconds != null ? `was up ${fmtDuration(e.up_seconds)}` : ''}</span>
|
||||
</div>
|
||||
`).join('')}
|
||||
<h4>${escapeHtml(subj)}${isHost ? ' <span class="muted small">[host]</span>' : ' <span class="muted small">[service]</span>'}${mac ? ` <span class="muted small">${escapeHtml(mac)}</span>` : ''}</h4>
|
||||
<div class="conn-summary">${summaryParts.join(' · ') || 'no events'}</div>
|
||||
${evs.slice(-30).reverse().map(e => renderConnEvent(e)).join('')}
|
||||
</div>
|
||||
`;
|
||||
}).join('');
|
||||
@@ -175,6 +189,33 @@ function openConnectivityDialog() {
|
||||
dlg.showModal();
|
||||
}
|
||||
|
||||
function renderConnEvent(e) {
|
||||
const when = escapeHtml((e.at || '').replace('T', ' ').replace('Z', ''));
|
||||
const kind = e.kind || 'transition';
|
||||
if (kind === 'report') {
|
||||
const ok = !!e.ok;
|
||||
const source = escapeHtml(e.source || 'external');
|
||||
const detail = e.detail ? ` — ${escapeHtml(e.detail)}` : '';
|
||||
const latency = e.latency_ms != null ? ` (${e.latency_ms} ms)` : '';
|
||||
return `
|
||||
<div class="conn-event ${ok ? 'up' : 'down'} report">
|
||||
<span class="when">${when}</span>
|
||||
<span class="what">${ok ? '◷ report: ok' : '◷ report: failed'} <span class="muted">from</span> ${source}${detail}</span>
|
||||
<span class="dur">${latency}</span>
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
const down = e.down_seconds != null ? `was down ${fmtDuration(e.down_seconds)}` : '';
|
||||
const up = e.up_seconds != null ? `was up ${fmtDuration(e.up_seconds)}` : '';
|
||||
return `
|
||||
<div class="conn-event ${e.transition}">
|
||||
<span class="when">${when}</span>
|
||||
<span class="what">${e.transition === 'up' ? '↑ came back online' : '↓ dropped offline'}</span>
|
||||
<span class="dur">${down}${up}</span>
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
|
||||
async function wakeSpark(name) {
|
||||
try {
|
||||
const r = await fetchJSON(`/api/spark/${name}/wake`, { method: 'POST' });
|
||||
|
||||
@@ -411,6 +411,8 @@ main {
|
||||
.conn-event .what { flex: 1; }
|
||||
.conn-event.up .what { color: var(--accent); }
|
||||
.conn-event.down .what { color: var(--error); }
|
||||
.conn-event.report .what { font-style: italic; }
|
||||
.conn-event .muted { color: var(--muted); font-style: normal; }
|
||||
.conn-event .dur { color: var(--muted); }
|
||||
.conn-summary { color: var(--muted); font-size: 11px; padding: 4px 0 10px; }
|
||||
.hw-metric { display: flex; align-items: center; gap: 10px; font-size: 12px; }
|
||||
|
||||
Reference in New Issue
Block a user