Keysat
df9f244eae
The dashboard menu is now the set of models actually downloaded on the Sparks, not a hard-coded catalog. models.yaml + overrides are reframed as launch recipes matched to an on-disk model by repo; an on-disk model with no recipe is flagged needs_setup and its launch settings are inferred from its config.json for a one-time operator confirmation (discovery.py). - delete now removes weights AND the menu card (delete_from_disk sweeps all hosts; the delete endpoint resolves keys via the live menu) - new GET /api/models/suggest; /api/models returns the menu + a recipes list (download autocomplete); GET /api/models/disk-status removed - dropped the two legacy Qwen recipes (235B FP8, 2.5 72B) - tests: +test_discovery.py (cache parsing, infer_recipe, build_menu merge)
2271 lines
90 KiB
JavaScript
2271 lines
90 KiB
JavaScript
// spark-control front-end
|
||
|
||
const state = {
|
||
models: {},
|
||
defaults: {},
|
||
current_model_key: null,
|
||
swap_job_id: null,
|
||
swap_eventsource: null,
|
||
swap_started_at: null,
|
||
swap_lines: [], // local accumulator for phase detection
|
||
swap_phase: 'Starting…',
|
||
swap_phase_detail: '',
|
||
swap_progress: 0, // 0–1
|
||
services: {},
|
||
service_action_in_flight: null, // e.g. "parakeet:restart"
|
||
mb_update_in_flight: false, // matrix-bridge update job running
|
||
hardware: {},
|
||
config: {},
|
||
configured: true,
|
||
timer_handle: null,
|
||
deep_health: {},
|
||
models_loaded: false, // true once the first disk scan (/api/models) returns
|
||
recipes: [], // known launch recipes (for the download autocomplete)
|
||
lock: { held: false }, // GPU swap reservation (coordination layer)
|
||
schedules: [], // schedules external automation has registered
|
||
};
|
||
|
||
const el = (sel) => document.querySelector(sel);
|
||
const $$ = (sel) => document.querySelectorAll(sel);
|
||
|
||
// ISO timestamp -> local clock string (e.g. "2:45:10 PM"); '' if unparseable.
|
||
function fmtClock(iso) {
|
||
const t = Date.parse(iso);
|
||
return isNaN(t) ? '' : new Date(t).toLocaleTimeString();
|
||
}
|
||
|
||
function escapeHtml(s) {
|
||
if (s == null) return '';
|
||
return String(s)
|
||
.replaceAll('&', '&')
|
||
.replaceAll('<', '<')
|
||
.replaceAll('>', '>')
|
||
.replaceAll('"', '"')
|
||
.replaceAll("'", ''');
|
||
}
|
||
|
||
async function fetchJSON(url, opts) {
|
||
const r = await fetch(url, opts);
|
||
if (!r.ok) {
|
||
const text = await r.text().catch(() => '');
|
||
throw new Error(`${r.status} ${r.statusText}: ${text}`);
|
||
}
|
||
return r.json();
|
||
}
|
||
|
||
// ===================== rendering =====================
|
||
|
||
function renderCards() {
|
||
const root = el('#cards');
|
||
root.innerHTML = '';
|
||
const isSwapping = !!state.swap_job_id;
|
||
// GPU reserved by external automation — manual swaps are refused server-side
|
||
// (423); reflect that in the buttons so the click never bounces.
|
||
const locked = !!(state.lock && state.lock.held);
|
||
const lockTip = locked
|
||
? `Reserved by ${state.lock.holder || 'automation'}${state.lock.expires_at ? ' until ' + fmtClock(state.lock.expires_at) : ''}`
|
||
: '';
|
||
const keys = Object.keys(state.models);
|
||
if (keys.length === 0) {
|
||
// The menu is the disk: nothing downloaded (or the scan hasn't returned yet).
|
||
root.innerHTML = state.models_loaded
|
||
? `<div class="empty-menu muted">No models downloaded on the Sparks yet. Use <strong>+ Download a new model</strong> above to fetch one — it'll appear here when it's done.</div>`
|
||
: `<div class="empty-menu muted">Scanning the Sparks for downloaded models…</div>`;
|
||
return;
|
||
}
|
||
for (const key of keys) {
|
||
const m = state.models[key];
|
||
const isActive = key === state.current_model_key;
|
||
const card = document.createElement('div');
|
||
card.className = 'card' + (isActive ? ' active' : '') + (m.needs_setup ? ' needs-setup' : '');
|
||
const desc = m.description
|
||
? `<div class="desc">${escapeHtml(m.description)}</div>`
|
||
: '';
|
||
const customPill = m.custom ? `<span class="tag custom-pill">custom</span>` : '';
|
||
const localPill = m.local_path ? `<span class="tag local-pill" title="Served from a directory on the Spark, not Hugging Face">local</span>` : '';
|
||
// Every card on the menu is on disk by definition — show its real size.
|
||
const gb = (m.total_bytes || 0) / 1e9;
|
||
const diskPill = gb > 0
|
||
? `<span class="tag on-disk" title="Weights present on the Spark(s)">on disk · ${gb.toFixed(1)} GB</span>`
|
||
: '';
|
||
const setupPill = m.needs_setup
|
||
? `<span class="tag setup-pill" title="On disk, but Spark Control hasn't been told how to launch it">needs setup</span>`
|
||
: '';
|
||
// Trash = remove weights from disk AND from the menu. Disabled if active / mid-swap.
|
||
// Never offered for local models: their directory is hand-placed training output,
|
||
// not a re-downloadable HF cache (the server refuses the delete too).
|
||
let trashBtn = '';
|
||
if (!m.local_path) {
|
||
const disabled = isActive || isSwapping;
|
||
const tip = isActive
|
||
? 'Currently loaded — switch to another model first'
|
||
: isSwapping
|
||
? 'A swap is in progress'
|
||
: 'Remove weights from disk & menu';
|
||
trashBtn = `<button class="icon-btn danger" data-disk-del-key="${key}" title="${escapeHtml(tip)}" aria-label="Remove from disk and menu" ${disabled ? 'disabled' : ''}>${trashIcon}</button>`;
|
||
}
|
||
// Primary action: "Current" / "Switch to this", or "Set up & switch" for a
|
||
// model on disk that has no launch recipe yet.
|
||
const swapBlocked = isSwapping || locked;
|
||
const lockTipAttr = locked ? ` title="${escapeHtml(lockTip)}"` : '';
|
||
let primaryBtn = '';
|
||
if (isActive) {
|
||
primaryBtn = `<button class="btn" disabled>Current</button>`;
|
||
} else if (m.needs_setup) {
|
||
primaryBtn = `<button class="btn primary" data-setup-key="${key}"${lockTipAttr} ${swapBlocked ? 'disabled' : ''}>Set up & switch</button>`;
|
||
} else {
|
||
primaryBtn = `<button class="btn primary" data-swap-key="${key}"${lockTipAttr} ${swapBlocked ? 'disabled' : ''}>Switch to this</button>`;
|
||
}
|
||
// The Test/Advanced controls need a saved recipe; hide them until setup is done.
|
||
const recipeActions = m.needs_setup ? '' : `
|
||
<button class="btn test-btn" data-test-key="${key}" title="Pre-flight check the launch command without starting the engine">Test</button>
|
||
<button class="btn adv-btn" data-adv-key="${key}" title="Advanced settings">Advanced</button>`;
|
||
card.innerHTML = `
|
||
<div class="name">${escapeHtml(m.display_name)}</div>
|
||
<div class="meta">
|
||
<span class="tag mode-${m.mode}">${m.mode}</span>
|
||
${diskPill}
|
||
${setupPill}
|
||
${customPill}
|
||
${localPill}
|
||
${(m.capabilities || []).map(c => `<span class="tag cap">${escapeHtml(c)}</span>`).join('')}
|
||
</div>
|
||
${desc}
|
||
<div class="muted small repo">
|
||
${m.local_path
|
||
? `<span class="local-path" title="Local model directory on the Spark">${escapeHtml(m.local_path)}</span>`
|
||
: `<a href="https://huggingface.co/${encodeURIComponent(m.repo)}" target="_blank" rel="noopener" title="View on Hugging Face">${escapeHtml(m.repo)} <span class="hf-icon">↗</span></a>`}
|
||
</div>
|
||
<div class="spacer"></div>
|
||
<div class="card-actions">
|
||
${primaryBtn}${recipeActions}
|
||
${trashBtn}
|
||
</div>
|
||
<div class="test-result hidden" data-test-result-for="${key}"></div>
|
||
`;
|
||
root.appendChild(card);
|
||
}
|
||
for (const btn of root.querySelectorAll('[data-swap-key]')) {
|
||
btn.addEventListener('click', () => triggerSwap(btn.dataset.swapKey));
|
||
}
|
||
for (const btn of root.querySelectorAll('[data-setup-key]')) {
|
||
btn.addEventListener('click', () => openSetupForKey(btn.dataset.setupKey));
|
||
}
|
||
for (const btn of root.querySelectorAll('[data-adv-key]')) {
|
||
btn.addEventListener('click', () => openAdvanced(btn.dataset.advKey));
|
||
}
|
||
for (const btn of root.querySelectorAll('[data-test-key]')) {
|
||
btn.addEventListener('click', () => testLaunch(btn.dataset.testKey, btn));
|
||
}
|
||
for (const btn of root.querySelectorAll('[data-disk-del-key]')) {
|
||
btn.addEventListener('click', () => openDiskDeleteDialog(btn.dataset.diskDelKey));
|
||
}
|
||
}
|
||
|
||
const trashIcon = '<svg viewBox="0 0 24 24" width="14" height="14" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true"><polyline points="3 6 5 6 21 6"></polyline><path d="M19 6l-1 14a2 2 0 0 1-2 2H8a2 2 0 0 1-2-2L5 6"></path><path d="M10 11v6"></path><path d="M14 11v6"></path><path d="M9 6V4a2 2 0 0 1 2-2h2a2 2 0 0 1 2 2v2"></path></svg>';
|
||
|
||
async function testLaunch(key, btn) {
|
||
const resultEl = document.querySelector(`[data-test-result-for="${key}"]`);
|
||
if (!resultEl) return;
|
||
const originalText = btn.textContent;
|
||
btn.disabled = true;
|
||
btn.textContent = 'Testing…';
|
||
resultEl.classList.remove('hidden', 'ok', 'fail');
|
||
resultEl.innerHTML = '<span class="muted small">Checking launch args against vLLM\'s parser…</span>';
|
||
try {
|
||
const r = await fetchJSON(`/api/swap/${encodeURIComponent(key)}/validate`, { method: 'POST' });
|
||
if (r.ok) {
|
||
resultEl.classList.add('ok');
|
||
resultEl.innerHTML = `<span class="ok-mark">✓</span> Launch args parse OK. <span class="muted small">(Doesn't guarantee runtime success — only catches argparse-level issues.)</span>`;
|
||
} else {
|
||
resultEl.classList.add('fail');
|
||
const err = escapeHtml(r.error || 'unknown error');
|
||
const stage = r.stage ? ` <span class="muted small">(${escapeHtml(r.stage)})</span>` : '';
|
||
resultEl.innerHTML = `<span class="fail-mark">✗</span> Would fail: ${err}${stage}`;
|
||
}
|
||
} catch (e) {
|
||
resultEl.classList.add('fail');
|
||
resultEl.innerHTML = `<span class="fail-mark">✗</span> Test failed: ${escapeHtml(e.message)}`;
|
||
} finally {
|
||
btn.disabled = false;
|
||
btn.textContent = originalText;
|
||
}
|
||
}
|
||
|
||
function renderCurrent(status) {
|
||
const c = el('#current');
|
||
if (!status.configured) { c.innerHTML = `<span class="muted">not configured</span>`; return; }
|
||
if (status.current_swap_job) { c.innerHTML = `<span class="muted">swap in progress</span>`; return; }
|
||
const v = status.vllm || {};
|
||
if (!v.ok) { c.innerHTML = `<span class="muted">vLLM unreachable</span>`; return; }
|
||
const m = status.current_model_key ? state.models[status.current_model_key] : null;
|
||
const label = m ? m.display_name : (v.current_model || '(unknown)');
|
||
c.innerHTML = `<strong>${label}</strong>`;
|
||
}
|
||
|
||
// ===================== hardware dashboard =====================
|
||
|
||
function fmtBytes(n) {
|
||
if (!n && n !== 0) return '—';
|
||
const u = ['B', 'KB', 'MB', 'GB', 'TB'];
|
||
let i = 0; let v = n;
|
||
while (v >= 1024 && i < u.length - 1) { v /= 1024; i++; }
|
||
return v < 10 ? `${v.toFixed(1)} ${u[i]}` : `${Math.round(v)} ${u[i]}`;
|
||
}
|
||
function fmtMiB(n) {
|
||
if (!n && n !== 0) return null;
|
||
// n is in MiB; render in GB
|
||
const gb = n / 1024;
|
||
return gb < 10 ? gb.toFixed(1) : Math.round(gb).toString();
|
||
}
|
||
|
||
function bar(usedPct, warn) {
|
||
const pct = Math.max(2, Math.min(100, usedPct));
|
||
return `<div class="bar ${warn ? 'warn' : ''}"><span style="width:${pct}%"></span></div>`;
|
||
}
|
||
|
||
async function pollHardware() {
|
||
try {
|
||
state.hardware = await fetchJSON('/api/hardware');
|
||
try { state.connectivity = await fetchJSON('/api/connectivity'); } catch {}
|
||
renderHardware();
|
||
} catch (e) { console.warn('hardware poll failed', e); }
|
||
}
|
||
|
||
function fmtDuration(sec) {
|
||
if (sec == null) return '';
|
||
if (sec < 60) return `${Math.round(sec)}s`;
|
||
if (sec < 3600) return `${Math.round(sec / 60)}m`;
|
||
if (sec < 86400) {
|
||
const h = Math.floor(sec / 3600);
|
||
const m = Math.round((sec % 3600) / 60);
|
||
return m ? `${h}h ${m}m` : `${h}h`;
|
||
}
|
||
const d = Math.floor(sec / 86400);
|
||
const h = Math.round((sec % 86400) / 3600);
|
||
return h ? `${d}d ${h}h` : `${d}d`;
|
||
}
|
||
|
||
function openConnectivityDialog() {
|
||
const dlg = el('#connectivity-dialog');
|
||
const content = el('#connectivity-content');
|
||
const c = state.connectivity || {};
|
||
const events = c.events || [];
|
||
if (events.length === 0) {
|
||
content.innerHTML = '<div class="muted small">No events recorded yet. Once a Spark or service goes down and back up (or an external app reports a failure), entries appear here.</div>';
|
||
dlg.showModal();
|
||
return;
|
||
}
|
||
const bySubject = {};
|
||
for (const e of events) {
|
||
const subj = e.subject || e.spark || 'unknown'; // legacy fallback
|
||
(bySubject[subj] = bySubject[subj] || []).push(e);
|
||
}
|
||
// Sort subjects: hosts first, then services, alphabetical
|
||
const hostOrder = ['spark1', 'spark2'];
|
||
const subjects = Object.keys(bySubject).sort((a, b) => {
|
||
const ia = hostOrder.indexOf(a);
|
||
const ib = hostOrder.indexOf(b);
|
||
if (ia >= 0 && ib >= 0) return ia - ib;
|
||
if (ia >= 0) return -1;
|
||
if (ib >= 0) return 1;
|
||
return a.localeCompare(b);
|
||
});
|
||
|
||
const html = subjects.map((subj) => {
|
||
const evs = bySubject[subj];
|
||
const transitions = evs.filter(e => (e.kind || 'transition') === 'transition');
|
||
const reports = evs.filter(e => e.kind === 'report');
|
||
const downs = transitions.filter(e => e.transition === 'down').length;
|
||
const failedReports = reports.filter(e => !e.ok).length;
|
||
const mac = c.macs?.[subj];
|
||
const summaryParts = [];
|
||
if (transitions.length) summaryParts.push(`${transitions.length} probe transition${transitions.length===1?'':'s'} (${downs} down)`);
|
||
if (reports.length) summaryParts.push(`${reports.length} app report${reports.length===1?'':'s'} (${failedReports} failed)`);
|
||
const isHost = hostOrder.includes(subj);
|
||
return `
|
||
<div class="conn-spark">
|
||
<h4>${escapeHtml(subj)}${isHost ? ' <span class="muted small">[host]</span>' : ' <span class="muted small">[service]</span>'}${mac ? ` <span class="muted small">${escapeHtml(mac)}</span>` : ''}</h4>
|
||
<div class="conn-summary">${summaryParts.join(' · ') || 'no events'}</div>
|
||
${evs.slice(-30).reverse().map(e => renderConnEvent(e)).join('')}
|
||
</div>
|
||
`;
|
||
}).join('');
|
||
content.innerHTML = html;
|
||
dlg.showModal();
|
||
}
|
||
|
||
function renderConnEvent(e) {
|
||
const when = escapeHtml((e.at || '').replace('T', ' ').replace('Z', ''));
|
||
const kind = e.kind || 'transition';
|
||
if (kind === 'report') {
|
||
const ok = !!e.ok;
|
||
const source = escapeHtml(e.source || 'external');
|
||
const detail = e.detail ? ` — ${escapeHtml(e.detail)}` : '';
|
||
const latency = e.latency_ms != null ? ` (${e.latency_ms} ms)` : '';
|
||
return `
|
||
<div class="conn-event ${ok ? 'up' : 'down'} report">
|
||
<span class="when">${when}</span>
|
||
<span class="what">${ok ? '◷ report: ok' : '◷ report: failed'} <span class="muted">from</span> ${source}${detail}</span>
|
||
<span class="dur">${latency}</span>
|
||
</div>
|
||
`;
|
||
}
|
||
const down = e.down_seconds != null ? `was down ${fmtDuration(e.down_seconds)}` : '';
|
||
const up = e.up_seconds != null ? `was up ${fmtDuration(e.up_seconds)}` : '';
|
||
return `
|
||
<div class="conn-event ${e.transition}">
|
||
<span class="when">${when}</span>
|
||
<span class="what">${e.transition === 'up' ? '↑ came back online' : '↓ dropped offline'}</span>
|
||
<span class="dur">${down}${up}</span>
|
||
</div>
|
||
`;
|
||
}
|
||
|
||
async function wakeSpark(name) {
|
||
try {
|
||
const r = await fetchJSON(`/api/spark/${name}/wake`, { method: 'POST' });
|
||
alert(`Wake-on-LAN sent to ${name} (MAC ${r.mac}, via ${r.delivered_via}). Give it ~30 seconds to wake; the card will go green when it comes back.`);
|
||
} catch (e) {
|
||
alert(`Wake failed: ${e.message}`);
|
||
}
|
||
}
|
||
|
||
// Generate-if-missing + copy this Spark's OUTBOUND ssh public key (the key the
|
||
// Spark uses to log in to other machines, e.g. the Mac). Distinct from the
|
||
// package's own key in the StartOS "Show Public Key" action.
|
||
async function copySparkSshKey(name, btn) {
|
||
if (btn) btn.disabled = true;
|
||
try {
|
||
const r = await fetchJSON(`/api/spark/${name}/ssh-key`, { method: 'POST' });
|
||
// Best-effort clipboard copy; on plain-HTTP this no-ops, but the dialog
|
||
// below always shows the key for manual selection.
|
||
await copyText(r.pubkey, btn);
|
||
const label = r.host ? `${name} (${r.host})` : name;
|
||
el('#sshkey-title').textContent = `${name} — SSH public key`;
|
||
el('#sshkey-intro').textContent = r.created
|
||
? `Generated a new SSH key on ${label} and copied it to your clipboard. This is the key ${name} uses to log in to OTHER machines.`
|
||
: `${label} already had an SSH key; copied its public key to your clipboard. This is the key ${name} uses to log in to OTHER machines.`;
|
||
el('#sshkey-value').textContent = r.pubkey;
|
||
el('#sshkey-install').textContent =
|
||
`mkdir -p ~/.ssh && echo '${r.pubkey}' >> ~/.ssh/authorized_keys && chmod 600 ~/.ssh/authorized_keys`;
|
||
el('#sshkey-dialog').showModal();
|
||
} catch (e) {
|
||
alert(`Couldn't get the SSH key for ${name}: ${e.message}`);
|
||
} finally {
|
||
if (btn) btn.disabled = false;
|
||
}
|
||
}
|
||
|
||
function renderHardware() {
|
||
const panel = el('#hardware-panel');
|
||
const grid = el('#hardware-grid');
|
||
const hw = state.hardware || {};
|
||
const keys = Object.keys(hw).filter(k => hw[k] && (hw[k].configured !== false));
|
||
if (keys.length === 0) { panel.classList.add('hidden'); return; }
|
||
panel.classList.remove('hidden');
|
||
grid.innerHTML = '';
|
||
for (const key of keys) {
|
||
const s = hw[key];
|
||
const card = document.createElement('div');
|
||
if (!s.reachable) {
|
||
card.className = 'hw-card unreachable';
|
||
const mac = state.connectivity?.macs?.[key];
|
||
const wolRow = mac
|
||
? `<div class="wol-row">
|
||
<span class="mac-display">${escapeHtml(mac)}</span>
|
||
<span class="spacer"></span>
|
||
<button class="btn" data-wake="${escapeHtml(key)}">Wake (WoL)</button>
|
||
</div>`
|
||
: `<div class="muted small">MAC not yet known — once it's been up once with this dashboard installed, "Wake" will appear here.</div>`;
|
||
card.innerHTML = `
|
||
<div class="head">
|
||
<span class="name">${escapeHtml(key)}</span>
|
||
<span class="meta">unreachable</span>
|
||
</div>
|
||
<div class="muted small">${escapeHtml(s.host || '')} — ${escapeHtml(s.error || 'no response')}</div>
|
||
${wolRow}
|
||
<div class="muted small" style="line-height:1.5">
|
||
If Wake-on-LAN doesn't bring it back, manual steps:
|
||
<ol style="margin: 6px 0 0 18px; padding: 0;">
|
||
<li>Verify it's powered on (check the front LED).</li>
|
||
<li>Ping it from another LAN device.</li>
|
||
<li>Power-cycle it physically.</li>
|
||
<li>If it boots, this card will go green again automatically.</li>
|
||
</ol>
|
||
</div>
|
||
`;
|
||
grid.appendChild(card);
|
||
continue;
|
||
}
|
||
const ramPct = s.ram_used_bytes && s.ram_total_bytes ? (s.ram_used_bytes / s.ram_total_bytes) * 100 : 0;
|
||
const diskPct = s.disk_used_bytes && s.disk_total_bytes ? (s.disk_used_bytes / s.disk_total_bytes) * 100 : 0;
|
||
const loadPct = (s.load && s.cores) ? Math.min(100, (s.load[0] / s.cores) * 100) : 0;
|
||
// GPU memory: on unified-memory systems (DGX Spark) total is N/A, so use system RAM as the pool.
|
||
const gpuMemTotalMiB = s.gpu_mem_total_mib || (s.gpu_unified_memory ? (s.ram_total_bytes / (1024 * 1024)) : null);
|
||
const gpuMemUsedMiB = s.gpu_mem_used_mib ?? null;
|
||
const gpuMemPct = (gpuMemTotalMiB && gpuMemUsedMiB != null) ? (gpuMemUsedMiB / gpuMemTotalMiB) * 100 : 0;
|
||
const gpuMemNote = s.gpu_unified_memory ? ' <span class="muted">(unified)</span>' : '';
|
||
const gpuExtras = [];
|
||
if (s.gpu_temp_c != null) gpuExtras.push(`${s.gpu_temp_c}°C`);
|
||
if (s.gpu_power_w != null) gpuExtras.push(`${s.gpu_power_w.toFixed(0)}W`);
|
||
const gpuExtrasStr = gpuExtras.length ? ` · ${gpuExtras.join(' · ')}` : '';
|
||
// Read-only WireGuard badge: shown only when the Spark has a wg interface up.
|
||
// "VPN <ip>" means it's a peer on that tunnel (reachable off-LAN when the
|
||
// tunnel is up); it reflects interface presence, not live peer reachability.
|
||
const wgIp = s.wg_addr ? String(s.wg_addr).split('/')[0] : '';
|
||
const wgBadge = s.wg_iface
|
||
? ` · <span class="wg-badge" title="On WireGuard tunnel '${escapeHtml(s.wg_iface)}'${wgIp ? ' as ' + escapeHtml(wgIp) : ''} — reachable off-LAN while the tunnel is up">VPN${wgIp ? ' ' + escapeHtml(wgIp) : ''}</span>`
|
||
: '';
|
||
card.className = 'hw-card';
|
||
card.innerHTML = `
|
||
<div class="head">
|
||
<span class="name">${escapeHtml(s.hostname || key)}</span>
|
||
<span class="meta">${escapeHtml(key)} · ${escapeHtml(s.gpu_name || '')} · ${escapeHtml(s.uptime || '')}${wgBadge}</span>
|
||
<button class="icon-btn ssh-key-btn" data-ssh-key="${escapeHtml(key)}" title="Copy this Spark's SSH public key (creates one if it doesn't have one) — e.g. to let it log in to your Mac" aria-label="Copy SSH public key">
|
||
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="9" y="9" width="13" height="13" rx="2"/><path d="M5 15H4a2 2 0 0 1-2-2V4a2 2 0 0 1 2-2h9a2 2 0 0 1 2 2v1"/></svg>
|
||
</button>
|
||
</div>
|
||
<div class="hw-metric">
|
||
<span class="label">CPU</span>
|
||
${bar(loadPct, loadPct > 80)}
|
||
<span class="val">${s.load ? s.load[0].toFixed(2) : '—'} / ${s.cores || '?'} cores</span>
|
||
</div>
|
||
<div class="hw-metric">
|
||
<span class="label">RAM</span>
|
||
${bar(ramPct, ramPct > 85)}
|
||
<span class="val">${fmtBytes(s.ram_used_bytes)} / ${fmtBytes(s.ram_total_bytes)}</span>
|
||
</div>
|
||
<div class="hw-metric">
|
||
<span class="label">GPU mem${gpuMemNote}</span>
|
||
${bar(gpuMemPct, gpuMemPct > 90)}
|
||
<span class="val">${fmtMiB(gpuMemUsedMiB) || '—'} / ${fmtMiB(gpuMemTotalMiB) || '?'} GB</span>
|
||
</div>
|
||
<div class="hw-metric">
|
||
<span class="label">GPU util</span>
|
||
${bar(s.gpu_util_pct || 0, (s.gpu_util_pct || 0) > 90)}
|
||
<span class="val">${s.gpu_util_pct ?? 0}%${gpuExtrasStr}</span>
|
||
</div>
|
||
<div class="hw-metric">
|
||
<span class="label">Disk</span>
|
||
${bar(diskPct, diskPct > 85)}
|
||
<span class="val">${fmtBytes(s.disk_used_bytes)} / ${fmtBytes(s.disk_total_bytes)}</span>
|
||
</div>
|
||
`;
|
||
grid.appendChild(card);
|
||
}
|
||
}
|
||
|
||
// ===================== service classification =====================
|
||
|
||
function classifyService(s) {
|
||
// returns one of: running | unhealthy | missing | unconfigured | starting
|
||
if (!s.host) return 'unconfigured';
|
||
if (s.docker_state === 'missing') return 'missing';
|
||
if (s.docker_state === 'restarting') return 'unhealthy';
|
||
if (s.docker_state === 'exited') return 'unhealthy';
|
||
if (s.docker_state === 'running') {
|
||
// http_ready === false means an HTTP probe is expected but failing → still
|
||
// warming up. null means the service has no HTTP surface (e.g. the bot), so
|
||
// a running container is simply healthy.
|
||
if (s.http_ready === false) return 'starting';
|
||
return 'running';
|
||
}
|
||
return s.docker_state || 'unknown';
|
||
}
|
||
|
||
function statusLabel(cls) {
|
||
return {
|
||
running: 'Healthy',
|
||
unhealthy: 'Unhealthy',
|
||
starting: 'Starting…',
|
||
missing: 'Not installed',
|
||
unconfigured: 'Not configured',
|
||
unknown: 'Unknown',
|
||
}[cls] || cls;
|
||
}
|
||
|
||
async function renderServices() {
|
||
let services = state.services;
|
||
// First render: fetch.
|
||
if (!services || Object.keys(services).length === 0) {
|
||
try {
|
||
services = await fetchJSON('/api/services');
|
||
state.services = services;
|
||
} catch (e) { console.error('services fetch failed', e); return; }
|
||
}
|
||
const panel = el('#services-panel');
|
||
const grid = el('#services-grid');
|
||
const entries = Object.entries(services);
|
||
if (entries.length === 0) { panel.classList.add('hidden'); return; }
|
||
panel.classList.remove('hidden');
|
||
grid.innerHTML = '';
|
||
for (const [name, s] of entries) {
|
||
const cls = classifyService(s);
|
||
const isBot = s.kind === 'bot';
|
||
// The bot tile is opt-in: it only belongs to deployments that actually run
|
||
// matrix-bridge. When the container is absent (missing) or the host isn't
|
||
// configured, hide the tile entirely rather than show a stray red card.
|
||
if (isBot && (cls === 'missing' || cls === 'unconfigured')) continue;
|
||
const card = document.createElement('div');
|
||
card.className = `service-card ${cls}`;
|
||
const inFlight = state.service_action_in_flight && state.service_action_in_flight.startsWith(name + ':');
|
||
const disable = (action) => {
|
||
// Disable buttons that don't make sense for the current state
|
||
if (inFlight) return true;
|
||
if (cls === 'unconfigured' || cls === 'missing') return true;
|
||
if (action === 'start' && (cls === 'running' || cls === 'starting')) return true;
|
||
if (action === 'stop' && cls !== 'running' && cls !== 'starting' && cls !== 'unhealthy') return true;
|
||
return false;
|
||
};
|
||
const copyIcon = `<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="9" y="9" width="13" height="13" rx="2"/><path d="M5 15H4a2 2 0 0 1-2-2V4a2 2 0 0 1 2-2h9a2 2 0 0 1 2 2v1"/></svg>`;
|
||
const hostStr = s.host ? (s.port ? `${s.host}:${s.port}` : s.host) : '';
|
||
const hostRow = s.host
|
||
? `<div class="row"><span class="k">Host</span><span class="v copyable" data-copy-self title="Click to copy">${escapeHtml(hostStr)}</span><button class="icon-btn" data-copy-text="${escapeHtml(hostStr)}" title="Copy host" aria-label="Copy">${copyIcon}</button></div>`
|
||
: `<div class="row"><span class="k">Host</span><span class="v muted-v">not configured</span></div>`;
|
||
const urlRow = s.base_url
|
||
? `<div class="row"><span class="k">URL</span><span class="v copyable" data-copy-self title="Click to copy">${escapeHtml(s.base_url)}</span><button class="icon-btn" data-copy-text="${escapeHtml(s.base_url)}" title="Copy URL" aria-label="Copy">${copyIcon}</button></div>`
|
||
: '';
|
||
const modelRow = s.model
|
||
? `<div class="row"><span class="k">Model</span><span class="v copyable" data-copy-self title="Click to copy">${escapeHtml(s.model)}</span><button class="icon-btn" data-copy-text="${escapeHtml(s.model)}" title="Copy model" aria-label="Copy">${copyIcon}</button></div>`
|
||
: '';
|
||
const restartsRow = s.restart_count != null && s.restart_count > 1
|
||
? `<div class="row"><span class="k">Restarts</span><span class="v">${s.restart_count}</span></div>`
|
||
: '';
|
||
const dh = state.deep_health?.[name];
|
||
let deepRow = '';
|
||
if (dh && dh.last) {
|
||
const last = dh.last;
|
||
const when = (last.at || '').slice(11, 19); // HH:MM:SS
|
||
const verdict = last.ok
|
||
? `<span class="dh-ok">deep check ok</span>`
|
||
: `<span class="dh-fail">deep check FAILED</span>`;
|
||
const lat = last.latency_ms != null ? ` <span class="muted">${last.latency_ms} ms</span>` : '';
|
||
const restarts = dh.auto_restarts_window > 0
|
||
? ` <span class="muted">· ${dh.auto_restarts_window} auto-restart${dh.auto_restarts_window === 1 ? '' : 's'} in 30 min</span>`
|
||
: '';
|
||
deepRow = `
|
||
<div class="row deep-row">
|
||
<span class="k">Deep</span>
|
||
<span class="v deep-v">${verdict} <span class="muted small">${escapeHtml(when)}</span>${lat}${restarts}</span>
|
||
<button class="icon-btn dh-run-btn" data-dh-run="${escapeHtml(name)}" title="Run deep check now">↻</button>
|
||
</div>
|
||
${last.ok ? '' : `<div class="deep-error muted small">${escapeHtml((last.error || last.note || '').slice(0, 200))}</div>`}
|
||
`;
|
||
} else if (dh) {
|
||
deepRow = `
|
||
<div class="row deep-row">
|
||
<span class="k">Deep</span>
|
||
<span class="v muted-v">no probe yet</span>
|
||
<button class="icon-btn dh-run-btn" data-dh-run="${escapeHtml(name)}" title="Run deep check now">↻</button>
|
||
</div>
|
||
`;
|
||
}
|
||
card.innerHTML = `
|
||
<div class="head">
|
||
<span class="name">${escapeHtml(name)}</span>
|
||
<span class="kind">${escapeHtml(s.kind || '')}</span>
|
||
<span class="status">${statusLabel(cls)}</span>
|
||
</div>
|
||
${hostRow}
|
||
${urlRow}
|
||
${modelRow}
|
||
${restartsRow}
|
||
${deepRow}
|
||
<div class="service-actions">
|
||
${isBot ? `<button class="btn primary" data-mb-update title="Pull latest code, rebuild, and recreate the bot" ${inFlight || state.mb_update_in_flight ? 'disabled' : ''}>Update</button>` : ''}
|
||
<button class="btn" data-svc-action="${name}:start" ${disable('start') ? 'disabled' : ''}>Start</button>
|
||
<button class="btn" data-svc-action="${name}:restart" ${disable('restart') ? 'disabled' : ''}>Restart</button>
|
||
<button class="btn danger" data-svc-action="${name}:stop" ${disable('stop') ? 'disabled' : ''}>Stop</button>
|
||
${isBot ? `<button class="btn" data-mb-logs title="Show the last 100 log lines">View logs</button>` : ''}
|
||
</div>
|
||
`;
|
||
grid.appendChild(card);
|
||
}
|
||
for (const btn of grid.querySelectorAll('.btn[data-svc-action]')) {
|
||
btn.addEventListener('click', () => onServiceAction(btn.dataset.svcAction));
|
||
}
|
||
const mbUpdateBtn = grid.querySelector('[data-mb-update]');
|
||
if (mbUpdateBtn) mbUpdateBtn.addEventListener('click', onMatrixBridgeUpdate);
|
||
const mbLogsBtn = grid.querySelector('[data-mb-logs]');
|
||
if (mbLogsBtn) mbLogsBtn.addEventListener('click', openMatrixBridgeLogs);
|
||
for (const btn of grid.querySelectorAll('[data-dh-run]')) {
|
||
btn.addEventListener('click', () => onDeepHealthRun(btn.dataset.dhRun, btn));
|
||
}
|
||
}
|
||
|
||
async function onDeepHealthRun(name, btn) {
|
||
btn.disabled = true;
|
||
const orig = btn.textContent;
|
||
btn.textContent = '…';
|
||
try {
|
||
await fetchJSON(`/api/deep-health/${encodeURIComponent(name)}/run`, { method: 'POST' });
|
||
} catch (e) {
|
||
console.warn('deep-health run failed', e);
|
||
} finally {
|
||
try { state.deep_health = await fetchJSON('/api/deep-health'); } catch {}
|
||
btn.textContent = orig;
|
||
btn.disabled = false;
|
||
renderServices();
|
||
}
|
||
}
|
||
|
||
// ===================== speech-model patches (v0.11) =====================
|
||
|
||
async function renderSpeechModels() {
|
||
const panel = el('#speech-models-panel');
|
||
const card = el('#speech-models-card');
|
||
if (!panel || !card) return;
|
||
|
||
let data;
|
||
try {
|
||
data = await fetchJSON('/api/speech-models');
|
||
} catch (e) {
|
||
// If parakeet host isn't even configured, hide the section entirely
|
||
panel.classList.add('hidden');
|
||
return;
|
||
}
|
||
if (!data || !data.patches) { panel.classList.add('hidden'); return; }
|
||
panel.classList.remove('hidden');
|
||
|
||
const patches = data.patches || {};
|
||
const health = data.container_health || {};
|
||
const status = patches.status || 'unknown';
|
||
|
||
let statusPill;
|
||
if (status === 'in_sync') {
|
||
statusPill = `<span class="tag ok">patches in sync</span>`;
|
||
} else if (status === 'drift') {
|
||
statusPill = `<span class="tag warn">spark-control has newer patches</span>`;
|
||
} else if (status === 'missing') {
|
||
statusPill = `<span class="tag bad">patches missing in container</span>`;
|
||
} else {
|
||
statusPill = `<span class="tag warn">unknown</span>`;
|
||
}
|
||
|
||
const asrLoaded = !!health.asr_loaded;
|
||
const diarLoaded = !!health.diarizer_loaded;
|
||
const asrModel = escapeHtml(health.model || '—');
|
||
const diarModel = escapeHtml(health.diarizer_model || '—');
|
||
|
||
const fileRows = (patches.files || []).map((f) => {
|
||
const sync = f.in_sync
|
||
? '<span class="sm-file-ok">✓ in sync</span>'
|
||
: f.remote_sha == null
|
||
? '<span class="sm-file-bad">✗ missing</span>'
|
||
: '<span class="sm-file-warn">⚠ drift</span>';
|
||
const local = f.local_sha ? `<code>${escapeHtml(f.local_sha)}</code>` : '<span class="muted">—</span>';
|
||
const remote = f.remote_sha ? `<code>${escapeHtml(f.remote_sha)}</code>` : '<span class="muted">—</span>';
|
||
return `
|
||
<div class="sm-file-row">
|
||
<span class="sm-file-name"><code>${escapeHtml(f.name)}</code></span>
|
||
<span class="sm-file-sync">${sync}</span>
|
||
<span class="sm-file-sha muted small">local ${local} → remote ${remote}</span>
|
||
</div>
|
||
`;
|
||
}).join('');
|
||
|
||
const lastReapply = patches.last_reapply_at ? new Date(patches.last_reapply_at).toLocaleString() : 'never (since spark-control boot)';
|
||
const lastRestart = patches.last_restart_at ? new Date(patches.last_restart_at).toLocaleString() : 'never (since spark-control boot)';
|
||
|
||
card.innerHTML = `
|
||
<div class="sm-header">
|
||
<div class="sm-title">parakeet-asr container</div>
|
||
${statusPill}
|
||
</div>
|
||
<div class="sm-models">
|
||
<div class="sm-model-row">
|
||
<span class="sm-model-kind">Parakeet ASR</span>
|
||
<span class="sm-model-name">${asrModel}</span>
|
||
<span class="sm-model-loaded">${asrLoaded ? '<span class="tag ok">loaded</span>' : '<span class="tag bad">not loaded</span>'}</span>
|
||
</div>
|
||
<div class="sm-model-row">
|
||
<span class="sm-model-kind">Sortformer diarizer</span>
|
||
<span class="sm-model-name">${diarModel}</span>
|
||
<span class="sm-model-loaded">${diarLoaded ? '<span class="tag ok">loaded</span>' : '<span class="tag bad">not loaded</span>'}</span>
|
||
</div>
|
||
</div>
|
||
<div class="sm-files">${fileRows}</div>
|
||
<div class="sm-meta muted small">
|
||
Last reapply: ${escapeHtml(lastReapply)} · Last manual restart: ${escapeHtml(lastRestart)}
|
||
</div>
|
||
<div class="sm-actions">
|
||
<button class="btn primary" id="sm-reapply">Reapply patches</button>
|
||
<button class="btn" id="sm-restart">Restart container</button>
|
||
</div>
|
||
`;
|
||
|
||
el('#sm-reapply').addEventListener('click', onSpeechModelsReapply);
|
||
el('#sm-restart').addEventListener('click', onSpeechModelsRestart);
|
||
}
|
||
|
||
async function onSpeechModelsReapply() {
|
||
if (!confirm('Reapply Sortformer patches to the parakeet-asr container? The container will restart and both ASR + diarizer will be unavailable for ~60–120 seconds.')) return;
|
||
const dlg = el('#speech-models-progress-dialog');
|
||
const steps = el('#sm-prog-steps');
|
||
const closeBtn = el('#sm-prog-close');
|
||
steps.innerHTML = '<div class="muted small">Starting…</div>';
|
||
closeBtn.disabled = true;
|
||
closeBtn.onclick = () => dlg.close();
|
||
dlg.showModal();
|
||
try {
|
||
const r = await fetchJSON('/api/speech-models/reapply', { method: 'POST' });
|
||
steps.innerHTML = (r.steps || []).map((s) => {
|
||
const mark = s.ok ? '<span class="sm-file-ok">✓</span>' : '<span class="sm-file-bad">✗</span>';
|
||
const extra = s.error ? `<div class="muted small">${escapeHtml(s.error)}</div>` : '';
|
||
return `<div class="sm-prog-step">${mark} <strong>${escapeHtml(s.step)}</strong>${s.name ? ` (${escapeHtml(s.name)})` : ''}${extra}</div>`;
|
||
}).join('') + `<div class="sm-prog-done sm-file-ok">Done — both models reloaded.</div>`;
|
||
} catch (e) {
|
||
let parsed = null;
|
||
try { parsed = JSON.parse(e.message.split(':').slice(2).join(':').trim()); } catch {}
|
||
const stepHtml = parsed && parsed.result && parsed.result.steps
|
||
? parsed.result.steps.map((s) => {
|
||
const mark = s.ok ? '<span class="sm-file-ok">✓</span>' : '<span class="sm-file-bad">✗</span>';
|
||
return `<div class="sm-prog-step">${mark} <strong>${escapeHtml(s.step)}</strong>${s.name ? ` (${escapeHtml(s.name)})` : ''}${s.error ? `<div class="muted small">${escapeHtml(s.error)}</div>` : ''}</div>`;
|
||
}).join('')
|
||
: `<div class="sm-file-bad">${escapeHtml(e.message)}</div>`;
|
||
steps.innerHTML = stepHtml + `<div class="sm-prog-done sm-file-bad">Failed.</div>`;
|
||
} finally {
|
||
closeBtn.disabled = false;
|
||
try { await renderSpeechModels(); } catch {}
|
||
}
|
||
}
|
||
|
||
async function onSpeechModelsRestart() {
|
||
if (!confirm('Restart parakeet-asr container? STT + diarization will be unavailable for ~30 seconds.')) return;
|
||
try {
|
||
await fetchJSON('/api/speech-models/restart', { method: 'POST' });
|
||
} catch (e) {
|
||
alert('Restart failed: ' + e.message);
|
||
} finally {
|
||
try { await renderSpeechModels(); } catch {}
|
||
}
|
||
}
|
||
|
||
// NOTE: a WhisperX install action lived here briefly in v0.12 but was
|
||
// reverted in v0.13.0:0 — the NGC PyTorch container on ARM64 doesn't ship
|
||
// torchaudio and we couldn't reliably build it from source. The existing
|
||
// Parakeet + Sortformer pipeline stays as the audio path. See release notes.
|
||
|
||
async function onServiceAction(key) {
|
||
if (state.service_action_in_flight) return;
|
||
const [name, action] = key.split(':');
|
||
state.service_action_in_flight = key;
|
||
renderServices();
|
||
try {
|
||
await fetchJSON(`/api/services/${name}/${action}`, { method: 'POST' });
|
||
} catch (e) {
|
||
alert(`${action} ${name} failed: ${e.message}`);
|
||
} finally {
|
||
state.service_action_in_flight = null;
|
||
// Refresh services state
|
||
try {
|
||
state.services = await fetchJSON('/api/services');
|
||
} catch {}
|
||
renderServices();
|
||
pollStatus();
|
||
}
|
||
}
|
||
|
||
// ===================== matrix-bridge bot (update + logs) =====================
|
||
|
||
const mbState = { job_id: null, eventsource: null, timer: null, started_at: null };
|
||
|
||
function mbTimerStart(at) {
|
||
mbState.started_at = at;
|
||
if (mbState.timer) clearInterval(mbState.timer);
|
||
const tick = () => {
|
||
if (!mbState.started_at) return;
|
||
const sec = Math.max(0, Math.floor((Date.now() - mbState.started_at) / 1000));
|
||
el('#mb-update-elapsed').textContent = `${Math.floor(sec / 60)}:${(sec % 60).toString().padStart(2, '0')}`;
|
||
};
|
||
tick();
|
||
mbState.timer = setInterval(tick, 500);
|
||
}
|
||
|
||
async function onMatrixBridgeUpdate() {
|
||
if (state.mb_update_in_flight) return;
|
||
if (!confirm('Update the matrix-bridge bot?\n\nThis pulls the latest code, rebuilds the container image, and recreates the container. The first build after a base-image change can take several minutes. The bot is briefly offline while it restarts.')) return;
|
||
state.mb_update_in_flight = true;
|
||
renderServices();
|
||
try {
|
||
const r = await fetchJSON('/api/matrix-bridge/update', { method: 'POST' });
|
||
attachMbUpdateProgress(r.job_id);
|
||
} catch (e) {
|
||
state.mb_update_in_flight = false;
|
||
renderServices();
|
||
alert('Update failed to start: ' + e.message);
|
||
}
|
||
}
|
||
|
||
async function attachMbUpdateProgress(jobId) {
|
||
mbState.job_id = jobId;
|
||
el('#mb-update-log').textContent = '';
|
||
el('#mb-update-title').textContent = 'Updating matrix-bridge…';
|
||
el('#mb-update-phase').textContent = 'Starting…';
|
||
el('#mb-update-dialog').showModal();
|
||
try {
|
||
const snap = await fetchJSON(`/api/matrix-bridge/update/${jobId}`);
|
||
mbTimerStart(Date.parse(snap.started_at));
|
||
el('#mb-update-phase').textContent = snap.phase || 'Working…';
|
||
el('#mb-update-log').textContent = (snap.lines || []).join('\n');
|
||
if (snap.returncode !== null) { onMbUpdateDone(snap); return; }
|
||
} catch { mbTimerStart(Date.now()); }
|
||
const es = new EventSource(`/api/matrix-bridge/update/${jobId}/stream`);
|
||
mbState.eventsource = es;
|
||
es.onmessage = ev => {
|
||
try {
|
||
const d = JSON.parse(ev.data);
|
||
if (d.line !== undefined) {
|
||
const log = el('#mb-update-log');
|
||
log.textContent += d.line + '\n';
|
||
log.scrollTop = log.scrollHeight;
|
||
}
|
||
} catch {}
|
||
};
|
||
es.addEventListener('phase', ev => {
|
||
try { el('#mb-update-phase').textContent = JSON.parse(ev.data).phase; } catch {}
|
||
});
|
||
es.addEventListener('done', ev => {
|
||
let d = {}; try { d = JSON.parse(ev.data); } catch {}
|
||
onMbUpdateDone(d);
|
||
});
|
||
es.onerror = () => {
|
||
// Don't leave the Update button wedged-disabled on a dropped stream. The
|
||
// job keeps running server-side; re-clicking Update returns a clean 409.
|
||
es.close();
|
||
mbState.eventsource = null;
|
||
state.mb_update_in_flight = false;
|
||
el('#mb-update-phase').textContent = 'Lost connection to the update stream — reopen or check logs.';
|
||
renderServices();
|
||
};
|
||
}
|
||
|
||
function onMbUpdateDone(d) {
|
||
if (mbState.eventsource) { mbState.eventsource.close(); mbState.eventsource = null; }
|
||
if (mbState.timer) { clearInterval(mbState.timer); mbState.timer = null; }
|
||
state.mb_update_in_flight = false;
|
||
if (d.state === 'failed') {
|
||
el('#mb-update-title').textContent = `Update failed (rc=${d.returncode})`;
|
||
el('#mb-update-phase').textContent = 'Failed — see the log above.';
|
||
} else {
|
||
el('#mb-update-title').textContent = 'Update complete';
|
||
el('#mb-update-phase').textContent = 'Done ✓';
|
||
}
|
||
// Refresh the tile's badge.
|
||
(async () => { try { state.services = await fetchJSON('/api/services'); } catch {} renderServices(); })();
|
||
}
|
||
|
||
async function openMatrixBridgeLogs() {
|
||
const pre = el('#mb-logs-pre');
|
||
el('#mb-logs-title').textContent = 'matrix-bridge logs';
|
||
pre.textContent = 'Loading…';
|
||
el('#mb-logs-dialog').showModal();
|
||
await loadMatrixBridgeLogs();
|
||
}
|
||
|
||
async function loadMatrixBridgeLogs() {
|
||
const pre = el('#mb-logs-pre');
|
||
const btn = el('#mb-logs-refresh');
|
||
if (btn) btn.disabled = true;
|
||
try {
|
||
const r = await fetchJSON('/api/matrix-bridge/logs?tail=100');
|
||
pre.textContent = r.output || '(no output)';
|
||
pre.scrollTop = pre.scrollHeight;
|
||
} catch (e) {
|
||
pre.textContent = 'Could not read logs: ' + e.message;
|
||
} finally {
|
||
if (btn) btn.disabled = false;
|
||
}
|
||
}
|
||
|
||
function renderEndpoint(status) {
|
||
const v = status.vllm || {};
|
||
const panel = el('#endpoint-panel');
|
||
const ready = v.ok && v.current_model && v.base_url;
|
||
panel.classList.toggle('hidden', !ready);
|
||
if (!ready) return;
|
||
el('#ep-url').textContent = v.base_url;
|
||
el('#ep-model').textContent = v.current_model;
|
||
const snippet =
|
||
`curl -s ${v.base_url}/chat/completions \\
|
||
-H 'content-type: application/json' \\
|
||
-d '{
|
||
"model": "${v.current_model}",
|
||
"messages": [{"role": "user", "content": "Hello"}]
|
||
}'`;
|
||
el('#ep-curl-snippet').textContent = snippet;
|
||
}
|
||
|
||
async function copyText(text, indicatorEl) {
|
||
try {
|
||
await navigator.clipboard.writeText(text);
|
||
if (indicatorEl) {
|
||
indicatorEl.classList.add('copied');
|
||
setTimeout(() => indicatorEl.classList.remove('copied'), 1200);
|
||
}
|
||
return true;
|
||
} catch {
|
||
// Plain HTTP fallback: select the text so the user can ⌘C
|
||
if (indicatorEl) {
|
||
const range = document.createRange();
|
||
range.selectNode(indicatorEl);
|
||
window.getSelection().removeAllRanges();
|
||
window.getSelection().addRange(range);
|
||
}
|
||
return false;
|
||
}
|
||
}
|
||
|
||
function setupCopyButtons() {
|
||
document.body.addEventListener('click', async (e) => {
|
||
// Inline icon copy with literal text (used for dynamically-rendered service rows)
|
||
const litBtn = e.target.closest('[data-copy-text]');
|
||
if (litBtn) {
|
||
await copyText(litBtn.dataset.copyText, litBtn);
|
||
return;
|
||
}
|
||
// Copy buttons (with svg icon) referenced by data-copy="selector"
|
||
const btn = e.target.closest('[data-copy]');
|
||
if (btn) {
|
||
const target = el(btn.dataset.copy);
|
||
if (target) {
|
||
await copyText(target.textContent, btn);
|
||
target.classList.add('copied');
|
||
setTimeout(() => target.classList.remove('copied'), 1200);
|
||
}
|
||
return;
|
||
}
|
||
// Self-copy: clicking the text itself
|
||
const selfCopy = e.target.closest('[data-copy-self]');
|
||
if (selfCopy) {
|
||
await copyText(selfCopy.textContent, selfCopy);
|
||
}
|
||
});
|
||
}
|
||
|
||
function renderHealth(status) {
|
||
function setDot(id, ok, payload) {
|
||
const item = el(id);
|
||
if (!item) return;
|
||
// A service switched off via DISABLED_SERVICES isn't part of this
|
||
// deployment — hide its indicator entirely rather than show it as down.
|
||
if (payload && payload.disabled) { item.classList.add('hidden'); return; }
|
||
item.classList.remove('hidden');
|
||
const dot = item.querySelector('.dot');
|
||
dot.classList.remove('ok', 'bad', 'warn');
|
||
if (ok === true) dot.classList.add('ok');
|
||
else if (ok === false) dot.classList.add('bad');
|
||
else dot.classList.add('warn');
|
||
item.title = JSON.stringify(payload || {}, null, 2);
|
||
}
|
||
setDot('#h-vllm', status.vllm && status.vllm.ok, status.vllm);
|
||
setDot('#h-parakeet', status.parakeet && status.parakeet.ok, status.parakeet);
|
||
setDot('#h-kokoro', status.kokoro && status.kokoro.ok, status.kokoro);
|
||
setDot('#h-embeddings', status.embeddings && status.embeddings.ok, status.embeddings);
|
||
setDot('#h-qdrant', status.qdrant && status.qdrant.ok, status.qdrant);
|
||
el('#updated').textContent = `updated ${new Date().toLocaleTimeString()}`;
|
||
}
|
||
|
||
function renderBanner(status) {
|
||
el('#setup-banner').classList.toggle('hidden', !!status.configured);
|
||
// Dashboard tabs share the same "configured" gate as the rest of the
|
||
// body — hidden until SSH is set up, then visible.
|
||
const tabs = el('#dashboard-tabs');
|
||
if (tabs) tabs.classList.toggle('hidden', !status.configured);
|
||
}
|
||
|
||
// ===================== dashboard tabs (LLM / Audio) =====================
|
||
|
||
const TABS_STORAGE_KEY = 'sparkcontrol.dashboard.activeTab';
|
||
|
||
function setupDashboardTabs() {
|
||
const buttons = $$('.dashboard-tab');
|
||
if (!buttons.length) return;
|
||
|
||
// Restore the last-selected tab, default to "llm"
|
||
let saved;
|
||
try { saved = localStorage.getItem(TABS_STORAGE_KEY); } catch {}
|
||
const initial = saved === 'audio' || saved === 'llm' ? saved : 'llm';
|
||
|
||
function selectTab(name) {
|
||
buttons.forEach((b) => {
|
||
const active = b.dataset.tab === name;
|
||
b.classList.toggle('active', active);
|
||
b.setAttribute('aria-selected', active ? 'true' : 'false');
|
||
});
|
||
$$('.tab-content').forEach((c) => {
|
||
c.classList.toggle('active', c.id === `tab-${name}`);
|
||
});
|
||
try { localStorage.setItem(TABS_STORAGE_KEY, name); } catch {}
|
||
}
|
||
|
||
buttons.forEach((b) => {
|
||
b.addEventListener('click', () => selectTab(b.dataset.tab));
|
||
});
|
||
selectTab(initial);
|
||
}
|
||
|
||
// ===================== collapsible endpoint card =====================
|
||
|
||
const ENDPOINT_COLLAPSED_KEY = 'sparkcontrol.endpoint.collapsed';
|
||
|
||
function setupEndpointCollapse() {
|
||
const panel = el('#endpoint-panel');
|
||
const btn = el('#ep-collapse');
|
||
if (!panel || !btn) return;
|
||
// Default: collapsed (most of the time you don't need to see endpoint details)
|
||
let collapsed = true;
|
||
try {
|
||
const v = localStorage.getItem(ENDPOINT_COLLAPSED_KEY);
|
||
if (v === 'false') collapsed = false;
|
||
else if (v === 'true') collapsed = true;
|
||
} catch {}
|
||
panel.classList.toggle('collapsed', collapsed);
|
||
btn.addEventListener('click', () => {
|
||
const nowCollapsed = !panel.classList.contains('collapsed');
|
||
panel.classList.toggle('collapsed', nowCollapsed);
|
||
try { localStorage.setItem(ENDPOINT_COLLAPSED_KEY, nowCollapsed ? 'true' : 'false'); } catch {}
|
||
});
|
||
}
|
||
|
||
function renderSwapPanel() {
|
||
el('#swap-phase').textContent = state.swap_phase;
|
||
el('#swap-phase-detail').textContent = state.swap_phase_detail;
|
||
el('#swap-phase-fill').style.width = `${Math.max(2, Math.round(state.swap_progress * 100))}%`;
|
||
}
|
||
|
||
// ===================== phase detection =====================
|
||
|
||
const PHASE_ORDER = [
|
||
['Stopping current model…', 0.08],
|
||
['Starting new model…', 0.16],
|
||
['Joining Ray cluster…', 0.22],
|
||
['Loading weights…', 0.30],
|
||
['Compiling kernels…', 0.78],
|
||
['Warming up…', 0.88],
|
||
['Starting API server…', 0.94],
|
||
['Ready ✓', 1.00],
|
||
['Failed', 1.00],
|
||
];
|
||
|
||
function phaseProgress(name) {
|
||
const found = PHASE_ORDER.find(([n]) => n === name);
|
||
return found ? found[1] : 0.05;
|
||
}
|
||
|
||
function deriveSwapPhase(serverState, lines) {
|
||
// Default phase from server state
|
||
let phase = ({
|
||
starting: 'Starting…',
|
||
stopping: 'Stopping current model…',
|
||
launching: 'Starting new model…',
|
||
tailing: 'Loading weights…',
|
||
ready: 'Ready ✓',
|
||
failed: 'Failed',
|
||
})[serverState] || 'Working…';
|
||
|
||
let detail = '';
|
||
|
||
// Refine from log content (search recent lines first)
|
||
const tail = lines.slice(-40);
|
||
for (let i = tail.length - 1; i >= 0; i--) {
|
||
const line = tail[i];
|
||
if (line.includes('Application startup complete')) {
|
||
phase = 'Ready ✓';
|
||
break;
|
||
}
|
||
if (line.includes('Started server process')) {
|
||
phase = 'Starting API server…';
|
||
break;
|
||
}
|
||
if (line.includes('Initial profiling/warmup') || line.includes('init engine (profile, create kv cache, warmup model)')) {
|
||
phase = 'Warming up…';
|
||
break;
|
||
}
|
||
if (line.match(/Capturing CUDA graphs|Compiling a graph|torch\.compile took|Graph capturing/)) {
|
||
phase = 'Compiling kernels…';
|
||
break;
|
||
}
|
||
const shard = line.match(/Loading safetensors checkpoint shards:\s+(\d+)%\s+Completed\s+\|\s+(\d+)\/(\d+)/);
|
||
if (shard) {
|
||
phase = 'Loading weights…';
|
||
detail = `${shard[2]} of ${shard[3]} shards (${shard[1]}%)`;
|
||
const innerProgress = parseInt(shard[2], 10) / parseInt(shard[3], 10);
|
||
// Map shard progress 0..1 into the 0.30..0.78 band
|
||
state.swap_progress = 0.30 + (0.78 - 0.30) * innerProgress;
|
||
state.swap_phase = phase;
|
||
state.swap_phase_detail = detail;
|
||
return;
|
||
}
|
||
if (line.includes('Connecting to existing Ray cluster')) {
|
||
phase = 'Joining Ray cluster…';
|
||
break;
|
||
}
|
||
if (line.includes('Resolved architecture') || line.match(/launch-cluster\.sh.*exec vllm serve/)) {
|
||
phase = 'Starting new model…';
|
||
break;
|
||
}
|
||
if (line.match(/launch-cluster\.sh stop/)) {
|
||
phase = 'Stopping current model…';
|
||
break;
|
||
}
|
||
}
|
||
|
||
state.swap_phase = phase;
|
||
state.swap_phase_detail = detail;
|
||
state.swap_progress = phaseProgress(phase);
|
||
}
|
||
|
||
// ===================== timer =====================
|
||
|
||
function startTimer(startedAtMillis) {
|
||
state.swap_started_at = startedAtMillis;
|
||
if (state.timer_handle) clearInterval(state.timer_handle);
|
||
const tick = () => {
|
||
if (!state.swap_started_at) return;
|
||
const sec = Math.max(0, Math.floor((Date.now() - state.swap_started_at) / 1000));
|
||
const m = Math.floor(sec / 60);
|
||
const s = sec % 60;
|
||
el('#swap-elapsed').textContent = `${m}:${s.toString().padStart(2, '0')}`;
|
||
};
|
||
tick();
|
||
state.timer_handle = setInterval(tick, 500);
|
||
}
|
||
|
||
function stopTimer() {
|
||
if (state.timer_handle) { clearInterval(state.timer_handle); state.timer_handle = null; }
|
||
}
|
||
|
||
// ===================== polling + SSE =====================
|
||
|
||
async function pollStatus() {
|
||
try {
|
||
const status = await fetchJSON('/api/status');
|
||
state.current_model_key = status.current_model_key;
|
||
state.configured = status.configured;
|
||
renderBanner(status);
|
||
renderCurrent(status);
|
||
renderEndpoint(status);
|
||
renderHealth(status);
|
||
// If models hasn't loaded yet (init may have hit a transient proxy timeout), retry.
|
||
if (!state.models || Object.keys(state.models).length === 0) {
|
||
try { await loadModels(); } catch {}
|
||
}
|
||
// Refresh services state lazily — every 5s poll triggers this too.
|
||
try {
|
||
state.services = await fetchJSON('/api/services');
|
||
try { state.deep_health = await fetchJSON('/api/deep-health'); } catch {}
|
||
renderServices();
|
||
} catch {}
|
||
if (status.current_swap_job && status.current_swap_job !== state.swap_job_id) {
|
||
attachToSwap(status.current_swap_job, /*needsBackfill=*/true);
|
||
} else if (!status.current_swap_job && state.swap_job_id && !state.swap_eventsource) {
|
||
// Foreign swap ended
|
||
detachSwap();
|
||
}
|
||
renderCards();
|
||
} catch (e) {
|
||
console.error('status poll failed', e);
|
||
}
|
||
}
|
||
|
||
let menuLoadInFlight = false;
|
||
|
||
async function loadModels() {
|
||
// The menu is whatever's downloaded on the Sparks — /api/models does the scan
|
||
// (SSH), so this is the slower model call. Best-effort: a transient failure
|
||
// leaves the previous menu in place rather than blanking the dashboard.
|
||
// Guard against overlap: init() fires this un-awaited and pollStatus()'s
|
||
// empty-menu fallback may call it again before the scan returns.
|
||
if (menuLoadInFlight) return;
|
||
menuLoadInFlight = true;
|
||
try {
|
||
const data = await fetchJSON('/api/models');
|
||
state.defaults = data.defaults || {};
|
||
state.models = data.models || {};
|
||
state.recipes = data.recipes || [];
|
||
state.models_loaded = true;
|
||
populateDownloadSuggestions();
|
||
renderCards();
|
||
} catch (e) {
|
||
console.warn('model menu load failed:', e.message);
|
||
} finally {
|
||
menuLoadInFlight = false;
|
||
}
|
||
}
|
||
|
||
// Populate the download box's autocomplete with known recipes not currently on
|
||
// disk — so common/bundled models stay discoverable without phantom menu cards.
|
||
function populateDownloadSuggestions() {
|
||
const dl = el('#dl-suggestions');
|
||
if (!dl) return;
|
||
const onDiskRepos = new Set(Object.values(state.models).map(m => m.repo).filter(Boolean));
|
||
dl.innerHTML = '';
|
||
for (const r of state.recipes || []) {
|
||
if (onDiskRepos.has(r.repo)) continue;
|
||
const opt = document.createElement('option');
|
||
opt.value = r.repo;
|
||
opt.label = `${r.display_name} (${r.mode})`;
|
||
dl.appendChild(opt);
|
||
}
|
||
}
|
||
|
||
function fmtBytesShort(n) {
|
||
if (!Number.isFinite(n) || n <= 0) return '0 B';
|
||
if (n >= 1e9) return `${(n / 1e9).toFixed(1)} GB`;
|
||
if (n >= 1e6) return `${(n / 1e6).toFixed(1)} MB`;
|
||
if (n >= 1e3) return `${(n / 1e3).toFixed(1)} KB`;
|
||
return `${n} B`;
|
||
}
|
||
|
||
function openDiskDeleteDialog(key) {
|
||
const m = state.models[key];
|
||
if (!m || !m.on_disk) return;
|
||
const dlg = el('#disk-delete-dialog');
|
||
el('#dd-summary').innerHTML = `Free <strong>${fmtBytesShort(m.total_bytes)}</strong> by removing <strong>${escapeHtml(m.display_name)}</strong> (<code>${escapeHtml(m.repo)}</code>) from the Sparks. This also takes it off the menu.`;
|
||
const hostsEl = el('#dd-hosts');
|
||
hostsEl.innerHTML = '';
|
||
for (const h of (m.per_host || [])) {
|
||
const li = document.createElement('li');
|
||
li.innerHTML = `<code>${escapeHtml(h.host)}</code> — ${fmtBytesShort(h.size_bytes)}`;
|
||
hostsEl.appendChild(li);
|
||
}
|
||
const errEl = el('#dd-error');
|
||
errEl.classList.add('hidden');
|
||
errEl.textContent = '';
|
||
|
||
const confirm = el('#dd-confirm');
|
||
const cancel = el('#dd-cancel');
|
||
const onCancel = () => dlg.close();
|
||
const onConfirm = async () => {
|
||
confirm.disabled = true;
|
||
cancel.disabled = true;
|
||
confirm.textContent = 'Deleting…';
|
||
try {
|
||
const r = await fetchJSON(`/api/models/${encodeURIComponent(key)}/disk`, { method: 'DELETE' });
|
||
dlg.close();
|
||
// Optimistically drop the card, then re-scan the menu (it's gone from disk).
|
||
delete state.models[key];
|
||
renderCards();
|
||
await loadModels();
|
||
const freed = r && typeof r.bytes_freed === 'number' ? fmtBytesShort(r.bytes_freed) : '';
|
||
console.log(`Removed ${m.display_name} from disk${freed ? ` — freed ${freed}` : ''}.`);
|
||
} catch (e) {
|
||
errEl.textContent = e.message || 'Delete failed';
|
||
errEl.classList.remove('hidden');
|
||
} finally {
|
||
confirm.disabled = false;
|
||
cancel.disabled = false;
|
||
confirm.textContent = 'Remove from disk & menu';
|
||
}
|
||
};
|
||
cancel.onclick = onCancel;
|
||
confirm.onclick = onConfirm;
|
||
dlg.showModal();
|
||
}
|
||
|
||
async function triggerSwap(modelKey) {
|
||
if (state.swap_job_id) return;
|
||
if (state.lock && state.lock.held) {
|
||
const until = state.lock.expires_at ? ' until ' + fmtClock(state.lock.expires_at) : '';
|
||
alert(`The GPU swap path is reserved by ${state.lock.holder || 'automation'}${until}. Use "Release" on the reservation banner to override.`);
|
||
return;
|
||
}
|
||
try {
|
||
const r = await fetchJSON('/api/swap', {
|
||
method: 'POST',
|
||
headers: { 'content-type': 'application/json' },
|
||
body: JSON.stringify({ model_key: modelKey }),
|
||
});
|
||
attachToSwap(r.job_id, /*needsBackfill=*/false);
|
||
} catch (e) {
|
||
// 423 Locked: a reservation was acquired between our last poll and this click.
|
||
if (e.message && e.message.startsWith('423')) {
|
||
alert('The GPU swap path was just reserved by automation. Refreshing…');
|
||
pollCoordination();
|
||
} else {
|
||
alert('Failed to start swap: ' + e.message);
|
||
}
|
||
}
|
||
}
|
||
|
||
// ---- coordination layer: swap lock + schedule registry ----
|
||
|
||
async function pollCoordination() {
|
||
try {
|
||
state.lock = await fetchJSON('/api/swap/lock');
|
||
} catch { state.lock = { held: false }; }
|
||
try {
|
||
const r = await fetchJSON('/api/schedule');
|
||
state.schedules = r.schedules || [];
|
||
} catch { state.schedules = []; }
|
||
renderLockBanner();
|
||
renderSchedules();
|
||
renderCards(); // reflect lock state on the swap buttons
|
||
}
|
||
|
||
function renderLockBanner() {
|
||
const banner = el('#lock-banner');
|
||
if (!banner) return;
|
||
const lock = state.lock;
|
||
if (lock && lock.held) {
|
||
const until = lock.expires_at ? ` until ${fmtClock(lock.expires_at)}` : '';
|
||
const note = lock.note ? ` — ${escapeHtml(lock.note)}` : '';
|
||
el('#lock-text').innerHTML =
|
||
`GPU swap path reserved by <strong>${escapeHtml(lock.holder || 'automation')}</strong>${until}${note}. Manual swaps are paused.`;
|
||
banner.classList.remove('hidden');
|
||
} else {
|
||
banner.classList.add('hidden');
|
||
}
|
||
}
|
||
|
||
function renderSchedules() {
|
||
const panel = el('#schedule-panel');
|
||
const list = el('#schedule-list');
|
||
if (!panel || !list) return;
|
||
const items = state.schedules || [];
|
||
if (!items.length) {
|
||
panel.classList.add('hidden');
|
||
list.innerHTML = '';
|
||
return;
|
||
}
|
||
list.innerHTML = items.map((s) => {
|
||
const meta = [
|
||
s.cron ? `<code>${escapeHtml(s.cron)}</code>` : '',
|
||
s.next_run ? `next: ${escapeHtml(s.next_run)}` : '',
|
||
s.owner ? `by ${escapeHtml(s.owner)}` : '',
|
||
].filter(Boolean).join(' · ');
|
||
const desc = s.description ? `<div class="desc">${escapeHtml(s.description)}</div>` : '';
|
||
return `<div class="schedule-item">
|
||
<div class="name">${escapeHtml(s.name)}</div>
|
||
<div class="muted small">${meta}</div>
|
||
${desc}
|
||
</div>`;
|
||
}).join('');
|
||
panel.classList.remove('hidden');
|
||
}
|
||
|
||
async function releaseLock() {
|
||
const lock = state.lock || {};
|
||
const who = lock.holder || 'automation';
|
||
if (!confirm(`Force-release the GPU reservation held by ${who}? Any job relying on it may then collide with a manual swap.`)) return;
|
||
try {
|
||
await fetchJSON('/api/swap/lock?force=true', { method: 'DELETE' });
|
||
} catch (e) {
|
||
alert('Failed to release: ' + e.message);
|
||
}
|
||
pollCoordination();
|
||
}
|
||
|
||
async function attachToSwap(jobId, needsBackfill) {
|
||
if (state.swap_eventsource) {
|
||
state.swap_eventsource.close();
|
||
state.swap_eventsource = null;
|
||
}
|
||
state.swap_job_id = jobId;
|
||
state.swap_lines = [];
|
||
state.swap_phase = 'Starting…';
|
||
state.swap_phase_detail = '';
|
||
state.swap_progress = 0.05;
|
||
el('#swap-log').textContent = '';
|
||
el('#swap-panel').classList.remove('hidden');
|
||
renderSwapPanel();
|
||
|
||
// Backfill (if joining mid-swap) — fetch the snapshot so we have started_at + history
|
||
try {
|
||
const snap = await fetchJSON(`/api/swap/${jobId}`);
|
||
const ts = Date.parse(snap.started_at);
|
||
if (!isNaN(ts)) startTimer(ts);
|
||
state.swap_lines = snap.lines || [];
|
||
for (const line of state.swap_lines) appendLog(line);
|
||
deriveSwapPhase(snap.state, state.swap_lines);
|
||
renderSwapPanel();
|
||
if (snap.returncode !== null && snap.returncode !== undefined) {
|
||
// Already finished — close panel after a beat
|
||
handleSwapDone(snap);
|
||
return;
|
||
}
|
||
} catch (e) {
|
||
if (!needsBackfill) startTimer(Date.now());
|
||
console.warn('backfill failed', e);
|
||
}
|
||
|
||
const es = new EventSource(`/api/swap/${jobId}/stream`);
|
||
state.swap_eventsource = es;
|
||
es.onmessage = (ev) => {
|
||
try {
|
||
const d = JSON.parse(ev.data);
|
||
if (d.line !== undefined) {
|
||
state.swap_lines.push(d.line);
|
||
appendLog(d.line);
|
||
deriveSwapPhase(d.state, state.swap_lines);
|
||
renderSwapPanel();
|
||
} else if (d.state) {
|
||
deriveSwapPhase(d.state, state.swap_lines);
|
||
renderSwapPanel();
|
||
}
|
||
} catch {}
|
||
};
|
||
es.addEventListener('done', async (ev) => {
|
||
let d = {};
|
||
try { d = JSON.parse(ev.data); } catch {}
|
||
handleSwapDone(d);
|
||
});
|
||
es.onerror = () => {
|
||
// Tab backgrounded or network blip — close; status poll will reattach
|
||
es.close();
|
||
state.swap_eventsource = null;
|
||
};
|
||
renderCards();
|
||
}
|
||
|
||
function handleSwapDone(d) {
|
||
if (state.swap_eventsource) { state.swap_eventsource.close(); state.swap_eventsource = null; }
|
||
const finalState = d.state || 'ready';
|
||
state.swap_phase = finalState === 'failed' ? 'Failed' : 'Ready ✓';
|
||
state.swap_phase_detail = d.returncode !== undefined ? `exit code ${d.returncode}` : '';
|
||
state.swap_progress = 1.0;
|
||
renderSwapPanel();
|
||
setTimeout(() => detachSwap(), 4000);
|
||
pollStatus();
|
||
}
|
||
|
||
function detachSwap() {
|
||
state.swap_job_id = null;
|
||
if (state.swap_eventsource) { state.swap_eventsource.close(); state.swap_eventsource = null; }
|
||
stopTimer();
|
||
el('#swap-panel').classList.add('hidden');
|
||
renderCards();
|
||
}
|
||
|
||
function appendLog(line) {
|
||
const log = el('#swap-log');
|
||
log.textContent += line + '\n';
|
||
log.scrollTop = log.scrollHeight;
|
||
}
|
||
|
||
// ===================== model downloads =====================
|
||
|
||
const dlState = {
|
||
job_id: null,
|
||
eventsource: null,
|
||
started_at: null,
|
||
timer_handle: null,
|
||
};
|
||
|
||
function openDownloadForm() {
|
||
el('#download-panel').classList.remove('hidden');
|
||
el('#download-form').classList.remove('hidden');
|
||
el('#download-progress').classList.add('hidden');
|
||
el('#dl-repo').focus();
|
||
updateDlHfLink();
|
||
}
|
||
|
||
function updateDlHfLink() {
|
||
const repo = el('#dl-repo').value.trim();
|
||
const link = el('#dl-hf-link');
|
||
if (repo.includes('/')) {
|
||
link.href = `https://huggingface.co/${encodeURIComponent(repo)}`;
|
||
link.classList.remove('hidden');
|
||
} else {
|
||
link.classList.add('hidden');
|
||
}
|
||
}
|
||
|
||
function closeDownloadPanel() {
|
||
el('#download-panel').classList.add('hidden');
|
||
el('#download-form').classList.remove('hidden');
|
||
el('#download-progress').classList.add('hidden');
|
||
el('#dl-repo').value = '';
|
||
}
|
||
|
||
function dlTimerStart(startedAt) {
|
||
dlState.started_at = startedAt;
|
||
if (dlState.timer_handle) clearInterval(dlState.timer_handle);
|
||
const tick = () => {
|
||
if (!dlState.started_at) return;
|
||
const sec = Math.max(0, Math.floor((Date.now() - dlState.started_at) / 1000));
|
||
const m = Math.floor(sec / 60);
|
||
const s = sec % 60;
|
||
el('#dl-elapsed').textContent = `${m}:${s.toString().padStart(2, '0')}`;
|
||
};
|
||
tick();
|
||
dlState.timer_handle = setInterval(tick, 500);
|
||
}
|
||
|
||
function dlTimerStop() {
|
||
if (dlState.timer_handle) { clearInterval(dlState.timer_handle); dlState.timer_handle = null; }
|
||
}
|
||
|
||
async function startDownload() {
|
||
const repo = el('#dl-repo').value.trim();
|
||
const mode = document.querySelector('input[name="dl-mode"]:checked').value;
|
||
if (!repo || !repo.includes('/')) {
|
||
alert('Enter a HuggingFace repo in the form "org/name", e.g. RedHatAI/Qwen3.6-35B-A3B-NVFP4');
|
||
return;
|
||
}
|
||
dlState.last_repo = repo;
|
||
dlState.last_mode = mode;
|
||
try {
|
||
const r = await fetchJSON('/api/download', {
|
||
method: 'POST',
|
||
headers: { 'content-type': 'application/json' },
|
||
body: JSON.stringify({ repo, mode }),
|
||
});
|
||
attachToDownload(r.job_id);
|
||
} catch (e) {
|
||
alert('Failed to start download: ' + e.message);
|
||
}
|
||
}
|
||
|
||
function renderDownloadProgress(p) {
|
||
el('#dl-phase').textContent = p.phase || 'Working…';
|
||
const statsParts = [];
|
||
if (p.downloaded && p.total) statsParts.push(`${p.downloaded} / ${p.total}`);
|
||
if (p.rate) statsParts.push(p.rate);
|
||
if (p.eta) statsParts.push(`ETA ${p.eta}`);
|
||
el('#dl-stats').textContent = statsParts.join(' · ');
|
||
const pct = Math.max(2, Math.min(100, p.percent || 2));
|
||
el('#dl-progress-fill').style.width = `${pct}%`;
|
||
el('#dl-phase-detail').textContent = p.percent > 0 ? `${p.percent.toFixed(1)}%` : '';
|
||
}
|
||
|
||
function dlAppendLog(line) {
|
||
const log = el('#dl-log');
|
||
log.textContent += line + '\n';
|
||
log.scrollTop = log.scrollHeight;
|
||
}
|
||
|
||
async function attachToDownload(jobId) {
|
||
if (dlState.eventsource) { dlState.eventsource.close(); dlState.eventsource = null; }
|
||
dlState.job_id = jobId;
|
||
el('#download-form').classList.add('hidden');
|
||
el('#download-progress').classList.remove('hidden');
|
||
el('#dl-log').textContent = '';
|
||
el('#dl-title').textContent = 'Downloading…';
|
||
|
||
try {
|
||
const snap = await fetchJSON(`/api/download/${jobId}`);
|
||
dlTimerStart(Date.parse(snap.started_at));
|
||
for (const line of snap.lines || []) dlAppendLog(line);
|
||
renderDownloadProgress(snap.progress);
|
||
if (snap.returncode !== null && snap.returncode !== undefined) {
|
||
handleDownloadDone(snap);
|
||
return;
|
||
}
|
||
} catch (e) {
|
||
console.warn('download backfill failed', e);
|
||
dlTimerStart(Date.now());
|
||
}
|
||
|
||
const es = new EventSource(`/api/download/${jobId}/stream`);
|
||
dlState.eventsource = es;
|
||
es.onmessage = (ev) => {
|
||
try {
|
||
const d = JSON.parse(ev.data);
|
||
if (d.line !== undefined) dlAppendLog(d.line);
|
||
} catch {}
|
||
};
|
||
es.addEventListener('progress', (ev) => {
|
||
try { renderDownloadProgress(JSON.parse(ev.data)); } catch {}
|
||
});
|
||
es.addEventListener('done', (ev) => {
|
||
let d = {};
|
||
try { d = JSON.parse(ev.data); } catch {}
|
||
handleDownloadDone(d);
|
||
});
|
||
es.onerror = () => { es.close(); dlState.eventsource = null; };
|
||
}
|
||
|
||
function handleDownloadDone(d) {
|
||
if (dlState.eventsource) { dlState.eventsource.close(); dlState.eventsource = null; }
|
||
dlTimerStop();
|
||
if (d.state === 'failed') {
|
||
el('#dl-title').textContent = `Failed (rc=${d.returncode})`;
|
||
el('#dl-phase').textContent = 'Failed';
|
||
} else {
|
||
el('#dl-title').textContent = 'Done';
|
||
el('#dl-phase').textContent = 'Done ✓';
|
||
el('#dl-progress-fill').style.width = '100%';
|
||
// The new model now appears on the menu (the menu is the disk). If it matched
|
||
// a known recipe it's ready to switch to; if not, offer to set it up.
|
||
const repo = dlState.last_repo;
|
||
loadModels().then(() => {
|
||
if (!repo) return;
|
||
const entry = Object.values(state.models).find(m => m.repo === repo);
|
||
if (entry && entry.needs_setup) setTimeout(() => openSetupDialog(repo, { thenSwap: false }), 600);
|
||
});
|
||
}
|
||
dlState.job_id = null;
|
||
}
|
||
|
||
// ===================== Advanced / Add to catalog =====================
|
||
|
||
function gpuTotalGB(modelMode) {
|
||
// Solo uses Spark 1's GPU only. Cluster shares across both — but loading is per-Spark.
|
||
const s1 = state.hardware?.spark1;
|
||
const s2 = state.hardware?.spark2;
|
||
const g1 = s1?.gpu_mem_total_mib ? s1.gpu_mem_total_mib / 1024 : null;
|
||
const g2 = s2?.gpu_mem_total_mib ? s2.gpu_mem_total_mib / 1024 : null;
|
||
if (modelMode === 'cluster' && g1 && g2) return Math.min(g1, g2); // bottleneck
|
||
return g1 || g2 || null;
|
||
}
|
||
|
||
function knobContextHint(field, value, mode) {
|
||
if (field === 'gpu_memory_utilization') {
|
||
const gb = gpuTotalGB(mode);
|
||
if (!gb) return '';
|
||
const used = (value * gb).toFixed(0);
|
||
const free = (gb - value * gb).toFixed(0);
|
||
return `~${used} GB allocated to model + KV cache · ~${free} GB left for OS, buffers, other GPU workloads.`;
|
||
}
|
||
if (field === 'max_model_len') {
|
||
if (!value) return '';
|
||
const pages = Math.round(value / 350); // ~350 tokens per page
|
||
const kvBytes = (value * 2 * 4 * 32 * 128); // rough fp16 KV cache size for typical 32-layer model
|
||
return `~${pages.toLocaleString()} pages of text (very rough). Larger context = more GPU memory reserved for KV cache.`;
|
||
}
|
||
if (field === 'fastsafetensors') return value ? 'Faster cold-start weight loading.' : 'Standard safetensors loading.';
|
||
if (field === 'prefix_caching') return value ? 'Reuses GPU state for repeated prompt prefixes (e.g. long system prompts).' : 'Off — every request re-processes the full prompt.';
|
||
if (field === 'kv_cache_dtype') return value === 'fp8' ? 'Halves KV cache memory (fits ~2× more context). Quality cost is usually imperceptible.' : 'Default precision.';
|
||
return '';
|
||
}
|
||
|
||
function ensureKnobHint(rowEl, id) {
|
||
let h = rowEl.querySelector(`.knob-hint[data-for="${id}"]`);
|
||
if (!h) {
|
||
h = document.createElement('div');
|
||
h.className = 'knob-hint muted small';
|
||
h.dataset.for = id;
|
||
rowEl.appendChild(h);
|
||
}
|
||
return h;
|
||
}
|
||
|
||
function openAdvanced(key) {
|
||
const m = state.models[key];
|
||
if (!m) return;
|
||
const dlg = el('#advanced-dialog');
|
||
el('#adv-title').textContent = `Advanced — ${m.display_name}`;
|
||
const k = m.effective_knobs || {};
|
||
el('#adv-mml').value = k.max_model_len ?? '';
|
||
el('#adv-gmu').value = k.gpu_memory_utilization ?? 0.85;
|
||
el('#adv-gmu-out').value = parseFloat(el('#adv-gmu').value).toFixed(2);
|
||
el('#adv-fst').checked = !!k.fastsafetensors;
|
||
el('#adv-pcache').checked = !!k.prefix_caching;
|
||
el('#adv-fp8').checked = k.kv_cache_dtype === 'fp8';
|
||
|
||
// Wire up live knob hints
|
||
const updateHints = () => {
|
||
const mml = parseInt(el('#adv-mml').value, 10);
|
||
const gmu = parseFloat(el('#adv-gmu').value);
|
||
ensureKnobHint(el('#adv-mml').parentElement, 'mml').textContent = knobContextHint('max_model_len', mml, m.mode);
|
||
ensureKnobHint(el('#adv-gmu').parentElement, 'gmu').textContent = knobContextHint('gpu_memory_utilization', gmu, m.mode);
|
||
ensureKnobHint(el('#adv-fst').parentElement, 'fst').textContent = knobContextHint('fastsafetensors', el('#adv-fst').checked, m.mode);
|
||
ensureKnobHint(el('#adv-pcache').parentElement, 'pcache').textContent = knobContextHint('prefix_caching', el('#adv-pcache').checked, m.mode);
|
||
ensureKnobHint(el('#adv-fp8').parentElement, 'fp8').textContent = knobContextHint('kv_cache_dtype', el('#adv-fp8').checked ? 'fp8' : 'auto', m.mode);
|
||
};
|
||
updateHints();
|
||
el('#adv-mml').oninput = updateHints;
|
||
el('#adv-gmu').oninput = (e) => { el('#adv-gmu-out').value = parseFloat(e.target.value).toFixed(2); updateHints(); };
|
||
el('#adv-fst').onchange = updateHints;
|
||
el('#adv-pcache').onchange = updateHints;
|
||
el('#adv-fp8').onchange = updateHints;
|
||
const del = el('#adv-delete');
|
||
del.classList.toggle('hidden', !m.custom);
|
||
del.onclick = async () => {
|
||
if (!confirm(`Delete "${m.display_name}" from the catalog? The model weights on disk are NOT deleted.`)) return;
|
||
try {
|
||
await fetchJSON(`/api/models/${encodeURIComponent(key)}`, { method: 'DELETE' });
|
||
dlg.close();
|
||
await loadModels();
|
||
pollStatus();
|
||
} catch (e) { alert('Delete failed: ' + e.message); }
|
||
};
|
||
const form = el('#advanced-form');
|
||
form.onsubmit = async (e) => {
|
||
e.preventDefault();
|
||
const knobs = {};
|
||
const mml = parseInt(el('#adv-mml').value, 10);
|
||
if (Number.isFinite(mml) && mml > 0) knobs.max_model_len = mml;
|
||
const gmu = parseFloat(el('#adv-gmu').value);
|
||
if (Number.isFinite(gmu)) knobs.gpu_memory_utilization = gmu;
|
||
if (el('#adv-fst').checked) knobs.fastsafetensors = true; else knobs.fastsafetensors = false;
|
||
if (el('#adv-pcache').checked) knobs.prefix_caching = true; else knobs.prefix_caching = false;
|
||
knobs.kv_cache_dtype = el('#adv-fp8').checked ? 'fp8' : 'auto';
|
||
try {
|
||
await fetchJSON(`/api/models/${encodeURIComponent(key)}/knobs`, {
|
||
method: 'PUT',
|
||
headers: { 'content-type': 'application/json' },
|
||
body: JSON.stringify({ knobs }),
|
||
});
|
||
dlg.close();
|
||
await loadModels();
|
||
pollStatus();
|
||
} catch (e) { alert('Save failed: ' + e.message); }
|
||
};
|
||
dlg.showModal();
|
||
}
|
||
|
||
// Context carried from openSetupDialog -> the submit handler: the inferred
|
||
// launch flags (parsers/MoE backend) and whether to swap right after saving.
|
||
let setupCtx = { key: '', repo: '', vllm_args: [], thenSwap: false };
|
||
|
||
// "Set up & switch" on a needs-setup card.
|
||
async function openSetupForKey(key) {
|
||
const m = state.models[key];
|
||
if (!m) return;
|
||
if (state.lock && state.lock.held) {
|
||
const until = state.lock.expires_at ? ' until ' + fmtClock(state.lock.expires_at) : '';
|
||
alert(`The GPU swap path is reserved by ${state.lock.holder || 'automation'}${until}. Use "Release" on the reservation banner to override.`);
|
||
return;
|
||
}
|
||
await openSetupDialog(m.repo, { thenSwap: true });
|
||
}
|
||
|
||
// Open the "set up this model" dialog, prefilled from inference (config.json +
|
||
// size). The operator confirms once; on save the recipe persists and (if
|
||
// thenSwap) we switch to it.
|
||
async function openSetupDialog(repo, opts = {}) {
|
||
const dlg = el('#catalog-dialog');
|
||
let sug = null;
|
||
try {
|
||
sug = await fetchJSON(`/api/models/suggest?repo=${encodeURIComponent(repo)}`);
|
||
} catch (e) {
|
||
console.warn('recipe suggestion failed:', e.message);
|
||
}
|
||
const fallbackKey = repo.toLowerCase().replace(/[^a-z0-9_-]+/g, '-').replace(/^-+|-+$/g, '');
|
||
setupCtx = {
|
||
key: (sug && sug.key) || fallbackKey,
|
||
repo,
|
||
vllm_args: (sug && sug.vllm_args) || [],
|
||
thenSwap: !!opts.thenSwap,
|
||
};
|
||
el('#cd-key').value = setupCtx.key;
|
||
el('#cd-name').value = (sug && sug.display_name) || repo.split('/').pop();
|
||
el('#cd-repo').value = repo;
|
||
el('#cd-size').value = '';
|
||
el('#cd-mode').value = (sug && sug.mode) || 'solo';
|
||
el('#cd-desc').value = '';
|
||
const knobs = (sug && sug.knobs) || {};
|
||
el('#cd-mml').value = knobs.max_model_len || 32768;
|
||
el('#cd-gmu').value = knobs.gpu_memory_utilization || 0.85;
|
||
el('#cd-gmu-out').value = parseFloat(el('#cd-gmu').value).toFixed(2);
|
||
el('#cd-fst').checked = knobs.fastsafetensors !== false;
|
||
el('#cd-pcache').checked = knobs.prefix_caching !== false;
|
||
el('#cd-fp8').checked = (knobs.kv_cache_dtype || 'fp8') === 'fp8';
|
||
|
||
const det = el('#cd-detected');
|
||
if (det) {
|
||
if (sug) {
|
||
const caps = (sug.capabilities || []).join(', ');
|
||
const flags = setupCtx.vllm_args.length ? `: <code>${escapeHtml(setupCtx.vllm_args.join(' '))}</code>` : '';
|
||
det.innerHTML = `Detected <strong>${escapeHtml(sug.family || 'Generic')}</strong>${caps ? ` · ${escapeHtml(caps)}` : ''}. Launch flags set automatically${flags}.`;
|
||
} else {
|
||
det.textContent = "Couldn't auto-detect this model's settings — pick mode and knobs manually.";
|
||
}
|
||
det.classList.remove('hidden');
|
||
}
|
||
const submit = el('#cd-submit');
|
||
if (submit) submit.textContent = setupCtx.thenSwap ? 'Save & switch' : 'Save settings';
|
||
dlg.showModal();
|
||
}
|
||
|
||
function setupCatalogDialog() {
|
||
el('#cd-cancel').addEventListener('click', () => el('#catalog-dialog').close());
|
||
el('#cd-gmu').addEventListener('input', (e) => { el('#cd-gmu-out').value = parseFloat(e.target.value).toFixed(2); });
|
||
el('#catalog-form').addEventListener('submit', async (e) => {
|
||
e.preventDefault();
|
||
const body = {
|
||
key: el('#cd-key').value.trim() || setupCtx.key,
|
||
display_name: el('#cd-name').value.trim(),
|
||
repo: el('#cd-repo').value.trim(),
|
||
size_gb: parseFloat(el('#cd-size').value) || 0,
|
||
mode: el('#cd-mode').value,
|
||
description: el('#cd-desc').value.trim() || null,
|
||
// The inferred family flags (parsers / MoE backend); knob-controlled flags
|
||
// are layered on by the server from `knobs`, so no duplication.
|
||
vllm_args: setupCtx.vllm_args || [],
|
||
knobs: {
|
||
max_model_len: parseInt(el('#cd-mml').value, 10) || 32768,
|
||
gpu_memory_utilization: parseFloat(el('#cd-gmu').value),
|
||
fastsafetensors: el('#cd-fst').checked,
|
||
prefix_caching: el('#cd-pcache').checked,
|
||
kv_cache_dtype: el('#cd-fp8').checked ? 'fp8' : 'auto',
|
||
},
|
||
};
|
||
try {
|
||
await fetchJSON('/api/models', {
|
||
method: 'POST',
|
||
headers: { 'content-type': 'application/json' },
|
||
body: JSON.stringify(body),
|
||
});
|
||
el('#catalog-dialog').close();
|
||
closeDownloadPanel();
|
||
await loadModels();
|
||
if (setupCtx.thenSwap) triggerSwap(body.key);
|
||
pollStatus();
|
||
} catch (e) { alert('Saving the model setup failed: ' + e.message); }
|
||
});
|
||
}
|
||
|
||
function setupAdvancedDialog() {
|
||
el('#adv-cancel').addEventListener('click', () => el('#advanced-dialog').close());
|
||
el('#adv-gmu').addEventListener('input', (e) => { el('#adv-gmu-out').value = parseFloat(e.target.value).toFixed(2); });
|
||
}
|
||
|
||
function openLocalModelDialog() {
|
||
const dlg = el('#local-model-dialog');
|
||
el('#lm-key').value = '';
|
||
el('#lm-name').value = '';
|
||
el('#lm-path').value = '';
|
||
el('#lm-chat').value = '';
|
||
el('#lm-size').value = '';
|
||
el('#lm-mode').value = 'solo';
|
||
el('#lm-desc').value = '';
|
||
el('#lm-mml').value = 32768;
|
||
el('#lm-gmu').value = 0.85;
|
||
el('#lm-gmu-out').value = '0.85';
|
||
el('#lm-fst').checked = true;
|
||
el('#lm-pcache').checked = true;
|
||
el('#lm-fp8').checked = true;
|
||
dlg.showModal();
|
||
}
|
||
|
||
function setupLocalModelDialog() {
|
||
el('#lm-cancel').addEventListener('click', () => el('#local-model-dialog').close());
|
||
el('#lm-gmu').addEventListener('input', (e) => { el('#lm-gmu-out').value = parseFloat(e.target.value).toFixed(2); });
|
||
el('#local-model-form').addEventListener('submit', async (e) => {
|
||
e.preventDefault();
|
||
const chat = el('#lm-chat').value.trim();
|
||
const body = {
|
||
key: el('#lm-key').value.trim(),
|
||
display_name: el('#lm-name').value.trim(),
|
||
local_path: el('#lm-path').value.trim(),
|
||
size_gb: parseFloat(el('#lm-size').value) || 0,
|
||
mode: el('#lm-mode').value,
|
||
description: el('#lm-desc').value.trim() || null,
|
||
// A fine-tune's chat template (if any) rides along as a launch flag.
|
||
vllm_args: chat ? [`--chat-template=${chat}`] : [],
|
||
knobs: {
|
||
max_model_len: parseInt(el('#lm-mml').value, 10) || 32768,
|
||
gpu_memory_utilization: parseFloat(el('#lm-gmu').value),
|
||
fastsafetensors: el('#lm-fst').checked,
|
||
prefix_caching: el('#lm-pcache').checked,
|
||
kv_cache_dtype: el('#lm-fp8').checked ? 'fp8' : 'auto',
|
||
},
|
||
};
|
||
try {
|
||
await fetchJSON('/api/models', {
|
||
method: 'POST',
|
||
headers: { 'content-type': 'application/json' },
|
||
body: JSON.stringify(body),
|
||
});
|
||
el('#local-model-dialog').close();
|
||
await loadModels();
|
||
pollStatus();
|
||
} catch (e) { alert('Add local model failed: ' + e.message); }
|
||
});
|
||
}
|
||
|
||
// ===================== NIM installer =====================
|
||
|
||
const nimState = {
|
||
catalog: null,
|
||
job_id: null,
|
||
eventsource: null,
|
||
timer: null,
|
||
started_at: null,
|
||
};
|
||
|
||
async function loadNimCatalog() {
|
||
try {
|
||
nimState.catalog = await fetchJSON('/api/nim/catalog');
|
||
el('#nim-catalog-link').href = nimState.catalog.catalog_url;
|
||
const warn = el('#nim-key-warn');
|
||
if (!nimState.catalog.ngc_key_configured) {
|
||
warn.classList.add('nim-key-warn');
|
||
warn.innerHTML = '⚠️ NGC API key not set. Open <strong>Configure Sparks</strong> in StartOS and paste your NGC personal API key, otherwise installs will fail. <a href="https://ngc.nvidia.com/setup/personal-key" target="_blank" rel="noopener">Get a key</a>';
|
||
} else {
|
||
warn.classList.remove('nim-key-warn');
|
||
warn.textContent = '';
|
||
}
|
||
const grid = el('#nim-suggested');
|
||
grid.innerHTML = '';
|
||
for (const s of nimState.catalog.suggested || []) {
|
||
const card = document.createElement('div');
|
||
card.className = 'nim-card';
|
||
card.innerHTML = `
|
||
<div class="info">
|
||
<div class="name">${escapeHtml(s.name)} <span class="muted small">· ${escapeHtml(s.kind || 'nim')}</span></div>
|
||
<div class="desc">${escapeHtml(s.description || '')}</div>
|
||
<div class="img">${escapeHtml(s.image)}</div>
|
||
<div class="links">${s.homepage ? `<a href="${escapeHtml(s.homepage)}" target="_blank" rel="noopener">View on NGC ↗</a>` : ''}</div>
|
||
</div>
|
||
<button type="button" class="btn primary nim-pick" data-image="${escapeHtml(s.image)}" data-container="${escapeHtml(s.default_container)}" data-port="${s.default_port}" data-kind="${escapeHtml(s.kind)}">Pick</button>
|
||
`;
|
||
grid.appendChild(card);
|
||
}
|
||
grid.querySelectorAll('.nim-pick').forEach(btn => {
|
||
btn.addEventListener('click', () => {
|
||
el('#nim-image').value = btn.dataset.image;
|
||
el('#nim-container').value = btn.dataset.container;
|
||
el('#nim-port').value = btn.dataset.port;
|
||
el('#nim-kind').value = btn.dataset.kind || 'nim';
|
||
});
|
||
});
|
||
} catch (e) { console.warn('nim catalog failed', e); }
|
||
}
|
||
|
||
function openNimDialog() {
|
||
loadNimCatalog();
|
||
el('#nim-dialog').showModal();
|
||
}
|
||
|
||
async function submitNim(e) {
|
||
e.preventDefault();
|
||
const body = {
|
||
image: el('#nim-image').value.trim(),
|
||
container: el('#nim-container').value.trim(),
|
||
port: parseInt(el('#nim-port').value, 10),
|
||
host: el('#nim-host').value,
|
||
kind: el('#nim-kind').value,
|
||
};
|
||
if (!body.image || !body.container || !body.port) {
|
||
alert('Image, container name, and port are required.');
|
||
return;
|
||
}
|
||
try {
|
||
const r = await fetchJSON('/api/nim/install', {
|
||
method: 'POST',
|
||
headers: { 'content-type': 'application/json' },
|
||
body: JSON.stringify(body),
|
||
});
|
||
el('#nim-dialog').close();
|
||
attachNimProgress(r.job_id);
|
||
} catch (e) {
|
||
alert('Install failed: ' + e.message);
|
||
}
|
||
}
|
||
|
||
function nimTimerStart(at) {
|
||
nimState.started_at = at;
|
||
if (nimState.timer) clearInterval(nimState.timer);
|
||
const tick = () => {
|
||
if (!nimState.started_at) return;
|
||
const sec = Math.max(0, Math.floor((Date.now() - nimState.started_at) / 1000));
|
||
const m = Math.floor(sec / 60);
|
||
const s = sec % 60;
|
||
el('#nim-prog-elapsed').textContent = `${m}:${s.toString().padStart(2, '0')}`;
|
||
};
|
||
tick();
|
||
nimState.timer = setInterval(tick, 500);
|
||
}
|
||
|
||
async function attachNimProgress(jobId) {
|
||
nimState.job_id = jobId;
|
||
el('#nim-prog-log').textContent = '';
|
||
el('#nim-prog-title').textContent = 'Installing…';
|
||
el('#nim-progress-dialog').showModal();
|
||
try {
|
||
const snap = await fetchJSON(`/api/nim/install/${jobId}`);
|
||
nimTimerStart(Date.parse(snap.started_at));
|
||
el('#nim-prog-phase').textContent = snap.phase || 'Working…';
|
||
el('#nim-prog-log').textContent = (snap.lines || []).join('\n');
|
||
if (snap.returncode !== null) { onNimDone(snap); return; }
|
||
} catch { nimTimerStart(Date.now()); }
|
||
const es = new EventSource(`/api/nim/install/${jobId}/stream`);
|
||
nimState.eventsource = es;
|
||
es.onmessage = ev => {
|
||
try {
|
||
const d = JSON.parse(ev.data);
|
||
if (d.line !== undefined) {
|
||
const log = el('#nim-prog-log');
|
||
log.textContent += d.line + '\n';
|
||
log.scrollTop = log.scrollHeight;
|
||
}
|
||
} catch {}
|
||
};
|
||
es.addEventListener('phase', ev => {
|
||
try { el('#nim-prog-phase').textContent = JSON.parse(ev.data).phase; } catch {}
|
||
});
|
||
es.addEventListener('done', ev => {
|
||
let d = {}; try { d = JSON.parse(ev.data); } catch {}
|
||
onNimDone(d);
|
||
});
|
||
es.onerror = () => { es.close(); nimState.eventsource = null; };
|
||
}
|
||
|
||
function onNimDone(d) {
|
||
if (nimState.eventsource) { nimState.eventsource.close(); nimState.eventsource = null; }
|
||
if (nimState.timer) { clearInterval(nimState.timer); nimState.timer = null; }
|
||
if (d.state === 'failed') {
|
||
el('#nim-prog-title').textContent = `Failed (rc=${d.returncode})`;
|
||
el('#nim-prog-phase').textContent = 'Failed';
|
||
} else {
|
||
el('#nim-prog-title').textContent = 'Installed';
|
||
el('#nim-prog-phase').textContent = 'Done ✓ — service will appear when the container reports healthy.';
|
||
}
|
||
pollStatus();
|
||
}
|
||
|
||
// ===================== Explain context (LLM commit summary) =====================
|
||
|
||
let explainEventSource = null;
|
||
|
||
async function explainContext() {
|
||
if (explainEventSource) { explainEventSource.close(); explainEventSource = null; }
|
||
const section = el('#ub-explain-section');
|
||
const content = el('#ub-explain-content');
|
||
section.classList.remove('hidden');
|
||
section.open = true;
|
||
content.innerHTML = '<span class="muted">Asking the loaded model…</span>';
|
||
let text = '';
|
||
const es = new EventSource('/api/explain-updates');
|
||
explainEventSource = es;
|
||
let firstChunk = true;
|
||
es.onmessage = (ev) => {
|
||
try {
|
||
const d = JSON.parse(ev.data);
|
||
if (d.error) {
|
||
content.innerHTML = `<span class="muted">Couldn't get explanation: ${escapeHtml(d.error)}</span>`;
|
||
return;
|
||
}
|
||
if (firstChunk) { content.innerHTML = ''; firstChunk = false; }
|
||
if (d.content) {
|
||
text += d.content;
|
||
content.textContent = text;
|
||
content.scrollTop = content.scrollHeight;
|
||
} else if (d.reasoning) {
|
||
// Show reasoning tokens but de-emphasized
|
||
let r = content.querySelector('.reasoning-current');
|
||
if (!r) {
|
||
r = document.createElement('div');
|
||
r.className = 'reasoning reasoning-current';
|
||
r.textContent = '';
|
||
content.appendChild(r);
|
||
}
|
||
r.textContent += d.reasoning;
|
||
}
|
||
} catch {}
|
||
};
|
||
es.addEventListener('done', () => {
|
||
es.close();
|
||
explainEventSource = null;
|
||
// strip the reasoning-current marker
|
||
const r = content.querySelector('.reasoning-current');
|
||
if (r) r.classList.remove('reasoning-current');
|
||
});
|
||
es.onerror = () => { es.close(); explainEventSource = null; };
|
||
}
|
||
|
||
// ===================== updates (spark-vllm-docker) =====================
|
||
|
||
const updState = {
|
||
info: null,
|
||
job_id: null,
|
||
eventsource: null,
|
||
started_at: null,
|
||
timer_handle: null,
|
||
};
|
||
|
||
async function pollUpdates() {
|
||
try {
|
||
const info = await fetchJSON('/api/updates');
|
||
updState.info = info;
|
||
renderUpdateBanner();
|
||
} catch (e) {
|
||
console.warn('updates poll failed', e);
|
||
}
|
||
}
|
||
|
||
function renderUpdateBanner() {
|
||
const banner = el('#update-banner');
|
||
const info = updState.info;
|
||
const text = el('#ub-text');
|
||
const details = el('#ub-details');
|
||
const apply = el('#ub-apply');
|
||
const list = el('#ub-list');
|
||
const log = el('#ub-log');
|
||
|
||
if (!info || !info.ok) {
|
||
banner.classList.add('hidden');
|
||
return;
|
||
}
|
||
banner.classList.remove('hidden');
|
||
const behind = info.behind || 0;
|
||
const dirty = info.dirty || 0;
|
||
banner.classList.toggle('up-to-date', behind === 0 && !dirty);
|
||
banner.classList.toggle('warn', !!dirty);
|
||
|
||
const explain = el('#ub-explain');
|
||
if (dirty > 0) {
|
||
text.textContent = `${dirty} local change${dirty === 1 ? '' : 's'} in ~/spark-vllm-docker. Resolve before updating.`;
|
||
details.classList.add('hidden');
|
||
apply.classList.add('hidden');
|
||
explain.classList.add('hidden');
|
||
} else if (behind === 0) {
|
||
text.textContent = `spark-vllm-docker is up to date (${info.current || ''})`;
|
||
details.classList.add('hidden');
|
||
apply.classList.add('hidden');
|
||
list.classList.add('hidden');
|
||
explain.classList.add('hidden');
|
||
} else {
|
||
text.textContent = `${behind} commit${behind === 1 ? '' : 's'} behind upstream`;
|
||
details.classList.remove('hidden');
|
||
apply.classList.remove('hidden');
|
||
explain.classList.remove('hidden');
|
||
log.textContent = (info.log || []).join('\n') || '(no log)';
|
||
}
|
||
}
|
||
|
||
function ubTimerStart(startedAt) {
|
||
updState.started_at = startedAt;
|
||
if (updState.timer_handle) clearInterval(updState.timer_handle);
|
||
const tick = () => {
|
||
if (!updState.started_at) return;
|
||
const sec = Math.max(0, Math.floor((Date.now() - updState.started_at) / 1000));
|
||
const m = Math.floor(sec / 60);
|
||
const s = sec % 60;
|
||
el('#ub-elapsed').textContent = `${m}:${s.toString().padStart(2, '0')}`;
|
||
};
|
||
tick();
|
||
updState.timer_handle = setInterval(tick, 500);
|
||
}
|
||
|
||
async function applyUpdate() {
|
||
if (!confirm('This pulls the latest spark-vllm-docker and rebuilds the vLLM container. Can take 5–40 minutes; the cluster is unaffected until you swap to a different model. Continue?')) return;
|
||
try {
|
||
const r = await fetchJSON('/api/updates/apply', {
|
||
method: 'POST',
|
||
headers: { 'content-type': 'application/json' },
|
||
body: JSON.stringify({ mode: 'cluster' }),
|
||
});
|
||
attachToUpdate(r.job_id);
|
||
} catch (e) {
|
||
alert('Failed to start update: ' + e.message);
|
||
}
|
||
}
|
||
|
||
async function attachToUpdate(jobId) {
|
||
updState.job_id = jobId;
|
||
el('#ub-progress').classList.remove('hidden');
|
||
el('#ub-apply').classList.add('hidden');
|
||
el('#ub-stream').textContent = '';
|
||
el('#ub-phase').textContent = 'Starting…';
|
||
try {
|
||
const snap = await fetchJSON(`/api/updates/${jobId}`);
|
||
ubTimerStart(Date.parse(snap.started_at));
|
||
el('#ub-phase').textContent = snap.phase || 'Working…';
|
||
el('#ub-stream').textContent = (snap.lines || []).join('\n');
|
||
if (snap.returncode !== null) { handleUpdateDone(snap); return; }
|
||
} catch (e) {
|
||
ubTimerStart(Date.now());
|
||
}
|
||
const es = new EventSource(`/api/updates/${jobId}/stream`);
|
||
updState.eventsource = es;
|
||
es.onmessage = (ev) => {
|
||
try {
|
||
const d = JSON.parse(ev.data);
|
||
if (d.line !== undefined) {
|
||
const log = el('#ub-stream');
|
||
log.textContent += d.line + '\n';
|
||
log.scrollTop = log.scrollHeight;
|
||
}
|
||
} catch {}
|
||
};
|
||
es.addEventListener('phase', (ev) => {
|
||
try { el('#ub-phase').textContent = JSON.parse(ev.data).phase; } catch {}
|
||
});
|
||
es.addEventListener('done', (ev) => {
|
||
let d = {}; try { d = JSON.parse(ev.data); } catch {}
|
||
handleUpdateDone(d);
|
||
});
|
||
es.onerror = () => { es.close(); updState.eventsource = null; };
|
||
}
|
||
|
||
function handleUpdateDone(d) {
|
||
if (updState.eventsource) { updState.eventsource.close(); updState.eventsource = null; }
|
||
if (updState.timer_handle) { clearInterval(updState.timer_handle); updState.timer_handle = null; }
|
||
el('#ub-phase').textContent = d.state === 'failed' ? `Failed (rc=${d.returncode})` : 'Done ✓ — re-check from the banner.';
|
||
setTimeout(pollUpdates, 2000);
|
||
}
|
||
|
||
async function init() {
|
||
setupCopyButtons();
|
||
el('#open-download').addEventListener('click', openDownloadForm);
|
||
el('#dl-cancel').addEventListener('click', closeDownloadPanel);
|
||
el('#dl-start').addEventListener('click', startDownload);
|
||
el('#dl-repo').addEventListener('keydown', (e) => { if (e.key === 'Enter') startDownload(); });
|
||
el('#ub-details').addEventListener('click', () => {
|
||
const list = el('#ub-list');
|
||
list.classList.toggle('hidden');
|
||
list.open = !list.open;
|
||
});
|
||
el('#ub-apply').addEventListener('click', applyUpdate);
|
||
el('#ub-explain').addEventListener('click', explainContext);
|
||
el('#dl-repo').addEventListener('input', updateDlHfLink);
|
||
el('#open-nim').addEventListener('click', openNimDialog);
|
||
el('#nim-cancel').addEventListener('click', () => el('#nim-dialog').close());
|
||
el('#nim-form').addEventListener('submit', submitNim);
|
||
el('#nim-prog-close').addEventListener('click', () => el('#nim-progress-dialog').close());
|
||
el('#mb-update-close').addEventListener('click', () => el('#mb-update-dialog').close());
|
||
// Dismissing the modal (Close or Esc) stops streaming; the job runs on
|
||
// server-side and re-clicking Update returns a 409 if still in progress.
|
||
el('#mb-update-dialog').addEventListener('close', () => {
|
||
if (mbState.eventsource) { mbState.eventsource.close(); mbState.eventsource = null; }
|
||
if (mbState.timer) { clearInterval(mbState.timer); mbState.timer = null; }
|
||
state.mb_update_in_flight = false;
|
||
renderServices();
|
||
});
|
||
el('#mb-logs-close').addEventListener('click', () => el('#mb-logs-dialog').close());
|
||
el('#mb-logs-refresh').addEventListener('click', loadMatrixBridgeLogs);
|
||
el('#open-connectivity').addEventListener('click', openConnectivityDialog);
|
||
el('#connectivity-close').addEventListener('click', () => el('#connectivity-dialog').close());
|
||
// Hardware-card buttons (Wake-on-LAN on unreachable cards; SSH-key copy on
|
||
// reachable ones) are rendered dynamically, so delegate from the grid.
|
||
el('#hardware-grid').addEventListener('click', (e) => {
|
||
const wbtn = e.target.closest('[data-wake]');
|
||
if (wbtn) { wakeSpark(wbtn.dataset.wake); return; }
|
||
const kbtn = e.target.closest('[data-ssh-key]');
|
||
if (kbtn) { copySparkSshKey(kbtn.dataset.sshKey, kbtn); return; }
|
||
});
|
||
el('#sshkey-close').addEventListener('click', () => el('#sshkey-dialog').close());
|
||
el('#open-local').addEventListener('click', openLocalModelDialog);
|
||
el('#lock-release').addEventListener('click', releaseLock);
|
||
setupCatalogDialog();
|
||
setupAdvancedDialog();
|
||
setupLocalModelDialog();
|
||
// Open WebUI link from /api/config
|
||
try {
|
||
state.config = await fetchJSON('/api/config');
|
||
if (state.config.open_webui_url) {
|
||
const a = el('#open-webui-link');
|
||
a.href = state.config.open_webui_url;
|
||
a.classList.remove('hidden');
|
||
}
|
||
} catch {}
|
||
setupDashboardTabs();
|
||
setupEndpointCollapse();
|
||
// Fire the (SSH-backed) menu scan without awaiting — it self-renders a
|
||
// "Scanning…" state and fills in when it returns, so a slow/unreachable
|
||
// cluster never blocks first paint. pollStatus() below paints the rest.
|
||
loadModels();
|
||
await pollStatus();
|
||
await renderServices();
|
||
pollCoordination();
|
||
pollHardware();
|
||
pollUpdates();
|
||
// Speech-model patches panel — slow over SSH, runs after first paint.
|
||
renderSpeechModels();
|
||
setInterval(pollStatus, 5000);
|
||
setInterval(pollCoordination, 5000); // swap lock + schedule registry
|
||
setInterval(pollHardware, 8000); // every 8s
|
||
setInterval(pollUpdates, 300000); // every 5 min
|
||
setInterval(loadModels, 60000); // every 60s — re-scan the Sparks for added/removed models
|
||
setInterval(renderSpeechModels, 120000); // every 2 min — patches change rarely
|
||
}
|
||
|
||
init();
|