Ship v0.1.1–v0.1.5: first-live-run fixes and dashboard viewer

- v0.1.1: config persistence — FileHelper paths made absolute
  (/media/startos/volumes/main/...); relative paths resolved into the JS
  runtime's ephemeral cwd so action saves never reached /data
- v0.1.2: preJobStopContainers (Configure Grading) — docker-stop resident
  vLLM containers on the head Spark at job start, no auto-restart
- v0.1.3: preflight auth (LiteLLM master_key gates /models),
  poll-until-loaded, crash fast-fail (restarting counts as dead)
- v0.1.4: HF_HUB_OFFLINE/TRANSFORMERS_OFFLINE in airgapped serving
  (--internal network has no DNS); grader _post timeout 600→1800s for
  ~3.6 tok/s GB10 generation
- v0.1.5: dashboard viewer survives the periodic background refresh;
  download buttons for deck reports, deck JSON, and SCORECARD.md
- .gitignore: .startos/ build workspace, start-technologies/

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Jonathan Kirkwood
2026-07-30 09:14:24 -05:00
co-authored by Claude Fable 5
parent 1d1074b625
commit 91212322c1
16 changed files with 262 additions and 46 deletions
+2
View File
@@ -5,3 +5,5 @@ javascript/
**/__pycache__/
*.pyc
.claude/
.startos/
start-technologies/
+4
View File
@@ -69,6 +69,10 @@ CONFIG_DEFAULTS = {
"wipeRemoteDocs": True,
"autoRunOnDrop": False,
"networkName": "boardroom-net",
# Space/comma-separated docker container names stopped on the head Spark at
# the start of every job to free GPU memory (e.g. an always-on vLLM another
# service runs). NOT restarted afterwards — their owner reloads them.
"preJobStopContainers": "",
# Portfolio companies (authoritative source of pinned targets / aliases).
# pinnedTargets: [{kpi, target, unit, direction: gte|lte, profitability}]
# kpiAliases: newline-separated "canonical=alias1;alias2" lines.
+1
View File
@@ -305,6 +305,7 @@ class JobRunner:
"primary Spark or switch to local-services mode.")
# 3. Infra once per job.
serving.clear_resident_containers(cfg, self.log)
gr_mod.ensure_grader_image(cfg, self.log)
serving.ensure_network(cfg, self.log)
preflight.check_searxng(cfg, self.log)
+72 -17
View File
@@ -9,6 +9,7 @@ container attached to the same network.
from __future__ import annotations
import shlex
import time
import spark_client as sc
import serving
@@ -27,40 +28,94 @@ def _probe_in_net(cfg: dict, inner_cmd: str, timeout: int) -> sc.subprocess.Comp
return sc.run(sc.head(cfg), cmd, timeout=timeout)
def check_wave(cfg: dict, wave: list[dict], log) -> None:
"""The proxy answers AND each model alias in the wave returns a completion."""
def _container_running(cfg: dict, name: str) -> bool:
"""True only for a stably running container — a crash-looping one under
`--restart unless-stopped` reports Status "running"/"restarting" between
crashes, so treat repeated restarts as dead."""
cmd = ("docker inspect -f '{{.State.Status}} {{.RestartCount}}' "
+ shlex.quote(name) + " 2>/dev/null")
r = sc.run(sc.head(cfg), cmd, timeout=30)
parts = (r.stdout or "").strip().split()
if len(parts) != 2:
return False
status, restarts = parts[0], parts[1]
if status != "running":
return False
try:
return int(restarts) < 3
except ValueError:
return False
def _container_logs(cfg: dict, name: str, tail: int = 30) -> str:
cmd = f"docker logs --tail {tail} " + shlex.quote(name) + " 2>&1"
r = sc.run(sc.head(cfg), cmd, timeout=30)
return (r.stdout or "").strip()
def check_wave(cfg: dict, wave: list[dict], log, timeout: int = 900) -> None:
"""The proxy answers AND each model alias in the wave returns a completion.
Both are POLLED up to `timeout` seconds: `docker run -d` returns long before
LiteLLM has booted (~10-30s) or vLLM has loaded a 31B model into GPU memory
(minutes). A crashed container fails fast with its log tail instead of
burning the whole timeout. The proxy requires its master key even on
/models, so every probe authenticates."""
base = serving.reviewer_proxy_base(cfg).rstrip("/") # http://boardroom-proxy:PORT/v1
deadline = time.time() + timeout
# 1. Proxy reachable at all.
reach = f"curl -sf -m 8 {shlex.quote(base + '/models')} -o /dev/null && echo OK || echo FAIL"
# 1. Proxy reachable (LiteLLM boot).
reach = (
f"curl -sf -m 8 -H 'authorization: Bearer sk-local' "
f"{shlex.quote(base + '/models')} -o /dev/null && echo OK || echo FAIL"
)
while True:
r = _probe_in_net(cfg, reach, timeout=40)
if "OK" not in (r.stdout or ""):
raise RuntimeError(
f"model proxy not answering at {base} from inside the network. "
f"Check that the router container ({serving.PROXY_NAME}) came up.")
if "OK" in (r.stdout or ""):
log("[preflight] model proxy answering")
break
if not _container_running(cfg, serving.PROXY_NAME):
raise RuntimeError(
f"router container {serving.PROXY_NAME} is not running. Last log lines:\n"
+ _container_logs(cfg, serving.PROXY_NAME))
if time.time() > deadline:
raise RuntimeError(
f"model proxy did not answer at {base} within {timeout}s "
f"(container is up — check its logs on the Spark).")
log("[preflight] proxy still booting…")
time.sleep(10)
# 2. Each alias must actually return a completion.
dead = []
# 2. Each alias must actually return a completion (this is what waits out
# the multi-minute vLLM model load).
for m in wave:
alias = m["alias"]
vllm_name = f"bm-vllm-{alias}"
payload = '{"model":"%s","messages":[{"role":"user","content":"ping"}],"max_tokens":1}' % alias
probe = (
f"curl -sf -m 60 -X POST {shlex.quote(base + '/chat/completions')} "
f"-H 'content-type: application/json' -H 'authorization: Bearer sk-local' "
f"-d {shlex.quote(payload)} 2>/dev/null | head -c 600"
)
attempt = 0
while True:
rr = _probe_in_net(cfg, probe, timeout=90)
out = rr.stdout or ""
if '"choices"' not in out and '"content"' not in out:
dead.append(alias)
else:
if '"choices"' in out or '"content"' in out:
log(f"[preflight] model {alias} responded")
if dead:
break
if not _container_running(cfg, vllm_name):
raise RuntimeError(
"These models are not responding through the proxy: " + ", ".join(dead) +
". Check the corresponding vLLM container(s) on the Spark — they may have "
"failed to load, OOM'd, or (in air-gapped mode) the model isn't in the HF cache.")
f"vLLM container {vllm_name} is not running — it likely crashed "
f"during startup or model load. Last log lines:\n"
+ _container_logs(cfg, vllm_name))
if time.time() > deadline:
raise RuntimeError(
f"model {alias} did not answer through the proxy within {timeout}s. "
f"Last probe output: {out[:200] or '(empty)'}")
attempt += 1
if attempt % 4 == 1:
log(f"[preflight] {alias} still loading…")
time.sleep(15)
def check_searxng(cfg: dict, log) -> None:
+20
View File
@@ -65,6 +65,22 @@ def remove_network(cfg: dict, log) -> None:
sc.run(head, f"docker network rm {shlex.quote(net_name(cfg))} 2>/dev/null; true", timeout=30)
# ---------------------------------------------------------- resident models
def clear_resident_containers(cfg: dict, log) -> None:
"""Stop the user-listed containers on the head Spark so a grading job gets
the GPU to itself (both Sparks normally run an always-on 31B vLLM).
Deliberately NOT restarted after the job: whatever owns them is responsible
for bringing them back (the Gazette's Fleet job reloads its own models)."""
names = (cfg.get("preJobStopContainers") or "").replace(",", " ").split()
if not names:
return
head = sc.head(cfg)
quoted = " ".join(shlex.quote(n) for n in names)
log(f"[serving] freeing the GPU on {head.host}: docker stop {' '.join(names)}")
sc.run(head, f"docker stop {quoted} 2>/dev/null; true", timeout=180)
# ------------------------------------------------------------------ vLLM
def _vllm_run(cfg: dict, model: dict, hf_token: str | None) -> tuple[sc.Spark, str]:
"""Build the docker run command for one model on its assigned Spark."""
@@ -75,6 +91,10 @@ def _vllm_run(cfg: dict, model: dict, hf_token: str | None) -> tuple[sc.Spark, s
parser = cfg.get("toolCallParser", "hermes")
tools = (f"--enable-auto-tool-choice --tool-call-parser {shlex.quote(parser)} " if parser else "")
env = f"-e HF_TOKEN={shlex.quote(hf_token)} " if hf_token else ""
if cfg.get("networkMode") == "airgapped":
# The --internal network has no DNS/egress; without offline mode the HF
# hub client dies on name resolution even with the model fully cached.
env += "-e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 -e VLLM_NO_USAGE_STATS=1 -e DO_NOT_TRACK=1 "
cache = _hf_cache(cfg)
name = _vllm_name(alias)
+39 -14
View File
@@ -34,6 +34,9 @@
padding:8px 12px;font-size:13px;cursor:pointer} button:hover{background:#2c3650}
button.primary{background:#3a2f12;border-color:#6b551f;color:#ffdf9a}
button.mini{padding:2px 8px;font-size:11px;border-radius:7px}
a.mini{display:inline-block;background:#222a3a;border:1px solid var(--edge);border-radius:7px;
padding:2px 8px;font-size:11px;color:var(--ink);text-decoration:none;cursor:pointer}
a.mini:hover{background:#2c3650}
select,input[type=text]{background:#0c0f16;color:var(--ink);border:1px solid var(--edge);
border-radius:9px;padding:7px 10px;font-size:13px}
.drop{border:1.5px dashed var(--edge);border-radius:12px;padding:18px;text-align:center;color:var(--dim);cursor:pointer;margin-top:10px}
@@ -303,15 +306,21 @@ function renderPortfolio(){
async function openCompany(slug,silent){
try{
const d=await getJSON('/api/companies/'+encodeURIComponent(slug));
const changed = currentSlug!==slug;
currentSlug=slug;
renderDetail(d);
renderDetail(d,changed);
document.getElementById('companyCard').style.display='';
if(!silent) document.getElementById('companyCard').scrollIntoView({behavior:'smooth'});
}catch(e){ if(!silent) alert('load company failed: '+(e.message||e)); }
}
function closeCompany(){ currentSlug=null; document.getElementById('companyCard').style.display='none'; }
function closeCompany(){
currentSlug=null; scorecardOpen=false;
document.getElementById('coViewer').innerHTML='';
document.getElementById('scBtn').textContent='View SCORECARD.md';
document.getElementById('companyCard').style.display='none';
}
function renderDetail(d){
function renderDetail(d,changed){
const co=d.company||{}, recs=d.records||[];
document.getElementById('coName').textContent=co.name||co.slug||'?';
document.getElementById('coBadges').innerHTML=co.auto_created?'<span class="tag unreg">unregistered</span>':'';
@@ -345,30 +354,41 @@ function renderDetail(d){
`<tr><td>${esc(r.period||'?')}</td><td>${scoreBadge(r.composite)}</td>`+
`<td class="muted">${esc(String(r.graded_at||'').slice(0,16).replace('T',' '))}</td>`+
`<td><button class="mini" onclick="viewDeckReport('${esc(r.deck_id)}')">report</button> `+
`<button class="mini" onclick="viewDeckJson('${esc(r.deck_id)}')">json</button></td></tr>`).join('')+
`<button class="mini" onclick="viewDeckJson('${esc(r.deck_id)}')">json</button> `+
`<a class="mini" href="${deckBase(r.deck_id)}/report" `+
`download="${esc(co.slug||currentSlug)}-${esc(r.deck_id)}-report.md" title="Download report">⬇</a></td></tr>`).join('')+
'</tbody></table>';
// Only reset the viewer when switching companies — periodic re-renders must
// not close whatever report the user has open.
if(changed){
scorecardOpen=false;
document.getElementById('coViewer').innerHTML='';
document.getElementById('scBtn').textContent='View SCORECARD.md';
}
}
function deckBase(deckId){
return '/api/companies/'+encodeURIComponent(currentSlug)+'/decks/'+encodeURIComponent(deckId);
}
async function viewDeckReport(deckId){
if(!currentSlug) return;
const base='/api/companies/'+encodeURIComponent(currentSlug)+'/decks/'+encodeURIComponent(deckId);
const base=deckBase(deckId);
try{
const r=await fetch(base+'/report');
const t=await r.text();
setViewer('Deck report — '+deckId, `<pre class="tall">${esc(r.ok?t:('error: '+t))}</pre>`);
setViewer('Deck report — '+deckId, `<pre class="tall">${esc(r.ok?t:('error: '+t))}</pre>`,
base+'/report', `${currentSlug}-${deckId}-report.md`);
}catch(e){ alert(e); }
}
async function viewDeckJson(deckId){
if(!currentSlug) return;
try{
const d=await getJSON('/api/companies/'+encodeURIComponent(currentSlug)+
'/decks/'+encodeURIComponent(deckId));
const d=await getJSON(deckBase(deckId));
setViewer('Deck record — '+deckId,
`<details class="jsonv" open><summary>collapse / expand raw JSON</summary>`+
`<pre class="tall">${esc(JSON.stringify(d,null,2))}</pre></details>`);
`<pre class="tall">${esc(JSON.stringify(d,null,2))}</pre></details>`,
deckBase(deckId), `${currentSlug}-${deckId}.json`);
}catch(e){ alert('load record failed: '+(e.message||e)); }
}
let scorecardOpen=false;
@@ -377,18 +397,23 @@ async function toggleScorecard(){
const v=document.getElementById('coViewer'), btn=document.getElementById('scBtn');
if(scorecardOpen){ v.innerHTML=''; scorecardOpen=false; btn.textContent='View SCORECARD.md'; return; }
try{
const r=await fetch('/api/companies/'+encodeURIComponent(currentSlug)+'/scorecard');
const url='/api/companies/'+encodeURIComponent(currentSlug)+'/scorecard';
const r=await fetch(url);
const t=await r.text();
setViewer('SCORECARD.md', `<pre class="tall">${esc(r.ok?t:('error: '+t))}</pre>`);
setViewer('SCORECARD.md', `<pre class="tall">${esc(r.ok?t:('error: '+t))}</pre>`,
url, `${currentSlug}-SCORECARD.md`);
scorecardOpen=true; btn.textContent='Hide SCORECARD.md';
}catch(e){ alert(e); }
}
function setViewer(title,html){
function setViewer(title,html,dlHref,dlName){
scorecardOpen=false;
document.getElementById('scBtn').textContent='View SCORECARD.md';
const dl=dlHref?`<a class="mini" style="margin-left:auto" href="${dlHref}" `+
`download="${esc(dlName||'report.txt')}" title="Download">⬇ download</a>`:'';
document.getElementById('coViewer').innerHTML=
`<h3 class="sub" style="display:flex;align-items:center">${esc(title)}`+
`<button class="mini" style="margin-left:auto" onclick="document.getElementById('coViewer').innerHTML=''">close</button></h3>`+html;
`<h3 class="sub" style="display:flex;align-items:center;gap:8px">${esc(title)}${dl}`+
`<button class="mini" ${dlHref?'':'style="margin-left:auto"'} `+
`onclick="document.getElementById('coViewer').innerHTML=''">close</button></h3>`+html;
}
// ------------------------------------------------------------------ drop card
+5 -1
View File
@@ -134,7 +134,11 @@ def write_invalid_marker(err: str) -> None:
# ---------------------------------------------------------------- LLM client
def _post(payload: dict, timeout: int = 600) -> dict:
def _post(payload: dict, timeout: int = 1800) -> dict:
# 1800s: the DGX Sparks generate a 31B at only a few tokens/sec, and
# structured-output (json_schema/guided) decoding is slower still — a full
# extraction can legitimately run past 10 minutes. Non-streaming urlopen
# times out on total wait, so this must cover the whole completion.
req = urllib.request.Request(
f"{LLM_BASE}/chat/completions",
data=json.dumps(payload).encode(),
+14
View File
@@ -84,6 +84,18 @@ const inputSpec = InputSpec.of({
'land in the inbox. Off by default so you trigger grading explicitly.',
default: false,
}),
preJobStopContainers: Value.text({
name: 'Stop These Containers Before Grading (optional)',
description:
'Space- or comma-separated docker container names stopped on the HEAD ' +
'Spark at the start of every grading job, so the job gets the GPU to ' +
'itself (e.g. an always-on vLLM another service runs). They are NOT ' +
'restarted afterwards — the service that owns them reloads its own ' +
'models (e.g. the Gazette\'s Fleet job). Empty = stop nothing.',
required: false,
default: null,
placeholder: 'vllm-gemma4-clerk',
}),
// --- Deterministic-scorer weights (composite = quant 60 + qual 40 - flags) ---
profitabilityKpi: Value.number({
name: 'Weight: Profitability KPIs',
@@ -204,6 +216,7 @@ export const configureGrading = sdk.Action.withInput(
adjudicatorPersona: cfg.adjudicatorPersona || undefined,
wipeRemoteDocs: cfg.wipeRemoteDocs,
autoRunOnDrop: cfg.autoRunOnDrop,
preJobStopContainers: cfg.preJobStopContainers || undefined,
profitabilityKpi: cfg.weights.profitabilityKpi,
otherKpi: cfg.weights.otherKpi,
forecastIntegrity: cfg.weights.forecastIntegrity,
@@ -228,6 +241,7 @@ export const configureGrading = sdk.Action.withInput(
adjudicatorPersona: input.adjudicatorPersona ?? '',
wipeRemoteDocs: input.wipeRemoteDocs,
autoRunOnDrop: input.autoRunOnDrop,
preJobStopContainers: input.preJobStopContainers ?? '',
weights: {
profitabilityKpi: input.profitabilityKpi,
otherKpi: input.otherKpi,
+12 -1
View File
@@ -172,6 +172,10 @@ export const configShape = z.object({
autoRunOnDrop: z.boolean().default(false),
// Name of the per-job Docker network created on the head Spark.
networkName: z.string().default('boardroom-net'),
// Space/comma-separated docker container names stopped on the head Spark at
// the start of every job to free GPU memory (e.g. an always-on vLLM another
// service runs). NOT restarted afterwards — their owner reloads them.
preJobStopContainers: z.string().default(''),
// --- Portfolio companies ---
// The authoritative source of pinned KPI targets and KPI-name aliases. Decks
@@ -212,4 +216,11 @@ export const configShape = z.object({
export type Config = z.infer<typeof configShape>
export const configFile = FileHelper.json('./config.json', configShape)
// Absolute path into the `main` volume (mounted at /media/startos/volumes/main
// in the JS runtime, /data in the orchestrator container). A relative path here
// resolves against the JS runtime's EPHEMERAL working directory and silently
// loses the config on restart.
export const configFile = FileHelper.json(
'/media/startos/volumes/main/config.json',
configShape,
)
+7 -3
View File
@@ -5,9 +5,13 @@ import { FileHelper } from '@start9labs/start-sdk'
* `main` volume. Kept out of config.json so it is never returned in plaintext
* config reads. The container copies it to a 600 path at runtime.
*
* Volume path './ssh/id_spark' -> /data/ssh/id_spark inside the container.
* Volume path (absolute — a relative path would resolve against the JS
* runtime's ephemeral cwd, not the volume) -> /data/ssh/id_spark inside the
* orchestrator container.
*/
export const sshKeyFile = FileHelper.string('./ssh/id_spark')
export const sshKeyFile = FileHelper.string(
'/media/startos/volumes/main/ssh/id_spark',
)
/**
* Optional Hugging Face token (for gated/private model pulls on the Sparks).
@@ -15,7 +19,7 @@ export const sshKeyFile = FileHelper.string('./ssh/id_spark')
* network mode models are served from a pre-pulled cache, so this is only used
* the first time you warm a model (or in local_services mode).
*
* Volume path './secrets/hf_token' -> /data/secrets/hf_token.
* Volume path (absolute, see above) -> /data/secrets/hf_token.
*
* NOTE: Boardroom Map has NO frontier/cloud key. There is deliberately no Anthropic
* key and no Gitea token — the whole point is that confidential documents and
+7 -2
View File
@@ -1,8 +1,13 @@
import { VersionGraph } from '@start9labs/start-sdk'
import { v_0_1_0 } from './v_0_1_0'
import { v_0_1_1 } from './v_0_1_1'
import { v_0_1_2 } from './v_0_1_2'
import { v_0_1_3 } from './v_0_1_3'
import { v_0_1_4 } from './v_0_1_4'
import { v_0_1_5 } from './v_0_1_5'
/** The current version MUST be the first argument (`current`). */
export const versions = VersionGraph.of({
current: v_0_1_0,
other: [],
current: v_0_1_5,
other: [v_0_1_4, v_0_1_3, v_0_1_2, v_0_1_1, v_0_1_0],
})
+16
View File
@@ -0,0 +1,16 @@
import { VersionInfo } from '@start9labs/start-sdk'
/**
* Packaging fix. ExVer form `<upstream>:<downstream>` — we track our own
* packaging revision since Boardroom Map has no separate upstream semver.
*/
export const v_0_1_1 = VersionInfo.of({
version: '0.1.1:0',
releaseNotes:
'Fix: persist config.json, the Spark SSH key, and the optional HF token ' +
'to the main volume (absolute volume paths in the file models). ' +
'Previously the Configure actions wrote to an ephemeral runtime directory, ' +
'so the orchestrator never saw /data/config.json and configuration was ' +
'lost on restart. Re-run the Configure actions after updating.',
migrations: {},
})
+15
View File
@@ -0,0 +1,15 @@
import { VersionInfo } from '@start9labs/start-sdk'
/**
* GPU co-residency support. ExVer form `<upstream>:<downstream>`.
*/
export const v_0_1_2 = VersionInfo.of({
version: '0.1.2:0',
releaseNotes:
'New "Stop These Containers Before Grading" setting (Configure Grading): ' +
'container names listed there are docker-stopped on the head Spark at the ' +
'start of every grading job, so the job gets the GPU to itself when ' +
'another service keeps an always-on vLLM resident. They are deliberately ' +
'not restarted afterwards — the owning service reloads its own models.',
migrations: {},
})
+13
View File
@@ -0,0 +1,13 @@
import { VersionInfo } from '@start9labs/start-sdk'
/** Serving-readiness fix. ExVer form `<upstream>:<downstream>`. */
export const v_0_1_3 = VersionInfo.of({
version: '0.1.3:0',
releaseNotes:
'Fix wave preflight: the model-proxy probe now authenticates (the router ' +
'requires its master key even on /models) and both the proxy check and the ' +
'per-model completion check poll until ready instead of failing on a ' +
'single early attempt — a 31B model takes minutes to load into GPU ' +
'memory. Crashed containers fail fast with their log tail.',
migrations: {},
})
+13
View File
@@ -0,0 +1,13 @@
import { VersionInfo } from '@start9labs/start-sdk'
/** Air-gapped serving fix. ExVer form `<upstream>:<downstream>`. */
export const v_0_1_4 = VersionInfo.of({
version: '0.1.4:0',
releaseNotes:
'Air-gapped mode now serves models with HF_HUB_OFFLINE/TRANSFORMERS_OFFLINE ' +
'set: the per-job --internal network has no DNS, so without offline mode ' +
'the HF hub client crashed on name resolution even with the model fully ' +
'cached. Preflight also treats a crash-looping (restarting) vLLM container ' +
'as dead and fails fast with its logs instead of waiting out the timeout.',
migrations: {},
})
+14
View File
@@ -0,0 +1,14 @@
import { VersionInfo } from '@start9labs/start-sdk'
/** Dashboard viewer fixes. ExVer form `<upstream>:<downstream>`. */
export const v_0_1_5 = VersionInfo.of({
version: '0.1.5:0',
releaseNotes:
'Dashboard: an open deck report / JSON / scorecard viewer now stays open ' +
'across the periodic background refresh (previously it silently closed ' +
'within ~20s) — it closes only via its close button or when switching ' +
'companies. Added ⬇ download buttons: per-deck report downloads in the ' +
'deck-history table, plus a download link in the viewer header for ' +
'reports, deck JSON records, and SCORECARD.md.',
migrations: {},
})