jonathan/boardroom-map
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Ship v0.1.1–v0.1.5: first-live-run fixes and dashboard viewer

Browse Source 
- v0.1.1: config persistence — FileHelper paths made absolute
  (/media/startos/volumes/main/...); relative paths resolved into the JS
  runtime's ephemeral cwd so action saves never reached /data
- v0.1.2: preJobStopContainers (Configure Grading) — docker-stop resident
  vLLM containers on the head Spark at job start, no auto-restart
- v0.1.3: preflight auth (LiteLLM master_key gates /models),
  poll-until-loaded, crash fast-fail (restarting counts as dead)
- v0.1.4: HF_HUB_OFFLINE/TRANSFORMERS_OFFLINE in airgapped serving
  (--internal network has no DNS); grader _post timeout 600→1800s for
  ~3.6 tok/s GB10 generation
- v0.1.5: dashboard viewer survives the periodic background refresh;
  download buttons for deck reports, deck JSON, and SCORECARD.md
- .gitignore: .startos/ build workspace, start-technologies/

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Jonathan Kirkwood
2026-07-30 09:14:24 -05:00
co-authored by Claude Fable 5
parent 1d1074b625
commit 91212322c1
16 changed files with 262 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
orchestrator/preflight.py
+78 -23
View File
@@ -9,6 +9,7 @@ container attached to the same network.
from __future__ import annotations
import shlex
import time
import spark_client as sc
import serving
@@ -27,40 +28,94 @@ def _probe_in_net(cfg: dict, inner_cmd: str, timeout: int) -> sc.subprocess.Comp
return sc.run(sc.head(cfg), cmd, timeout=timeout)
def check_wave(cfg: dict, wave: list[dict], log) -> None:
"""The proxy answers AND each model alias in the wave returns a completion."""
def _container_running(cfg: dict, name: str) -> bool:
"""True only for a stably running container — a crash-looping one under
`--restart unless-stopped` reports Status "running"/"restarting" between
crashes, so treat repeated restarts as dead."""
cmd = ("docker inspect -f '{{.State.Status}} {{.RestartCount}}' "
+ shlex.quote(name) + " 2>/dev/null")
r = sc.run(sc.head(cfg), cmd, timeout=30)
parts = (r.stdout or "").strip().split()
if len(parts) != 2:
return False
status, restarts = parts[0], parts[1]
if status != "running":
return False
try:
return int(restarts) < 3
except ValueError:
return False
def _container_logs(cfg: dict, name: str, tail: int = 30) -> str:
cmd = f"docker logs --tail {tail} " + shlex.quote(name) + " 2>&1"
r = sc.run(sc.head(cfg), cmd, timeout=30)
return (r.stdout or "").strip()
def check_wave(cfg: dict, wave: list[dict], log, timeout: int = 900) -> None:
"""The proxy answers AND each model alias in the wave returns a completion.
Both are POLLED up to `timeout` seconds: `docker run -d` returns long before
LiteLLM has booted (~10-30s) or vLLM has loaded a 31B model into GPU memory
(minutes). A crashed container fails fast with its log tail instead of
burning the whole timeout. The proxy requires its master key even on
/models, so every probe authenticates."""
base = serving.reviewer_proxy_base(cfg).rstrip("/") # http://boardroom-proxy:PORT/v1
deadline = time.time() + timeout
# 1. Proxy reachable at all.
reach = f"curl -sf -m 8 {shlex.quote(base + '/models')} -o /dev/null && echo OK || echo FAIL"
r = _probe_in_net(cfg, reach, timeout=40)
if "OK" not in (r.stdout or ""):
raise RuntimeError(
f"model proxy not answering at {base} from inside the network. "
f"Check that the router container ({serving.PROXY_NAME}) came up.")
log("[preflight] model proxy answering")
# 1. Proxy reachable (LiteLLM boot).
reach = (
f"curl -sf -m 8 -H 'authorization: Bearer sk-local' "
f"{shlex.quote(base + '/models')} -o /dev/null && echo OK || echo FAIL"
)
while True:
r = _probe_in_net(cfg, reach, timeout=40)
if "OK" in (r.stdout or ""):
log("[preflight] model proxy answering")
break
if not _container_running(cfg, serving.PROXY_NAME):
raise RuntimeError(
f"router container {serving.PROXY_NAME} is not running. Last log lines:\n"
+ _container_logs(cfg, serving.PROXY_NAME))
if time.time() > deadline:
raise RuntimeError(
f"model proxy did not answer at {base} within {timeout}s "
f"(container is up — check its logs on the Spark).")
log("[preflight] proxy still booting…")
time.sleep(10)
# 2. Each alias must actually return a completion.
dead = []
# 2. Each alias must actually return a completion (this is what waits out
# the multi-minute vLLM model load).
for m in wave:
alias = m["alias"]
vllm_name = f"bm-vllm-{alias}"
payload = '{"model":"%s","messages":[{"role":"user","content":"ping"}],"max_tokens":1}' % alias
probe = (
f"curl -sf -m 60 -X POST {shlex.quote(base + '/chat/completions')} "
f"-H 'content-type: application/json' -H 'authorization: Bearer sk-local' "
f"-d {shlex.quote(payload)} 2>/dev/null | head -c 600"
)
rr = _probe_in_net(cfg, probe, timeout=90)
out = rr.stdout or ""
if '"choices"' not in out and '"content"' not in out:
dead.append(alias)
else:
log(f"[preflight] model {alias} responded")
if dead:
raise RuntimeError(
"These models are not responding through the proxy: " + ", ".join(dead) +
". Check the corresponding vLLM container(s) on the Spark — they may have "
"failed to load, OOM'd, or (in air-gapped mode) the model isn't in the HF cache.")
attempt = 0
while True:
rr = _probe_in_net(cfg, probe, timeout=90)
out = rr.stdout or ""
if '"choices"' in out or '"content"' in out:
log(f"[preflight] model {alias} responded")
break
if not _container_running(cfg, vllm_name):
raise RuntimeError(
f"vLLM container {vllm_name} is not running — it likely crashed "
f"during startup or model load. Last log lines:\n"
+ _container_logs(cfg, vllm_name))
if time.time() > deadline:
raise RuntimeError(
f"model {alias} did not answer through the proxy within {timeout}s. "
f"Last probe output: {out[:200] or '(empty)'}")
attempt += 1
if attempt % 4 == 1:
log(f"[preflight] {alias} still loading…")
time.sleep(15)
def check_searxng(cfg: dict, log) -> None: