diff --git a/.gitignore b/.gitignore
index e5390b4..534a291 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,3 +5,5 @@ javascript/
**/__pycache__/
*.pyc
.claude/
+.startos/
+start-technologies/
diff --git a/orchestrator/bm_config.py b/orchestrator/bm_config.py
index 330f110..1de44d4 100644
--- a/orchestrator/bm_config.py
+++ b/orchestrator/bm_config.py
@@ -69,6 +69,10 @@ CONFIG_DEFAULTS = {
"wipeRemoteDocs": True,
"autoRunOnDrop": False,
"networkName": "boardroom-net",
+ # Space/comma-separated docker container names stopped on the head Spark at
+ # the start of every job to free GPU memory (e.g. an always-on vLLM another
+ # service runs). NOT restarted afterwards — their owner reloads them.
+ "preJobStopContainers": "",
# Portfolio companies (authoritative source of pinned targets / aliases).
# pinnedTargets: [{kpi, target, unit, direction: gte|lte, profitability}]
# kpiAliases: newline-separated "canonical=alias1;alias2" lines.
diff --git a/orchestrator/jobs.py b/orchestrator/jobs.py
index c17f652..a4a4e8c 100644
--- a/orchestrator/jobs.py
+++ b/orchestrator/jobs.py
@@ -305,6 +305,7 @@ class JobRunner:
"primary Spark or switch to local-services mode.")
# 3. Infra once per job.
+ serving.clear_resident_containers(cfg, self.log)
gr_mod.ensure_grader_image(cfg, self.log)
serving.ensure_network(cfg, self.log)
preflight.check_searxng(cfg, self.log)
diff --git a/orchestrator/preflight.py b/orchestrator/preflight.py
index 53e91ac..3fff957 100644
--- a/orchestrator/preflight.py
+++ b/orchestrator/preflight.py
@@ -9,6 +9,7 @@ container attached to the same network.
from __future__ import annotations
import shlex
+import time
import spark_client as sc
import serving
@@ -27,40 +28,94 @@ def _probe_in_net(cfg: dict, inner_cmd: str, timeout: int) -> sc.subprocess.Comp
return sc.run(sc.head(cfg), cmd, timeout=timeout)
-def check_wave(cfg: dict, wave: list[dict], log) -> None:
- """The proxy answers AND each model alias in the wave returns a completion."""
+def _container_running(cfg: dict, name: str) -> bool:
+ """True only for a stably running container — a crash-looping one under
+ `--restart unless-stopped` reports Status "running"/"restarting" between
+ crashes, so treat repeated restarts as dead."""
+ cmd = ("docker inspect -f '{{.State.Status}} {{.RestartCount}}' "
+ + shlex.quote(name) + " 2>/dev/null")
+ r = sc.run(sc.head(cfg), cmd, timeout=30)
+ parts = (r.stdout or "").strip().split()
+ if len(parts) != 2:
+ return False
+ status, restarts = parts[0], parts[1]
+ if status != "running":
+ return False
+ try:
+ return int(restarts) < 3
+ except ValueError:
+ return False
+
+
+def _container_logs(cfg: dict, name: str, tail: int = 30) -> str:
+ cmd = f"docker logs --tail {tail} " + shlex.quote(name) + " 2>&1"
+ r = sc.run(sc.head(cfg), cmd, timeout=30)
+ return (r.stdout or "").strip()
+
+
+def check_wave(cfg: dict, wave: list[dict], log, timeout: int = 900) -> None:
+ """The proxy answers AND each model alias in the wave returns a completion.
+
+ Both are POLLED up to `timeout` seconds: `docker run -d` returns long before
+ LiteLLM has booted (~10-30s) or vLLM has loaded a 31B model into GPU memory
+ (minutes). A crashed container fails fast with its log tail instead of
+ burning the whole timeout. The proxy requires its master key even on
+ /models, so every probe authenticates."""
base = serving.reviewer_proxy_base(cfg).rstrip("/") # http://boardroom-proxy:PORT/v1
+ deadline = time.time() + timeout
- # 1. Proxy reachable at all.
- reach = f"curl -sf -m 8 {shlex.quote(base + '/models')} -o /dev/null && echo OK || echo FAIL"
- r = _probe_in_net(cfg, reach, timeout=40)
- if "OK" not in (r.stdout or ""):
- raise RuntimeError(
- f"model proxy not answering at {base} from inside the network. "
- f"Check that the router container ({serving.PROXY_NAME}) came up.")
- log("[preflight] model proxy answering")
+ # 1. Proxy reachable (LiteLLM boot).
+ reach = (
+ f"curl -sf -m 8 -H 'authorization: Bearer sk-local' "
+ f"{shlex.quote(base + '/models')} -o /dev/null && echo OK || echo FAIL"
+ )
+ while True:
+ r = _probe_in_net(cfg, reach, timeout=40)
+ if "OK" in (r.stdout or ""):
+ log("[preflight] model proxy answering")
+ break
+ if not _container_running(cfg, serving.PROXY_NAME):
+ raise RuntimeError(
+ f"router container {serving.PROXY_NAME} is not running. Last log lines:\n"
+ + _container_logs(cfg, serving.PROXY_NAME))
+ if time.time() > deadline:
+ raise RuntimeError(
+ f"model proxy did not answer at {base} within {timeout}s "
+ f"(container is up — check its logs on the Spark).")
+ log("[preflight] proxy still booting…")
+ time.sleep(10)
- # 2. Each alias must actually return a completion.
- dead = []
+ # 2. Each alias must actually return a completion (this is what waits out
+ # the multi-minute vLLM model load).
for m in wave:
alias = m["alias"]
+ vllm_name = f"bm-vllm-{alias}"
payload = '{"model":"%s","messages":[{"role":"user","content":"ping"}],"max_tokens":1}' % alias
probe = (
f"curl -sf -m 60 -X POST {shlex.quote(base + '/chat/completions')} "
f"-H 'content-type: application/json' -H 'authorization: Bearer sk-local' "
f"-d {shlex.quote(payload)} 2>/dev/null | head -c 600"
)
- rr = _probe_in_net(cfg, probe, timeout=90)
- out = rr.stdout or ""
- if '"choices"' not in out and '"content"' not in out:
- dead.append(alias)
- else:
- log(f"[preflight] model {alias} responded")
- if dead:
- raise RuntimeError(
- "These models are not responding through the proxy: " + ", ".join(dead) +
- ". Check the corresponding vLLM container(s) on the Spark — they may have "
- "failed to load, OOM'd, or (in air-gapped mode) the model isn't in the HF cache.")
+ attempt = 0
+ while True:
+ rr = _probe_in_net(cfg, probe, timeout=90)
+ out = rr.stdout or ""
+ if '"choices"' in out or '"content"' in out:
+ log(f"[preflight] model {alias} responded")
+ break
+ if not _container_running(cfg, vllm_name):
+ raise RuntimeError(
+ f"vLLM container {vllm_name} is not running — it likely crashed "
+ f"during startup or model load. Last log lines:\n"
+ + _container_logs(cfg, vllm_name))
+ if time.time() > deadline:
+ raise RuntimeError(
+ f"model {alias} did not answer through the proxy within {timeout}s. "
+ f"Last probe output: {out[:200] or '(empty)'}")
+ attempt += 1
+ if attempt % 4 == 1:
+ log(f"[preflight] {alias} still loading…")
+ time.sleep(15)
def check_searxng(cfg: dict, log) -> None:
diff --git a/orchestrator/serving.py b/orchestrator/serving.py
index 75134c2..1411092 100644
--- a/orchestrator/serving.py
+++ b/orchestrator/serving.py
@@ -65,6 +65,22 @@ def remove_network(cfg: dict, log) -> None:
sc.run(head, f"docker network rm {shlex.quote(net_name(cfg))} 2>/dev/null; true", timeout=30)
+# ---------------------------------------------------------- resident models
+def clear_resident_containers(cfg: dict, log) -> None:
+ """Stop the user-listed containers on the head Spark so a grading job gets
+ the GPU to itself (both Sparks normally run an always-on 31B vLLM).
+
+ Deliberately NOT restarted after the job: whatever owns them is responsible
+ for bringing them back (the Gazette's Fleet job reloads its own models)."""
+ names = (cfg.get("preJobStopContainers") or "").replace(",", " ").split()
+ if not names:
+ return
+ head = sc.head(cfg)
+ quoted = " ".join(shlex.quote(n) for n in names)
+ log(f"[serving] freeing the GPU on {head.host}: docker stop {' '.join(names)}")
+ sc.run(head, f"docker stop {quoted} 2>/dev/null; true", timeout=180)
+
+
# ------------------------------------------------------------------ vLLM
def _vllm_run(cfg: dict, model: dict, hf_token: str | None) -> tuple[sc.Spark, str]:
"""Build the docker run command for one model on its assigned Spark."""
@@ -75,6 +91,10 @@ def _vllm_run(cfg: dict, model: dict, hf_token: str | None) -> tuple[sc.Spark, s
parser = cfg.get("toolCallParser", "hermes")
tools = (f"--enable-auto-tool-choice --tool-call-parser {shlex.quote(parser)} " if parser else "")
env = f"-e HF_TOKEN={shlex.quote(hf_token)} " if hf_token else ""
+ if cfg.get("networkMode") == "airgapped":
+ # The --internal network has no DNS/egress; without offline mode the HF
+ # hub client dies on name resolution even with the model fully cached.
+ env += "-e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 -e VLLM_NO_USAGE_STATS=1 -e DO_NOT_TRACK=1 "
cache = _hf_cache(cfg)
name = _vllm_name(alias)
diff --git a/orchestrator/templates/index.html b/orchestrator/templates/index.html
index 3eb7d41..38b8d11 100644
--- a/orchestrator/templates/index.html
+++ b/orchestrator/templates/index.html
@@ -34,6 +34,9 @@
padding:8px 12px;font-size:13px;cursor:pointer} button:hover{background:#2c3650}
button.primary{background:#3a2f12;border-color:#6b551f;color:#ffdf9a}
button.mini{padding:2px 8px;font-size:11px;border-radius:7px}
+ a.mini{display:inline-block;background:#222a3a;border:1px solid var(--edge);border-radius:7px;
+ padding:2px 8px;font-size:11px;color:var(--ink);text-decoration:none;cursor:pointer}
+ a.mini:hover{background:#2c3650}
select,input[type=text]{background:#0c0f16;color:var(--ink);border:1px solid var(--edge);
border-radius:9px;padding:7px 10px;font-size:13px}
.drop{border:1.5px dashed var(--edge);border-radius:12px;padding:18px;text-align:center;color:var(--dim);cursor:pointer;margin-top:10px}
@@ -303,15 +306,21 @@ function renderPortfolio(){
async function openCompany(slug,silent){
try{
const d=await getJSON('/api/companies/'+encodeURIComponent(slug));
+ const changed = currentSlug!==slug;
currentSlug=slug;
- renderDetail(d);
+ renderDetail(d,changed);
document.getElementById('companyCard').style.display='';
if(!silent) document.getElementById('companyCard').scrollIntoView({behavior:'smooth'});
}catch(e){ if(!silent) alert('load company failed: '+(e.message||e)); }
}
-function closeCompany(){ currentSlug=null; document.getElementById('companyCard').style.display='none'; }
+function closeCompany(){
+ currentSlug=null; scorecardOpen=false;
+ document.getElementById('coViewer').innerHTML='';
+ document.getElementById('scBtn').textContent='View SCORECARD.md';
+ document.getElementById('companyCard').style.display='none';
+}
-function renderDetail(d){
+function renderDetail(d,changed){
const co=d.company||{}, recs=d.records||[];
document.getElementById('coName').textContent=co.name||co.slug||'?';
document.getElementById('coBadges').innerHTML=co.auto_created?'unregistered':'';
@@ -345,30 +354,41 @@ function renderDetail(d){
`
`+html;
}
// ------------------------------------------------------------------ drop card
diff --git a/sandbox/grader_agent.py b/sandbox/grader_agent.py
index d7cf407..e768538 100644
--- a/sandbox/grader_agent.py
+++ b/sandbox/grader_agent.py
@@ -134,7 +134,11 @@ def write_invalid_marker(err: str) -> None:
# ---------------------------------------------------------------- LLM client
-def _post(payload: dict, timeout: int = 600) -> dict:
+def _post(payload: dict, timeout: int = 1800) -> dict:
+ # 1800s: the DGX Sparks generate a 31B at only a few tokens/sec, and
+ # structured-output (json_schema/guided) decoding is slower still — a full
+ # extraction can legitimately run past 10 minutes. Non-streaming urlopen
+ # times out on total wait, so this must cover the whole completion.
req = urllib.request.Request(
f"{LLM_BASE}/chat/completions",
data=json.dumps(payload).encode(),
diff --git a/startos/actions/configure-grading.ts b/startos/actions/configure-grading.ts
index 1aa0c69..aef4f06 100644
--- a/startos/actions/configure-grading.ts
+++ b/startos/actions/configure-grading.ts
@@ -84,6 +84,18 @@ const inputSpec = InputSpec.of({
'land in the inbox. Off by default so you trigger grading explicitly.',
default: false,
}),
+ preJobStopContainers: Value.text({
+ name: 'Stop These Containers Before Grading (optional)',
+ description:
+ 'Space- or comma-separated docker container names stopped on the HEAD ' +
+ 'Spark at the start of every grading job, so the job gets the GPU to ' +
+ 'itself (e.g. an always-on vLLM another service runs). They are NOT ' +
+ 'restarted afterwards — the service that owns them reloads its own ' +
+ 'models (e.g. the Gazette\'s Fleet job). Empty = stop nothing.',
+ required: false,
+ default: null,
+ placeholder: 'vllm-gemma4-clerk',
+ }),
// --- Deterministic-scorer weights (composite = quant 60 + qual 40 - flags) ---
profitabilityKpi: Value.number({
name: 'Weight: Profitability KPIs',
@@ -204,6 +216,7 @@ export const configureGrading = sdk.Action.withInput(
adjudicatorPersona: cfg.adjudicatorPersona || undefined,
wipeRemoteDocs: cfg.wipeRemoteDocs,
autoRunOnDrop: cfg.autoRunOnDrop,
+ preJobStopContainers: cfg.preJobStopContainers || undefined,
profitabilityKpi: cfg.weights.profitabilityKpi,
otherKpi: cfg.weights.otherKpi,
forecastIntegrity: cfg.weights.forecastIntegrity,
@@ -228,6 +241,7 @@ export const configureGrading = sdk.Action.withInput(
adjudicatorPersona: input.adjudicatorPersona ?? '',
wipeRemoteDocs: input.wipeRemoteDocs,
autoRunOnDrop: input.autoRunOnDrop,
+ preJobStopContainers: input.preJobStopContainers ?? '',
weights: {
profitabilityKpi: input.profitabilityKpi,
otherKpi: input.otherKpi,
diff --git a/startos/file-models/config.ts b/startos/file-models/config.ts
index ee75912..2b78176 100644
--- a/startos/file-models/config.ts
+++ b/startos/file-models/config.ts
@@ -172,6 +172,10 @@ export const configShape = z.object({
autoRunOnDrop: z.boolean().default(false),
// Name of the per-job Docker network created on the head Spark.
networkName: z.string().default('boardroom-net'),
+ // Space/comma-separated docker container names stopped on the head Spark at
+ // the start of every job to free GPU memory (e.g. an always-on vLLM another
+ // service runs). NOT restarted afterwards — their owner reloads them.
+ preJobStopContainers: z.string().default(''),
// --- Portfolio companies ---
// The authoritative source of pinned KPI targets and KPI-name aliases. Decks
@@ -212,4 +216,11 @@ export const configShape = z.object({
export type Config = z.infer
-export const configFile = FileHelper.json('./config.json', configShape)
+// Absolute path into the `main` volume (mounted at /media/startos/volumes/main
+// in the JS runtime, /data in the orchestrator container). A relative path here
+// resolves against the JS runtime's EPHEMERAL working directory and silently
+// loses the config on restart.
+export const configFile = FileHelper.json(
+ '/media/startos/volumes/main/config.json',
+ configShape,
+)
diff --git a/startos/file-models/secrets.ts b/startos/file-models/secrets.ts
index 8d0d482..0ae31b1 100644
--- a/startos/file-models/secrets.ts
+++ b/startos/file-models/secrets.ts
@@ -5,9 +5,13 @@ import { FileHelper } from '@start9labs/start-sdk'
* `main` volume. Kept out of config.json so it is never returned in plaintext
* config reads. The container copies it to a 600 path at runtime.
*
- * Volume path './ssh/id_spark' -> /data/ssh/id_spark inside the container.
+ * Volume path (absolute — a relative path would resolve against the JS
+ * runtime's ephemeral cwd, not the volume) -> /data/ssh/id_spark inside the
+ * orchestrator container.
*/
-export const sshKeyFile = FileHelper.string('./ssh/id_spark')
+export const sshKeyFile = FileHelper.string(
+ '/media/startos/volumes/main/ssh/id_spark',
+)
/**
* Optional Hugging Face token (for gated/private model pulls on the Sparks).
@@ -15,7 +19,7 @@ export const sshKeyFile = FileHelper.string('./ssh/id_spark')
* network mode models are served from a pre-pulled cache, so this is only used
* the first time you warm a model (or in local_services mode).
*
- * Volume path './secrets/hf_token' -> /data/secrets/hf_token.
+ * Volume path (absolute, see above) -> /data/secrets/hf_token.
*
* NOTE: Boardroom Map has NO frontier/cloud key. There is deliberately no Anthropic
* key and no Gitea token — the whole point is that confidential documents and
diff --git a/startos/versions/index.ts b/startos/versions/index.ts
index ed8db91..bca7286 100644
--- a/startos/versions/index.ts
+++ b/startos/versions/index.ts
@@ -1,8 +1,13 @@
import { VersionGraph } from '@start9labs/start-sdk'
import { v_0_1_0 } from './v_0_1_0'
+import { v_0_1_1 } from './v_0_1_1'
+import { v_0_1_2 } from './v_0_1_2'
+import { v_0_1_3 } from './v_0_1_3'
+import { v_0_1_4 } from './v_0_1_4'
+import { v_0_1_5 } from './v_0_1_5'
/** The current version MUST be the first argument (`current`). */
export const versions = VersionGraph.of({
- current: v_0_1_0,
- other: [],
+ current: v_0_1_5,
+ other: [v_0_1_4, v_0_1_3, v_0_1_2, v_0_1_1, v_0_1_0],
})
diff --git a/startos/versions/v_0_1_1.ts b/startos/versions/v_0_1_1.ts
new file mode 100644
index 0000000..deb63ee
--- /dev/null
+++ b/startos/versions/v_0_1_1.ts
@@ -0,0 +1,16 @@
+import { VersionInfo } from '@start9labs/start-sdk'
+
+/**
+ * Packaging fix. ExVer form `:` — we track our own
+ * packaging revision since Boardroom Map has no separate upstream semver.
+ */
+export const v_0_1_1 = VersionInfo.of({
+ version: '0.1.1:0',
+ releaseNotes:
+ 'Fix: persist config.json, the Spark SSH key, and the optional HF token ' +
+ 'to the main volume (absolute volume paths in the file models). ' +
+ 'Previously the Configure actions wrote to an ephemeral runtime directory, ' +
+ 'so the orchestrator never saw /data/config.json and configuration was ' +
+ 'lost on restart. Re-run the Configure actions after updating.',
+ migrations: {},
+})
diff --git a/startos/versions/v_0_1_2.ts b/startos/versions/v_0_1_2.ts
new file mode 100644
index 0000000..a71e6d8
--- /dev/null
+++ b/startos/versions/v_0_1_2.ts
@@ -0,0 +1,15 @@
+import { VersionInfo } from '@start9labs/start-sdk'
+
+/**
+ * GPU co-residency support. ExVer form `:`.
+ */
+export const v_0_1_2 = VersionInfo.of({
+ version: '0.1.2:0',
+ releaseNotes:
+ 'New "Stop These Containers Before Grading" setting (Configure Grading): ' +
+ 'container names listed there are docker-stopped on the head Spark at the ' +
+ 'start of every grading job, so the job gets the GPU to itself when ' +
+ 'another service keeps an always-on vLLM resident. They are deliberately ' +
+ 'not restarted afterwards — the owning service reloads its own models.',
+ migrations: {},
+})
diff --git a/startos/versions/v_0_1_3.ts b/startos/versions/v_0_1_3.ts
new file mode 100644
index 0000000..b78144b
--- /dev/null
+++ b/startos/versions/v_0_1_3.ts
@@ -0,0 +1,13 @@
+import { VersionInfo } from '@start9labs/start-sdk'
+
+/** Serving-readiness fix. ExVer form `:`. */
+export const v_0_1_3 = VersionInfo.of({
+ version: '0.1.3:0',
+ releaseNotes:
+ 'Fix wave preflight: the model-proxy probe now authenticates (the router ' +
+ 'requires its master key even on /models) and both the proxy check and the ' +
+ 'per-model completion check poll until ready instead of failing on a ' +
+ 'single early attempt — a 31B model takes minutes to load into GPU ' +
+ 'memory. Crashed containers fail fast with their log tail.',
+ migrations: {},
+})
diff --git a/startos/versions/v_0_1_4.ts b/startos/versions/v_0_1_4.ts
new file mode 100644
index 0000000..4de0154
--- /dev/null
+++ b/startos/versions/v_0_1_4.ts
@@ -0,0 +1,13 @@
+import { VersionInfo } from '@start9labs/start-sdk'
+
+/** Air-gapped serving fix. ExVer form `:`. */
+export const v_0_1_4 = VersionInfo.of({
+ version: '0.1.4:0',
+ releaseNotes:
+ 'Air-gapped mode now serves models with HF_HUB_OFFLINE/TRANSFORMERS_OFFLINE ' +
+ 'set: the per-job --internal network has no DNS, so without offline mode ' +
+ 'the HF hub client crashed on name resolution even with the model fully ' +
+ 'cached. Preflight also treats a crash-looping (restarting) vLLM container ' +
+ 'as dead and fails fast with its logs instead of waiting out the timeout.',
+ migrations: {},
+})
diff --git a/startos/versions/v_0_1_5.ts b/startos/versions/v_0_1_5.ts
new file mode 100644
index 0000000..5328db4
--- /dev/null
+++ b/startos/versions/v_0_1_5.ts
@@ -0,0 +1,14 @@
+import { VersionInfo } from '@start9labs/start-sdk'
+
+/** Dashboard viewer fixes. ExVer form `:`. */
+export const v_0_1_5 = VersionInfo.of({
+ version: '0.1.5:0',
+ releaseNotes:
+ 'Dashboard: an open deck report / JSON / scorecard viewer now stays open ' +
+ 'across the periodic background refresh (previously it silently closed ' +
+ 'within ~20s) — it closes only via its close button or when switching ' +
+ 'companies. Added ⬇ download buttons: per-deck report downloads in the ' +
+ 'deck-history table, plus a download link in the viewer header for ' +
+ 'reports, deck JSON records, and SCORECARD.md.',
+ migrations: {},
+})