Ship v0.1.1–v0.1.5: first-live-run fixes and dashboard viewer
- v0.1.1: config persistence — FileHelper paths made absolute (/media/startos/volumes/main/...); relative paths resolved into the JS runtime's ephemeral cwd so action saves never reached /data - v0.1.2: preJobStopContainers (Configure Grading) — docker-stop resident vLLM containers on the head Spark at job start, no auto-restart - v0.1.3: preflight auth (LiteLLM master_key gates /models), poll-until-loaded, crash fast-fail (restarting counts as dead) - v0.1.4: HF_HUB_OFFLINE/TRANSFORMERS_OFFLINE in airgapped serving (--internal network has no DNS); grader _post timeout 600→1800s for ~3.6 tok/s GB10 generation - v0.1.5: dashboard viewer survives the periodic background refresh; download buttons for deck reports, deck JSON, and SCORECARD.md - .gitignore: .startos/ build workspace, start-technologies/ Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
1d1074b625
commit
91212322c1
@@ -5,3 +5,5 @@ javascript/
|
|||||||
**/__pycache__/
|
**/__pycache__/
|
||||||
*.pyc
|
*.pyc
|
||||||
.claude/
|
.claude/
|
||||||
|
.startos/
|
||||||
|
start-technologies/
|
||||||
|
|||||||
@@ -69,6 +69,10 @@ CONFIG_DEFAULTS = {
|
|||||||
"wipeRemoteDocs": True,
|
"wipeRemoteDocs": True,
|
||||||
"autoRunOnDrop": False,
|
"autoRunOnDrop": False,
|
||||||
"networkName": "boardroom-net",
|
"networkName": "boardroom-net",
|
||||||
|
# Space/comma-separated docker container names stopped on the head Spark at
|
||||||
|
# the start of every job to free GPU memory (e.g. an always-on vLLM another
|
||||||
|
# service runs). NOT restarted afterwards — their owner reloads them.
|
||||||
|
"preJobStopContainers": "",
|
||||||
# Portfolio companies (authoritative source of pinned targets / aliases).
|
# Portfolio companies (authoritative source of pinned targets / aliases).
|
||||||
# pinnedTargets: [{kpi, target, unit, direction: gte|lte, profitability}]
|
# pinnedTargets: [{kpi, target, unit, direction: gte|lte, profitability}]
|
||||||
# kpiAliases: newline-separated "canonical=alias1;alias2" lines.
|
# kpiAliases: newline-separated "canonical=alias1;alias2" lines.
|
||||||
|
|||||||
@@ -305,6 +305,7 @@ class JobRunner:
|
|||||||
"primary Spark or switch to local-services mode.")
|
"primary Spark or switch to local-services mode.")
|
||||||
|
|
||||||
# 3. Infra once per job.
|
# 3. Infra once per job.
|
||||||
|
serving.clear_resident_containers(cfg, self.log)
|
||||||
gr_mod.ensure_grader_image(cfg, self.log)
|
gr_mod.ensure_grader_image(cfg, self.log)
|
||||||
serving.ensure_network(cfg, self.log)
|
serving.ensure_network(cfg, self.log)
|
||||||
preflight.check_searxng(cfg, self.log)
|
preflight.check_searxng(cfg, self.log)
|
||||||
|
|||||||
+72
-17
@@ -9,6 +9,7 @@ container attached to the same network.
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import shlex
|
import shlex
|
||||||
|
import time
|
||||||
|
|
||||||
import spark_client as sc
|
import spark_client as sc
|
||||||
import serving
|
import serving
|
||||||
@@ -27,40 +28,94 @@ def _probe_in_net(cfg: dict, inner_cmd: str, timeout: int) -> sc.subprocess.Comp
|
|||||||
return sc.run(sc.head(cfg), cmd, timeout=timeout)
|
return sc.run(sc.head(cfg), cmd, timeout=timeout)
|
||||||
|
|
||||||
|
|
||||||
def check_wave(cfg: dict, wave: list[dict], log) -> None:
|
def _container_running(cfg: dict, name: str) -> bool:
|
||||||
"""The proxy answers AND each model alias in the wave returns a completion."""
|
"""True only for a stably running container — a crash-looping one under
|
||||||
|
`--restart unless-stopped` reports Status "running"/"restarting" between
|
||||||
|
crashes, so treat repeated restarts as dead."""
|
||||||
|
cmd = ("docker inspect -f '{{.State.Status}} {{.RestartCount}}' "
|
||||||
|
+ shlex.quote(name) + " 2>/dev/null")
|
||||||
|
r = sc.run(sc.head(cfg), cmd, timeout=30)
|
||||||
|
parts = (r.stdout or "").strip().split()
|
||||||
|
if len(parts) != 2:
|
||||||
|
return False
|
||||||
|
status, restarts = parts[0], parts[1]
|
||||||
|
if status != "running":
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
return int(restarts) < 3
|
||||||
|
except ValueError:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _container_logs(cfg: dict, name: str, tail: int = 30) -> str:
|
||||||
|
cmd = f"docker logs --tail {tail} " + shlex.quote(name) + " 2>&1"
|
||||||
|
r = sc.run(sc.head(cfg), cmd, timeout=30)
|
||||||
|
return (r.stdout or "").strip()
|
||||||
|
|
||||||
|
|
||||||
|
def check_wave(cfg: dict, wave: list[dict], log, timeout: int = 900) -> None:
|
||||||
|
"""The proxy answers AND each model alias in the wave returns a completion.
|
||||||
|
|
||||||
|
Both are POLLED up to `timeout` seconds: `docker run -d` returns long before
|
||||||
|
LiteLLM has booted (~10-30s) or vLLM has loaded a 31B model into GPU memory
|
||||||
|
(minutes). A crashed container fails fast with its log tail instead of
|
||||||
|
burning the whole timeout. The proxy requires its master key even on
|
||||||
|
/models, so every probe authenticates."""
|
||||||
base = serving.reviewer_proxy_base(cfg).rstrip("/") # http://boardroom-proxy:PORT/v1
|
base = serving.reviewer_proxy_base(cfg).rstrip("/") # http://boardroom-proxy:PORT/v1
|
||||||
|
deadline = time.time() + timeout
|
||||||
|
|
||||||
# 1. Proxy reachable at all.
|
# 1. Proxy reachable (LiteLLM boot).
|
||||||
reach = f"curl -sf -m 8 {shlex.quote(base + '/models')} -o /dev/null && echo OK || echo FAIL"
|
reach = (
|
||||||
|
f"curl -sf -m 8 -H 'authorization: Bearer sk-local' "
|
||||||
|
f"{shlex.quote(base + '/models')} -o /dev/null && echo OK || echo FAIL"
|
||||||
|
)
|
||||||
|
while True:
|
||||||
r = _probe_in_net(cfg, reach, timeout=40)
|
r = _probe_in_net(cfg, reach, timeout=40)
|
||||||
if "OK" not in (r.stdout or ""):
|
if "OK" in (r.stdout or ""):
|
||||||
raise RuntimeError(
|
|
||||||
f"model proxy not answering at {base} from inside the network. "
|
|
||||||
f"Check that the router container ({serving.PROXY_NAME}) came up.")
|
|
||||||
log("[preflight] model proxy answering")
|
log("[preflight] model proxy answering")
|
||||||
|
break
|
||||||
|
if not _container_running(cfg, serving.PROXY_NAME):
|
||||||
|
raise RuntimeError(
|
||||||
|
f"router container {serving.PROXY_NAME} is not running. Last log lines:\n"
|
||||||
|
+ _container_logs(cfg, serving.PROXY_NAME))
|
||||||
|
if time.time() > deadline:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"model proxy did not answer at {base} within {timeout}s "
|
||||||
|
f"(container is up — check its logs on the Spark).")
|
||||||
|
log("[preflight] proxy still booting…")
|
||||||
|
time.sleep(10)
|
||||||
|
|
||||||
# 2. Each alias must actually return a completion.
|
# 2. Each alias must actually return a completion (this is what waits out
|
||||||
dead = []
|
# the multi-minute vLLM model load).
|
||||||
for m in wave:
|
for m in wave:
|
||||||
alias = m["alias"]
|
alias = m["alias"]
|
||||||
|
vllm_name = f"bm-vllm-{alias}"
|
||||||
payload = '{"model":"%s","messages":[{"role":"user","content":"ping"}],"max_tokens":1}' % alias
|
payload = '{"model":"%s","messages":[{"role":"user","content":"ping"}],"max_tokens":1}' % alias
|
||||||
probe = (
|
probe = (
|
||||||
f"curl -sf -m 60 -X POST {shlex.quote(base + '/chat/completions')} "
|
f"curl -sf -m 60 -X POST {shlex.quote(base + '/chat/completions')} "
|
||||||
f"-H 'content-type: application/json' -H 'authorization: Bearer sk-local' "
|
f"-H 'content-type: application/json' -H 'authorization: Bearer sk-local' "
|
||||||
f"-d {shlex.quote(payload)} 2>/dev/null | head -c 600"
|
f"-d {shlex.quote(payload)} 2>/dev/null | head -c 600"
|
||||||
)
|
)
|
||||||
|
attempt = 0
|
||||||
|
while True:
|
||||||
rr = _probe_in_net(cfg, probe, timeout=90)
|
rr = _probe_in_net(cfg, probe, timeout=90)
|
||||||
out = rr.stdout or ""
|
out = rr.stdout or ""
|
||||||
if '"choices"' not in out and '"content"' not in out:
|
if '"choices"' in out or '"content"' in out:
|
||||||
dead.append(alias)
|
|
||||||
else:
|
|
||||||
log(f"[preflight] model {alias} responded")
|
log(f"[preflight] model {alias} responded")
|
||||||
if dead:
|
break
|
||||||
|
if not _container_running(cfg, vllm_name):
|
||||||
raise RuntimeError(
|
raise RuntimeError(
|
||||||
"These models are not responding through the proxy: " + ", ".join(dead) +
|
f"vLLM container {vllm_name} is not running — it likely crashed "
|
||||||
". Check the corresponding vLLM container(s) on the Spark — they may have "
|
f"during startup or model load. Last log lines:\n"
|
||||||
"failed to load, OOM'd, or (in air-gapped mode) the model isn't in the HF cache.")
|
+ _container_logs(cfg, vllm_name))
|
||||||
|
if time.time() > deadline:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"model {alias} did not answer through the proxy within {timeout}s. "
|
||||||
|
f"Last probe output: {out[:200] or '(empty)'}")
|
||||||
|
attempt += 1
|
||||||
|
if attempt % 4 == 1:
|
||||||
|
log(f"[preflight] {alias} still loading…")
|
||||||
|
time.sleep(15)
|
||||||
|
|
||||||
|
|
||||||
def check_searxng(cfg: dict, log) -> None:
|
def check_searxng(cfg: dict, log) -> None:
|
||||||
|
|||||||
@@ -65,6 +65,22 @@ def remove_network(cfg: dict, log) -> None:
|
|||||||
sc.run(head, f"docker network rm {shlex.quote(net_name(cfg))} 2>/dev/null; true", timeout=30)
|
sc.run(head, f"docker network rm {shlex.quote(net_name(cfg))} 2>/dev/null; true", timeout=30)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------- resident models
|
||||||
|
def clear_resident_containers(cfg: dict, log) -> None:
|
||||||
|
"""Stop the user-listed containers on the head Spark so a grading job gets
|
||||||
|
the GPU to itself (both Sparks normally run an always-on 31B vLLM).
|
||||||
|
|
||||||
|
Deliberately NOT restarted after the job: whatever owns them is responsible
|
||||||
|
for bringing them back (the Gazette's Fleet job reloads its own models)."""
|
||||||
|
names = (cfg.get("preJobStopContainers") or "").replace(",", " ").split()
|
||||||
|
if not names:
|
||||||
|
return
|
||||||
|
head = sc.head(cfg)
|
||||||
|
quoted = " ".join(shlex.quote(n) for n in names)
|
||||||
|
log(f"[serving] freeing the GPU on {head.host}: docker stop {' '.join(names)}")
|
||||||
|
sc.run(head, f"docker stop {quoted} 2>/dev/null; true", timeout=180)
|
||||||
|
|
||||||
|
|
||||||
# ------------------------------------------------------------------ vLLM
|
# ------------------------------------------------------------------ vLLM
|
||||||
def _vllm_run(cfg: dict, model: dict, hf_token: str | None) -> tuple[sc.Spark, str]:
|
def _vllm_run(cfg: dict, model: dict, hf_token: str | None) -> tuple[sc.Spark, str]:
|
||||||
"""Build the docker run command for one model on its assigned Spark."""
|
"""Build the docker run command for one model on its assigned Spark."""
|
||||||
@@ -75,6 +91,10 @@ def _vllm_run(cfg: dict, model: dict, hf_token: str | None) -> tuple[sc.Spark, s
|
|||||||
parser = cfg.get("toolCallParser", "hermes")
|
parser = cfg.get("toolCallParser", "hermes")
|
||||||
tools = (f"--enable-auto-tool-choice --tool-call-parser {shlex.quote(parser)} " if parser else "")
|
tools = (f"--enable-auto-tool-choice --tool-call-parser {shlex.quote(parser)} " if parser else "")
|
||||||
env = f"-e HF_TOKEN={shlex.quote(hf_token)} " if hf_token else ""
|
env = f"-e HF_TOKEN={shlex.quote(hf_token)} " if hf_token else ""
|
||||||
|
if cfg.get("networkMode") == "airgapped":
|
||||||
|
# The --internal network has no DNS/egress; without offline mode the HF
|
||||||
|
# hub client dies on name resolution even with the model fully cached.
|
||||||
|
env += "-e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 -e VLLM_NO_USAGE_STATS=1 -e DO_NOT_TRACK=1 "
|
||||||
cache = _hf_cache(cfg)
|
cache = _hf_cache(cfg)
|
||||||
name = _vllm_name(alias)
|
name = _vllm_name(alias)
|
||||||
|
|
||||||
|
|||||||
@@ -34,6 +34,9 @@
|
|||||||
padding:8px 12px;font-size:13px;cursor:pointer} button:hover{background:#2c3650}
|
padding:8px 12px;font-size:13px;cursor:pointer} button:hover{background:#2c3650}
|
||||||
button.primary{background:#3a2f12;border-color:#6b551f;color:#ffdf9a}
|
button.primary{background:#3a2f12;border-color:#6b551f;color:#ffdf9a}
|
||||||
button.mini{padding:2px 8px;font-size:11px;border-radius:7px}
|
button.mini{padding:2px 8px;font-size:11px;border-radius:7px}
|
||||||
|
a.mini{display:inline-block;background:#222a3a;border:1px solid var(--edge);border-radius:7px;
|
||||||
|
padding:2px 8px;font-size:11px;color:var(--ink);text-decoration:none;cursor:pointer}
|
||||||
|
a.mini:hover{background:#2c3650}
|
||||||
select,input[type=text]{background:#0c0f16;color:var(--ink);border:1px solid var(--edge);
|
select,input[type=text]{background:#0c0f16;color:var(--ink);border:1px solid var(--edge);
|
||||||
border-radius:9px;padding:7px 10px;font-size:13px}
|
border-radius:9px;padding:7px 10px;font-size:13px}
|
||||||
.drop{border:1.5px dashed var(--edge);border-radius:12px;padding:18px;text-align:center;color:var(--dim);cursor:pointer;margin-top:10px}
|
.drop{border:1.5px dashed var(--edge);border-radius:12px;padding:18px;text-align:center;color:var(--dim);cursor:pointer;margin-top:10px}
|
||||||
@@ -303,15 +306,21 @@ function renderPortfolio(){
|
|||||||
async function openCompany(slug,silent){
|
async function openCompany(slug,silent){
|
||||||
try{
|
try{
|
||||||
const d=await getJSON('/api/companies/'+encodeURIComponent(slug));
|
const d=await getJSON('/api/companies/'+encodeURIComponent(slug));
|
||||||
|
const changed = currentSlug!==slug;
|
||||||
currentSlug=slug;
|
currentSlug=slug;
|
||||||
renderDetail(d);
|
renderDetail(d,changed);
|
||||||
document.getElementById('companyCard').style.display='';
|
document.getElementById('companyCard').style.display='';
|
||||||
if(!silent) document.getElementById('companyCard').scrollIntoView({behavior:'smooth'});
|
if(!silent) document.getElementById('companyCard').scrollIntoView({behavior:'smooth'});
|
||||||
}catch(e){ if(!silent) alert('load company failed: '+(e.message||e)); }
|
}catch(e){ if(!silent) alert('load company failed: '+(e.message||e)); }
|
||||||
}
|
}
|
||||||
function closeCompany(){ currentSlug=null; document.getElementById('companyCard').style.display='none'; }
|
function closeCompany(){
|
||||||
|
currentSlug=null; scorecardOpen=false;
|
||||||
|
document.getElementById('coViewer').innerHTML='';
|
||||||
|
document.getElementById('scBtn').textContent='View SCORECARD.md';
|
||||||
|
document.getElementById('companyCard').style.display='none';
|
||||||
|
}
|
||||||
|
|
||||||
function renderDetail(d){
|
function renderDetail(d,changed){
|
||||||
const co=d.company||{}, recs=d.records||[];
|
const co=d.company||{}, recs=d.records||[];
|
||||||
document.getElementById('coName').textContent=co.name||co.slug||'?';
|
document.getElementById('coName').textContent=co.name||co.slug||'?';
|
||||||
document.getElementById('coBadges').innerHTML=co.auto_created?'<span class="tag unreg">unregistered</span>':'';
|
document.getElementById('coBadges').innerHTML=co.auto_created?'<span class="tag unreg">unregistered</span>':'';
|
||||||
@@ -345,30 +354,41 @@ function renderDetail(d){
|
|||||||
`<tr><td>${esc(r.period||'?')}</td><td>${scoreBadge(r.composite)}</td>`+
|
`<tr><td>${esc(r.period||'?')}</td><td>${scoreBadge(r.composite)}</td>`+
|
||||||
`<td class="muted">${esc(String(r.graded_at||'').slice(0,16).replace('T',' '))}</td>`+
|
`<td class="muted">${esc(String(r.graded_at||'').slice(0,16).replace('T',' '))}</td>`+
|
||||||
`<td><button class="mini" onclick="viewDeckReport('${esc(r.deck_id)}')">report</button> `+
|
`<td><button class="mini" onclick="viewDeckReport('${esc(r.deck_id)}')">report</button> `+
|
||||||
`<button class="mini" onclick="viewDeckJson('${esc(r.deck_id)}')">json</button></td></tr>`).join('')+
|
`<button class="mini" onclick="viewDeckJson('${esc(r.deck_id)}')">json</button> `+
|
||||||
|
`<a class="mini" href="${deckBase(r.deck_id)}/report" `+
|
||||||
|
`download="${esc(co.slug||currentSlug)}-${esc(r.deck_id)}-report.md" title="Download report">⬇</a></td></tr>`).join('')+
|
||||||
'</tbody></table>';
|
'</tbody></table>';
|
||||||
|
|
||||||
|
// Only reset the viewer when switching companies — periodic re-renders must
|
||||||
|
// not close whatever report the user has open.
|
||||||
|
if(changed){
|
||||||
|
scorecardOpen=false;
|
||||||
document.getElementById('coViewer').innerHTML='';
|
document.getElementById('coViewer').innerHTML='';
|
||||||
document.getElementById('scBtn').textContent='View SCORECARD.md';
|
document.getElementById('scBtn').textContent='View SCORECARD.md';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
function deckBase(deckId){
|
||||||
|
return '/api/companies/'+encodeURIComponent(currentSlug)+'/decks/'+encodeURIComponent(deckId);
|
||||||
}
|
}
|
||||||
|
|
||||||
async function viewDeckReport(deckId){
|
async function viewDeckReport(deckId){
|
||||||
if(!currentSlug) return;
|
if(!currentSlug) return;
|
||||||
const base='/api/companies/'+encodeURIComponent(currentSlug)+'/decks/'+encodeURIComponent(deckId);
|
const base=deckBase(deckId);
|
||||||
try{
|
try{
|
||||||
const r=await fetch(base+'/report');
|
const r=await fetch(base+'/report');
|
||||||
const t=await r.text();
|
const t=await r.text();
|
||||||
setViewer('Deck report — '+deckId, `<pre class="tall">${esc(r.ok?t:('error: '+t))}</pre>`);
|
setViewer('Deck report — '+deckId, `<pre class="tall">${esc(r.ok?t:('error: '+t))}</pre>`,
|
||||||
|
base+'/report', `${currentSlug}-${deckId}-report.md`);
|
||||||
}catch(e){ alert(e); }
|
}catch(e){ alert(e); }
|
||||||
}
|
}
|
||||||
async function viewDeckJson(deckId){
|
async function viewDeckJson(deckId){
|
||||||
if(!currentSlug) return;
|
if(!currentSlug) return;
|
||||||
try{
|
try{
|
||||||
const d=await getJSON('/api/companies/'+encodeURIComponent(currentSlug)+
|
const d=await getJSON(deckBase(deckId));
|
||||||
'/decks/'+encodeURIComponent(deckId));
|
|
||||||
setViewer('Deck record — '+deckId,
|
setViewer('Deck record — '+deckId,
|
||||||
`<details class="jsonv" open><summary>collapse / expand raw JSON</summary>`+
|
`<details class="jsonv" open><summary>collapse / expand raw JSON</summary>`+
|
||||||
`<pre class="tall">${esc(JSON.stringify(d,null,2))}</pre></details>`);
|
`<pre class="tall">${esc(JSON.stringify(d,null,2))}</pre></details>`,
|
||||||
|
deckBase(deckId), `${currentSlug}-${deckId}.json`);
|
||||||
}catch(e){ alert('load record failed: '+(e.message||e)); }
|
}catch(e){ alert('load record failed: '+(e.message||e)); }
|
||||||
}
|
}
|
||||||
let scorecardOpen=false;
|
let scorecardOpen=false;
|
||||||
@@ -377,18 +397,23 @@ async function toggleScorecard(){
|
|||||||
const v=document.getElementById('coViewer'), btn=document.getElementById('scBtn');
|
const v=document.getElementById('coViewer'), btn=document.getElementById('scBtn');
|
||||||
if(scorecardOpen){ v.innerHTML=''; scorecardOpen=false; btn.textContent='View SCORECARD.md'; return; }
|
if(scorecardOpen){ v.innerHTML=''; scorecardOpen=false; btn.textContent='View SCORECARD.md'; return; }
|
||||||
try{
|
try{
|
||||||
const r=await fetch('/api/companies/'+encodeURIComponent(currentSlug)+'/scorecard');
|
const url='/api/companies/'+encodeURIComponent(currentSlug)+'/scorecard';
|
||||||
|
const r=await fetch(url);
|
||||||
const t=await r.text();
|
const t=await r.text();
|
||||||
setViewer('SCORECARD.md', `<pre class="tall">${esc(r.ok?t:('error: '+t))}</pre>`);
|
setViewer('SCORECARD.md', `<pre class="tall">${esc(r.ok?t:('error: '+t))}</pre>`,
|
||||||
|
url, `${currentSlug}-SCORECARD.md`);
|
||||||
scorecardOpen=true; btn.textContent='Hide SCORECARD.md';
|
scorecardOpen=true; btn.textContent='Hide SCORECARD.md';
|
||||||
}catch(e){ alert(e); }
|
}catch(e){ alert(e); }
|
||||||
}
|
}
|
||||||
function setViewer(title,html){
|
function setViewer(title,html,dlHref,dlName){
|
||||||
scorecardOpen=false;
|
scorecardOpen=false;
|
||||||
document.getElementById('scBtn').textContent='View SCORECARD.md';
|
document.getElementById('scBtn').textContent='View SCORECARD.md';
|
||||||
|
const dl=dlHref?`<a class="mini" style="margin-left:auto" href="${dlHref}" `+
|
||||||
|
`download="${esc(dlName||'report.txt')}" title="Download">⬇ download</a>`:'';
|
||||||
document.getElementById('coViewer').innerHTML=
|
document.getElementById('coViewer').innerHTML=
|
||||||
`<h3 class="sub" style="display:flex;align-items:center">${esc(title)}`+
|
`<h3 class="sub" style="display:flex;align-items:center;gap:8px">${esc(title)}${dl}`+
|
||||||
`<button class="mini" style="margin-left:auto" onclick="document.getElementById('coViewer').innerHTML=''">close</button></h3>`+html;
|
`<button class="mini" ${dlHref?'':'style="margin-left:auto"'} `+
|
||||||
|
`onclick="document.getElementById('coViewer').innerHTML=''">close</button></h3>`+html;
|
||||||
}
|
}
|
||||||
|
|
||||||
// ------------------------------------------------------------------ drop card
|
// ------------------------------------------------------------------ drop card
|
||||||
|
|||||||
@@ -134,7 +134,11 @@ def write_invalid_marker(err: str) -> None:
|
|||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------- LLM client
|
# ---------------------------------------------------------------- LLM client
|
||||||
def _post(payload: dict, timeout: int = 600) -> dict:
|
def _post(payload: dict, timeout: int = 1800) -> dict:
|
||||||
|
# 1800s: the DGX Sparks generate a 31B at only a few tokens/sec, and
|
||||||
|
# structured-output (json_schema/guided) decoding is slower still — a full
|
||||||
|
# extraction can legitimately run past 10 minutes. Non-streaming urlopen
|
||||||
|
# times out on total wait, so this must cover the whole completion.
|
||||||
req = urllib.request.Request(
|
req = urllib.request.Request(
|
||||||
f"{LLM_BASE}/chat/completions",
|
f"{LLM_BASE}/chat/completions",
|
||||||
data=json.dumps(payload).encode(),
|
data=json.dumps(payload).encode(),
|
||||||
|
|||||||
@@ -84,6 +84,18 @@ const inputSpec = InputSpec.of({
|
|||||||
'land in the inbox. Off by default so you trigger grading explicitly.',
|
'land in the inbox. Off by default so you trigger grading explicitly.',
|
||||||
default: false,
|
default: false,
|
||||||
}),
|
}),
|
||||||
|
preJobStopContainers: Value.text({
|
||||||
|
name: 'Stop These Containers Before Grading (optional)',
|
||||||
|
description:
|
||||||
|
'Space- or comma-separated docker container names stopped on the HEAD ' +
|
||||||
|
'Spark at the start of every grading job, so the job gets the GPU to ' +
|
||||||
|
'itself (e.g. an always-on vLLM another service runs). They are NOT ' +
|
||||||
|
'restarted afterwards — the service that owns them reloads its own ' +
|
||||||
|
'models (e.g. the Gazette\'s Fleet job). Empty = stop nothing.',
|
||||||
|
required: false,
|
||||||
|
default: null,
|
||||||
|
placeholder: 'vllm-gemma4-clerk',
|
||||||
|
}),
|
||||||
// --- Deterministic-scorer weights (composite = quant 60 + qual 40 - flags) ---
|
// --- Deterministic-scorer weights (composite = quant 60 + qual 40 - flags) ---
|
||||||
profitabilityKpi: Value.number({
|
profitabilityKpi: Value.number({
|
||||||
name: 'Weight: Profitability KPIs',
|
name: 'Weight: Profitability KPIs',
|
||||||
@@ -204,6 +216,7 @@ export const configureGrading = sdk.Action.withInput(
|
|||||||
adjudicatorPersona: cfg.adjudicatorPersona || undefined,
|
adjudicatorPersona: cfg.adjudicatorPersona || undefined,
|
||||||
wipeRemoteDocs: cfg.wipeRemoteDocs,
|
wipeRemoteDocs: cfg.wipeRemoteDocs,
|
||||||
autoRunOnDrop: cfg.autoRunOnDrop,
|
autoRunOnDrop: cfg.autoRunOnDrop,
|
||||||
|
preJobStopContainers: cfg.preJobStopContainers || undefined,
|
||||||
profitabilityKpi: cfg.weights.profitabilityKpi,
|
profitabilityKpi: cfg.weights.profitabilityKpi,
|
||||||
otherKpi: cfg.weights.otherKpi,
|
otherKpi: cfg.weights.otherKpi,
|
||||||
forecastIntegrity: cfg.weights.forecastIntegrity,
|
forecastIntegrity: cfg.weights.forecastIntegrity,
|
||||||
@@ -228,6 +241,7 @@ export const configureGrading = sdk.Action.withInput(
|
|||||||
adjudicatorPersona: input.adjudicatorPersona ?? '',
|
adjudicatorPersona: input.adjudicatorPersona ?? '',
|
||||||
wipeRemoteDocs: input.wipeRemoteDocs,
|
wipeRemoteDocs: input.wipeRemoteDocs,
|
||||||
autoRunOnDrop: input.autoRunOnDrop,
|
autoRunOnDrop: input.autoRunOnDrop,
|
||||||
|
preJobStopContainers: input.preJobStopContainers ?? '',
|
||||||
weights: {
|
weights: {
|
||||||
profitabilityKpi: input.profitabilityKpi,
|
profitabilityKpi: input.profitabilityKpi,
|
||||||
otherKpi: input.otherKpi,
|
otherKpi: input.otherKpi,
|
||||||
|
|||||||
@@ -172,6 +172,10 @@ export const configShape = z.object({
|
|||||||
autoRunOnDrop: z.boolean().default(false),
|
autoRunOnDrop: z.boolean().default(false),
|
||||||
// Name of the per-job Docker network created on the head Spark.
|
// Name of the per-job Docker network created on the head Spark.
|
||||||
networkName: z.string().default('boardroom-net'),
|
networkName: z.string().default('boardroom-net'),
|
||||||
|
// Space/comma-separated docker container names stopped on the head Spark at
|
||||||
|
// the start of every job to free GPU memory (e.g. an always-on vLLM another
|
||||||
|
// service runs). NOT restarted afterwards — their owner reloads them.
|
||||||
|
preJobStopContainers: z.string().default(''),
|
||||||
|
|
||||||
// --- Portfolio companies ---
|
// --- Portfolio companies ---
|
||||||
// The authoritative source of pinned KPI targets and KPI-name aliases. Decks
|
// The authoritative source of pinned KPI targets and KPI-name aliases. Decks
|
||||||
@@ -212,4 +216,11 @@ export const configShape = z.object({
|
|||||||
|
|
||||||
export type Config = z.infer<typeof configShape>
|
export type Config = z.infer<typeof configShape>
|
||||||
|
|
||||||
export const configFile = FileHelper.json('./config.json', configShape)
|
// Absolute path into the `main` volume (mounted at /media/startos/volumes/main
|
||||||
|
// in the JS runtime, /data in the orchestrator container). A relative path here
|
||||||
|
// resolves against the JS runtime's EPHEMERAL working directory and silently
|
||||||
|
// loses the config on restart.
|
||||||
|
export const configFile = FileHelper.json(
|
||||||
|
'/media/startos/volumes/main/config.json',
|
||||||
|
configShape,
|
||||||
|
)
|
||||||
|
|||||||
@@ -5,9 +5,13 @@ import { FileHelper } from '@start9labs/start-sdk'
|
|||||||
* `main` volume. Kept out of config.json so it is never returned in plaintext
|
* `main` volume. Kept out of config.json so it is never returned in plaintext
|
||||||
* config reads. The container copies it to a 600 path at runtime.
|
* config reads. The container copies it to a 600 path at runtime.
|
||||||
*
|
*
|
||||||
* Volume path './ssh/id_spark' -> /data/ssh/id_spark inside the container.
|
* Volume path (absolute — a relative path would resolve against the JS
|
||||||
|
* runtime's ephemeral cwd, not the volume) -> /data/ssh/id_spark inside the
|
||||||
|
* orchestrator container.
|
||||||
*/
|
*/
|
||||||
export const sshKeyFile = FileHelper.string('./ssh/id_spark')
|
export const sshKeyFile = FileHelper.string(
|
||||||
|
'/media/startos/volumes/main/ssh/id_spark',
|
||||||
|
)
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Optional Hugging Face token (for gated/private model pulls on the Sparks).
|
* Optional Hugging Face token (for gated/private model pulls on the Sparks).
|
||||||
@@ -15,7 +19,7 @@ export const sshKeyFile = FileHelper.string('./ssh/id_spark')
|
|||||||
* network mode models are served from a pre-pulled cache, so this is only used
|
* network mode models are served from a pre-pulled cache, so this is only used
|
||||||
* the first time you warm a model (or in local_services mode).
|
* the first time you warm a model (or in local_services mode).
|
||||||
*
|
*
|
||||||
* Volume path './secrets/hf_token' -> /data/secrets/hf_token.
|
* Volume path (absolute, see above) -> /data/secrets/hf_token.
|
||||||
*
|
*
|
||||||
* NOTE: Boardroom Map has NO frontier/cloud key. There is deliberately no Anthropic
|
* NOTE: Boardroom Map has NO frontier/cloud key. There is deliberately no Anthropic
|
||||||
* key and no Gitea token — the whole point is that confidential documents and
|
* key and no Gitea token — the whole point is that confidential documents and
|
||||||
|
|||||||
@@ -1,8 +1,13 @@
|
|||||||
import { VersionGraph } from '@start9labs/start-sdk'
|
import { VersionGraph } from '@start9labs/start-sdk'
|
||||||
import { v_0_1_0 } from './v_0_1_0'
|
import { v_0_1_0 } from './v_0_1_0'
|
||||||
|
import { v_0_1_1 } from './v_0_1_1'
|
||||||
|
import { v_0_1_2 } from './v_0_1_2'
|
||||||
|
import { v_0_1_3 } from './v_0_1_3'
|
||||||
|
import { v_0_1_4 } from './v_0_1_4'
|
||||||
|
import { v_0_1_5 } from './v_0_1_5'
|
||||||
|
|
||||||
/** The current version MUST be the first argument (`current`). */
|
/** The current version MUST be the first argument (`current`). */
|
||||||
export const versions = VersionGraph.of({
|
export const versions = VersionGraph.of({
|
||||||
current: v_0_1_0,
|
current: v_0_1_5,
|
||||||
other: [],
|
other: [v_0_1_4, v_0_1_3, v_0_1_2, v_0_1_1, v_0_1_0],
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -0,0 +1,16 @@
|
|||||||
|
import { VersionInfo } from '@start9labs/start-sdk'
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Packaging fix. ExVer form `<upstream>:<downstream>` — we track our own
|
||||||
|
* packaging revision since Boardroom Map has no separate upstream semver.
|
||||||
|
*/
|
||||||
|
export const v_0_1_1 = VersionInfo.of({
|
||||||
|
version: '0.1.1:0',
|
||||||
|
releaseNotes:
|
||||||
|
'Fix: persist config.json, the Spark SSH key, and the optional HF token ' +
|
||||||
|
'to the main volume (absolute volume paths in the file models). ' +
|
||||||
|
'Previously the Configure actions wrote to an ephemeral runtime directory, ' +
|
||||||
|
'so the orchestrator never saw /data/config.json and configuration was ' +
|
||||||
|
'lost on restart. Re-run the Configure actions after updating.',
|
||||||
|
migrations: {},
|
||||||
|
})
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
import { VersionInfo } from '@start9labs/start-sdk'
|
||||||
|
|
||||||
|
/**
|
||||||
|
* GPU co-residency support. ExVer form `<upstream>:<downstream>`.
|
||||||
|
*/
|
||||||
|
export const v_0_1_2 = VersionInfo.of({
|
||||||
|
version: '0.1.2:0',
|
||||||
|
releaseNotes:
|
||||||
|
'New "Stop These Containers Before Grading" setting (Configure Grading): ' +
|
||||||
|
'container names listed there are docker-stopped on the head Spark at the ' +
|
||||||
|
'start of every grading job, so the job gets the GPU to itself when ' +
|
||||||
|
'another service keeps an always-on vLLM resident. They are deliberately ' +
|
||||||
|
'not restarted afterwards — the owning service reloads its own models.',
|
||||||
|
migrations: {},
|
||||||
|
})
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
import { VersionInfo } from '@start9labs/start-sdk'
|
||||||
|
|
||||||
|
/** Serving-readiness fix. ExVer form `<upstream>:<downstream>`. */
|
||||||
|
export const v_0_1_3 = VersionInfo.of({
|
||||||
|
version: '0.1.3:0',
|
||||||
|
releaseNotes:
|
||||||
|
'Fix wave preflight: the model-proxy probe now authenticates (the router ' +
|
||||||
|
'requires its master key even on /models) and both the proxy check and the ' +
|
||||||
|
'per-model completion check poll until ready instead of failing on a ' +
|
||||||
|
'single early attempt — a 31B model takes minutes to load into GPU ' +
|
||||||
|
'memory. Crashed containers fail fast with their log tail.',
|
||||||
|
migrations: {},
|
||||||
|
})
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
import { VersionInfo } from '@start9labs/start-sdk'
|
||||||
|
|
||||||
|
/** Air-gapped serving fix. ExVer form `<upstream>:<downstream>`. */
|
||||||
|
export const v_0_1_4 = VersionInfo.of({
|
||||||
|
version: '0.1.4:0',
|
||||||
|
releaseNotes:
|
||||||
|
'Air-gapped mode now serves models with HF_HUB_OFFLINE/TRANSFORMERS_OFFLINE ' +
|
||||||
|
'set: the per-job --internal network has no DNS, so without offline mode ' +
|
||||||
|
'the HF hub client crashed on name resolution even with the model fully ' +
|
||||||
|
'cached. Preflight also treats a crash-looping (restarting) vLLM container ' +
|
||||||
|
'as dead and fails fast with its logs instead of waiting out the timeout.',
|
||||||
|
migrations: {},
|
||||||
|
})
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
import { VersionInfo } from '@start9labs/start-sdk'
|
||||||
|
|
||||||
|
/** Dashboard viewer fixes. ExVer form `<upstream>:<downstream>`. */
|
||||||
|
export const v_0_1_5 = VersionInfo.of({
|
||||||
|
version: '0.1.5:0',
|
||||||
|
releaseNotes:
|
||||||
|
'Dashboard: an open deck report / JSON / scorecard viewer now stays open ' +
|
||||||
|
'across the periodic background refresh (previously it silently closed ' +
|
||||||
|
'within ~20s) — it closes only via its close button or when switching ' +
|
||||||
|
'companies. Added ⬇ download buttons: per-deck report downloads in the ' +
|
||||||
|
'deck-history table, plus a download link in the viewer header for ' +
|
||||||
|
'reports, deck JSON records, and SCORECARD.md.',
|
||||||
|
migrations: {},
|
||||||
|
})
|
||||||
Reference in New Issue
Block a user