Scaffold: fork of Chambers architecture, renamed to Boardroom Map

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

orchestrator/adjudicator.py

@@ -0,0 +1,88 @@
+"""Local lead-reviewer synthesis — no frontier model.
+
+After the panel finishes, one more hardened container (the "lead reviewer") reads
+all the individual reports (mounted read-only at /reports) plus the documents
+(/docs), runs a configured local model, and writes a single consolidated report
+to /out/CONSOLIDATED_REPORT.md: shared themes, where reviewers disagree, the
+consensus, and an overall recommendation.
+
+It reuses the same one-shot reviewer image, switched to BM_ROLE=synthesizer.
+"""
+from __future__ import annotations
+
+import shlex
+
+import spark_client as sc
+import serving
+
+DEFAULT_LEAD_PERSONA = (
+    "You are the lead reviewer chairing the panel. You did not read the documents "
+    "first-hand for a fresh opinion — your job is to CONSOLIDATE the panel's "
+    "individual reports into one authoritative report. Identify the findings the "
+    "reviewers agree on, surface and adjudicate where they conflict, note anything "
+    "only one reviewer caught, and end with a prioritized recommendation. Attribute "
+    "points to the reviewer(s) who raised them. Do not invent findings."
+)
+
+
+def pick_model(cfg: dict) -> str:
+    alias = (cfg.get("synthesisModel") or "").strip()
+    if alias:
+        return alias
+    models = cfg.get("models") or []
+    return models[0]["alias"] if models else ""
+
+
+def run_synthesis(cfg: dict, jobdir: str, rubric: str, log, wait_timeout: int = 1800) -> dict:
+    """Launch the lead-reviewer container and wait for the consolidated report."""
+    head = sc.head(cfg)
+    q = shlex.quote
+    model = pick_model(cfg)
+    if not model:
+        raise RuntimeError("no model available for synthesis (configure a model catalog)")
+
+    persona = (cfg.get("synthesisPersona") or "").strip() or DEFAULT_LEAD_PERSONA
+    net = serving.net_name(cfg)
+    base = serving.reviewer_proxy_base(cfg)
+    rid = "lead-reviewer"
+
+    sc.run(head,
+           f"mkdir -p {q(jobdir)}/personas {q(jobdir)}/synth-out && "
+           f"printf '%s' {q(persona)} > {q(jobdir)}/personas/{rid}.md && "
+           f"printf '%s' {q(rubric)} > {q(jobdir)}/RUBRIC.md",
+           timeout=30)
+
+    env = (
+        f"-e BM_REVIEWER_ID={q(rid)} -e BM_REVIEWER_NAME=lead-reviewer -e BM_ROLE=synthesizer "
+        f"-e BM_MODEL={q(model)} -e BM_LLM_BASE={q(base)} -e BM_LLM_KEY=sk-local "
+        f"-e BM_MAX_MODEL_LEN={q(str(int(cfg.get('maxModelLen', 32768))))} -e HOME=/home/rev "
+    )
+    harden = (
+        "--user 1000:1000 --security-opt no-new-privileges --cap-drop ALL "
+        "--read-only --tmpfs /tmp:size=256m --tmpfs /home/rev:size=128m "
+        "--pids-limit 256 --memory 6g --cpus 4"
+    )
+    mounts = (
+        f"-v {q(jobdir)}/docs:/docs:ro "
+        f"-v {q(jobdir)}/out:/reports:ro "
+        f"-v {q(jobdir)}/synth-out:/out "
+        f"-v {q(jobdir)}/personas/{rid}.md:/persona/PERSONA.md:ro "
+        f"-v {q(jobdir)}/RUBRIC.md:/RUBRIC.md:ro "
+    )
+    cname = f"bm-grader-{rid}"
+    cmd = (
+        f"docker rm -f {cname} >/dev/null 2>&1; "
+        f"docker run -d --name {cname} --network {q(net)} {harden} {env} {mounts} {q(cfg['graderImage'])}"
+    )
+    log(f"[synthesis] lead reviewer up -> {model}")
+    r = sc.run(head, cmd, timeout=120)
+    if r.returncode != 0:
+        raise RuntimeError(f"synthesis launch failed: {r.stderr or r.stdout}")
+
+    w = sc.run(head, f"docker wait {cname}", timeout=wait_timeout)
+    code = (w.stdout or "").strip()
+    chk = sc.run(head, f"test -s {q(jobdir)}/synth-out/CONSOLIDATED_REPORT.md && echo OK || echo MISSING", timeout=30)
+    wrote = "OK" in (chk.stdout or "")
+    sc.run(head, f"docker rm -f {cname} 2>/dev/null; true", timeout=30)
+    log(f"[synthesis] lead reviewer exited (code={code or '?'}), report={'written' if wrote else 'MISSING'}")
+    return {"model": model, "exit": code, "report": wrote}

orchestrator/app.py

@@ -0,0 +1,194 @@
+"""Boardroom Map orchestrator web app.
+
+Serves the control-panel UI and a small JSON API, and starts the background job
+runner (see jobs.py) that actually convenes the panel. Most configuration happens
+through the StartOS *actions* (Configure Sparks / Models / Reviewers / Review);
+this UI is for dropping documents, triggering a review, watching it run, and
+reading reports.
+"""
+from __future__ import annotations
+
+import os
+import threading
+
+from fastapi import FastAPI, HTTPException, UploadFile, File
+from fastapi.responses import HTMLResponse, PlainTextResponse
+from fastapi.templating import Jinja2Templates
+from starlette.requests import Request
+
+import bm_config
+import extraction
+import reviewers as rev_mod
+import serving
+from jobs import runner, INBOX, REPORTS_DIR
+
+DATA_DIR = os.environ.get("BM_DATA_DIR", "/data")
+
+app = FastAPI(title="Boardroom Map Orchestrator")
+templates = Jinja2Templates(directory=os.path.join(os.path.dirname(__file__), "templates"))
+
+
+@app.on_event("startup")
+def _startup():
+    runner.start()
+
+
+# ----------------------------------------------------------------------- UI
+@app.get("/", response_class=HTMLResponse)
+def index(request: Request):
+    return templates.TemplateResponse("index.html", {"request": request})
+
+
+@app.get("/healthz")
+def healthz():
+    return {"ok": True}
+
+
+# ----------------------------------------------------------------------- status
+@app.get("/api/status")
+def status():
+    cfg = bm_config.load()
+    catalog = {m["alias"] for m in (cfg.get("models") or [])}
+    return {
+        "configured": {
+            "sparks": bool(cfg.get("primarySparkHost")),
+            "models": len(cfg.get("models") or []),
+            "reviewers": len(cfg.get("reviewers") or []),
+        },
+        "networkMode": cfg.get("networkMode"),
+        "synthesis": bool(cfg.get("synthesisEnabled")),
+        "wipeRemoteDocs": bool(cfg.get("wipeRemoteDocs")),
+        "autoRunOnDrop": bool(cfg.get("autoRunOnDrop")),
+        "models": [{"alias": m["alias"], "hfModel": m["hfModel"], "spark": m.get("spark", "primary")}
+                   for m in (cfg.get("models") or [])],
+        "panel": [{"name": r.get("name"), "model": r.get("model"),
+                   "persona": bool((r.get("persona") or "").strip()),
+                   "known": (r.get("model") in catalog)}
+                  for r in (cfg.get("reviewers") or [])],
+        "inbox": _inbox_list(),
+        "runtime": runner.snapshot(),
+    }
+
+
+@app.get("/api/events")
+def events():
+    return {"events": runner.events()}
+
+
+def _inbox_list() -> list[dict]:
+    if not os.path.isdir(INBOX):
+        return []
+    out = []
+    for fn in sorted(os.listdir(INBOX)):
+        p = os.path.join(INBOX, fn)
+        if os.path.isfile(p):
+            ext = os.path.splitext(fn)[1].lower()
+            out.append({"name": fn, "bytes": os.path.getsize(p),
+                        "supported": ext in extraction.SUPPORTED})
+    return out
+
+
+@app.get("/api/inbox")
+def inbox():
+    return {"inbox": _inbox_list()}
+
+
+# ----------------------------------------------------------------------- documents
+@app.post("/api/upload")
+async def upload(files: list[UploadFile] = File(...)):
+    os.makedirs(INBOX, exist_ok=True)
+    saved = []
+    for f in files:
+        name = os.path.basename(f.filename or "document")
+        dest = os.path.join(INBOX, name)
+        with open(dest, "wb") as out:
+            while chunk := await f.read(1 << 20):
+                out.write(chunk)
+        saved.append(name)
+    return {"ok": True, "saved": saved}
+
+
+@app.post("/api/inbox/clear")
+def inbox_clear():
+    if os.path.isdir(INBOX):
+        for fn in os.listdir(INBOX):
+            p = os.path.join(INBOX, fn)
+            if os.path.isfile(p):
+                os.remove(p)
+    return {"ok": True}
+
+
+# ----------------------------------------------------------------------- run
+@app.post("/api/run")
+def run_now():
+    cfg = bm_config.load()
+    if not cfg.get("primarySparkHost"):
+        raise HTTPException(400, "No Spark configured (Configure Sparks).")
+    if not (cfg.get("models") and cfg.get("reviewers")):
+        raise HTTPException(400, "Configure at least one model and one reviewer first.")
+    if not _inbox_list():
+        raise HTTPException(400, "Inbox is empty — upload documents first.")
+    runner.request_run()
+    return {"ok": True, "message": "Review requested — watch the activity log."}
+
+
+@app.get("/api/serving")
+def serving_status():
+    cfg = bm_config.load()
+    if not cfg.get("primarySparkHost"):
+        raise HTTPException(400, "No Spark configured.")
+    return {"serving": serving.health(cfg)}
+
+
+@app.post("/api/reviewer/build-image")
+def build_reviewer_image():
+    cfg = bm_config.load()
+    if not cfg.get("primarySparkHost"):
+        raise HTTPException(400, "No Spark configured.")
+    threading.Thread(target=lambda: _safe_build(cfg), daemon=True).start()
+    return {"ok": True, "message": "Building reviewer image on the head Spark — watch the activity log."}
+
+
+def _safe_build(cfg: dict):
+    try:
+        rev_mod.ensure_reviewer_image(cfg, runner.log)
+    except Exception as e:
+        runner.log(f"[reviewers] image build failed: {e}")
+
+
+@app.post("/api/stop")
+def stop():
+    cfg = bm_config.load()
+    if not cfg.get("primarySparkHost"):
+        raise HTTPException(400, "No Spark configured.")
+    serving.tear_down_all(cfg, runner.log)
+    runner.phase = "idle"
+    runner._persist()
+    return {"ok": True}
+
+
+# ----------------------------------------------------------------------- reports
+@app.get("/api/reports")
+def list_reports():
+    if not os.path.isdir(REPORTS_DIR):
+        return {"reports": []}
+    jobs = sorted((d for d in os.listdir(REPORTS_DIR)
+                   if os.path.isdir(os.path.join(REPORTS_DIR, d))), reverse=True)
+    return {"reports": jobs}
+
+
+@app.get("/api/report", response_class=PlainTextResponse)
+def latest_report():
+    path = os.path.join(REPORTS_DIR, "latest.md")
+    if not os.path.exists(path):
+        return "(no report yet — drop documents in the inbox and run a review)"
+    return open(path, errors="replace").read().strip() or "(empty report)"
+
+
+@app.get("/api/reports/{job}", response_class=PlainTextResponse)
+def get_report(job: str):
+    job = os.path.basename(job)
+    path = os.path.join(REPORTS_DIR, job, "report.md")
+    if not os.path.exists(path):
+        raise HTTPException(404, "no such report")
+    return open(path, errors="replace").read()

orchestrator/bm_config.py

@@ -0,0 +1,78 @@
+"""Config loading for the Boardroom Map orchestrator.
+
+Defaults mirror startos/file-models/config.ts. The StartOS actions only persist
+the fields the user actually touched, and Python (unlike the zod schema) does not
+auto-fill defaults — so we apply them here. Keep in sync with the zod schema.
+"""
+from __future__ import annotations
+
+import os
+
+import spark_client as sc
+
+DATA_DIR = os.environ.get("BM_DATA_DIR", "/data")
+HF_TOKEN_PATH = os.path.join(DATA_DIR, "secrets", "hf_token")
+
+DEFAULT_RUBRIC = (
+    "Review the attached document(s). Produce a structured report: a 3-5 sentence "
+    "summary, the key findings and insights, risks or red flags, open questions, "
+    "and concrete recommendations. Cite the document and section for each point. "
+    "Be honest about uncertainty; never invent facts not present in the documents."
+)
+
+CONFIG_DEFAULTS = {
+    # Spark connection
+    "primarySparkHost": "",
+    "primarySparkUser": "nvidia",
+    "sshPort": 22,
+    "secondarySparkHost": None,
+    "useBothSparks": False,
+    "headInternalHost": "127.0.0.1",
+    "remoteWorkDir": "/home/nvidia/boardroom-map",
+    # Images
+    "servingImage": "boardroom-vllm:latest",
+    "graderImage": "boardroom-grader:latest",
+    # Serving
+    "gpuMemoryUtilization": "0.85",
+    "maxModelLen": 32768,
+    "toolCallParser": "hermes",
+    "proxyPort": 4000,
+    "maxConcurrentModels": 1,
+    "models": [
+        {"alias": "reviewer-a", "hfModel": "Qwen/Qwen3-32B-FP8", "spark": "primary", "port": 8001},
+    ],
+    # Review panel
+    "reviewers": [
+        {"name": "reviewer-1", "model": "reviewer-a", "persona": "", "temperature": None},
+    ],
+    # Review job settings
+    "reviewInstructions": DEFAULT_RUBRIC,
+    "networkMode": "airgapped",
+    "searxngUrl": "",
+    "synthesisEnabled": True,
+    "synthesisModel": "",
+    "synthesisPersona": "",
+    "wipeRemoteDocs": True,
+    "autoRunOnDrop": False,
+    "networkName": "boardroom-net",
+    # Flags
+    "hfTokenSet": False,
+}
+
+
+def load() -> dict:
+    """Return the merged config (defaults <- saved), or just defaults if unset."""
+    merged = dict(CONFIG_DEFAULTS)
+    try:
+        saved = sc.load_config()
+    except FileNotFoundError:
+        return merged
+    merged.update({k: v for k, v in saved.items() if v is not None})
+    return merged
+
+
+def hf_token() -> str | None:
+    if os.path.exists(HF_TOKEN_PATH):
+        t = open(HF_TOKEN_PATH).read().strip()
+        return t or None
+    return None

orchestrator/extraction.py

@@ -0,0 +1,115 @@
+"""Document text extraction — runs on the StartOS box (CPU only).
+
+Confidential documents are dropped into /data/inbox. Before anything is shipped
+to the Sparks, we extract plain text here so that only normalized text (never the
+original binaries) crosses to the review containers. Supported formats:
+
+    .pdf            -> pypdf
+    .docx           -> python-docx
+    .txt .md .text  -> read as UTF-8
+
+Anything else is skipped with a note. Each extracted document becomes a single
+UTF-8 .txt file in the per-job staging directory, which is rsynced to the Spark
+and mounted read-only into every reviewer container at /docs.
+"""
+from __future__ import annotations
+
+import os
+
+TEXT_EXTS = {".txt", ".md", ".text", ".markdown"}
+SUPPORTED = TEXT_EXTS | {".pdf", ".docx"}
+
+
+def _extract_pdf(path: str) -> str:
+    from pypdf import PdfReader
+
+    reader = PdfReader(path)
+    parts = []
+    for i, page in enumerate(reader.pages, 1):
+        try:
+            txt = page.extract_text() or ""
+        except Exception as e:
+            txt = f"(page {i}: extraction error: {e})"
+        parts.append(f"\n\n===== page {i} =====\n{txt}")
+    return "".join(parts).strip()
+
+
+def _extract_docx(path: str) -> str:
+    import docx
+
+    doc = docx.Document(path)
+    lines = [p.text for p in doc.paragraphs]
+    # Include table cell text too — contracts/specs often hide content in tables.
+    for table in doc.tables:
+        for row in table.rows:
+            cells = [c.text.strip() for c in row.cells]
+            if any(cells):
+                lines.append(" | ".join(cells))
+    return "\n".join(lines).strip()
+
+
+def _extract_text(path: str) -> str:
+    with open(path, errors="replace") as f:
+        return f.read().strip()
+
+
+def extract_file(path: str) -> str:
+    ext = os.path.splitext(path)[1].lower()
+    if ext == ".pdf":
+        return _extract_pdf(path)
+    if ext == ".docx":
+        return _extract_docx(path)
+    if ext in TEXT_EXTS:
+        return _extract_text(path)
+    raise ValueError(f"unsupported file type: {ext or '(none)'}")
+
+
+def _safe_name(name: str) -> str:
+    base = os.path.splitext(os.path.basename(name))[0]
+    keep = "".join(c if (c.isalnum() or c in "-_ ") else "_" for c in base).strip()
+    return (keep or "document").replace(" ", "_")
+
+
+def extract_inbox(inbox: str, out_dir: str, log=print) -> list[dict]:
+    """Extract every supported file in `inbox` to a .txt in `out_dir`.
+
+    Returns a manifest: [{source, out, chars, ok, error}]. Skips unsupported
+    files (recorded with ok=False) rather than failing the whole job."""
+    os.makedirs(out_dir, exist_ok=True)
+    manifest: list[dict] = []
+    if not os.path.isdir(inbox):
+        return manifest
+    seen: dict[str, int] = {}
+    for fn in sorted(os.listdir(inbox)):
+        src = os.path.join(inbox, fn)
+        if not os.path.isfile(src):
+            continue
+        ext = os.path.splitext(fn)[1].lower()
+        rec = {"source": fn, "out": None, "chars": 0, "ok": False, "error": ""}
+        if ext not in SUPPORTED:
+            rec["error"] = f"unsupported type {ext or '(none)'}"
+            log(f"[extract] skip {fn}: {rec['error']}")
+            manifest.append(rec)
+            continue
+        try:
+            text = extract_file(src)
+        except Exception as e:
+            rec["error"] = str(e)[:300]
+            log(f"[extract] FAILED {fn}: {rec['error']}")
+            manifest.append(rec)
+            continue
+        stem = _safe_name(fn)
+        if stem in seen:
+            seen[stem] += 1
+            stem = f"{stem}-{seen[stem]}"
+        else:
+            seen[stem] = 1
+        out_name = f"{stem}.txt"
+        out_path = os.path.join(out_dir, out_name)
+        header = f"# Source document: {fn}\n\n"
+        with open(out_path, "w") as f:
+            f.write(header + text + "\n")
+        rec.update({"out": out_name, "chars": len(text), "ok": True})
+        log(f"[extract] {fn} -> {out_name} ({len(text)} chars)")
+        manifest.append(rec)
+    return manifest

orchestrator/graders.py

@@ -0,0 +1,162 @@
+"""Launch the reviewer panel on the head Spark over SSH.
+
+Each reviewer is a ONE-SHOT, hardened, read-only container: it reads the document
+text mounted at /docs, runs its model (through the on-Spark proxy) under its
+persona + the shared rubric, writes a single report to /out/<id>.md, and exits.
+There is no shared writable workspace and no git — reviewers cannot alter the
+documents or each other's reports.
+
+Sandbox (per the operator's confidentiality requirement):
+  * non-root, --cap-drop ALL, --security-opt no-new-privileges
+  * read-only rootfs + small writable tmpfs; only /docs (ro), /persona (ro), and
+    /out (rw, this reviewer's report dir) are mounted
+  * NO docker socket, cpu/mem/pid caps
+  * attached to the per-job network — in airgapped mode that network is
+    --internal, so the reviewer can reach ONLY the model proxy, never the internet
+
+Reviewers hold no credentials beyond a dummy proxy key.
+"""
+from __future__ import annotations
+
+import os
+import re
+import shlex
+
+import spark_client as sc
+import serving
+
+SANDBOX_SRC = os.path.join(os.path.dirname(os.path.abspath(__file__)), "sandbox")
+
+
+def ensure_reviewer_image(cfg: dict, log) -> None:
+    """Build the reviewer image on the head Spark if missing (aarch64, native)."""
+    head = sc.head(cfg)
+    image = cfg["graderImage"]
+    q = shlex.quote
+    r = sc.run(head, f"docker image inspect {q(image)} >/dev/null 2>&1 && echo PRESENT || echo MISSING",
+               timeout=30)
+    if "PRESENT" in (r.stdout or ""):
+        log(f"[reviewers] image {image} already present on {head.host}")
+        return
+    if not os.path.isdir(SANDBOX_SRC):
+        raise RuntimeError(f"reviewer build context missing at {SANDBOX_SRC} (image not baked in?)")
+    remote_dir = f"{cfg['remoteWorkDir']}/sandbox-build"
+    log(f"[reviewers] building reviewer image {image} on {head.host} (first run; a few minutes)…")
+    push = sc.push_dir(head, SANDBOX_SRC, remote_dir)
+    if push.returncode != 0:
+        raise RuntimeError(f"rsync reviewer build context to {head.host} failed: {push.stderr}")
+    b = sc.run(head, f"cd {q(remote_dir)} && IMAGE={q(image)} bash build.sh", timeout=1800)
+    if b.returncode != 0:
+        raise RuntimeError(f"reviewer image build failed on {head.host}: {b.stderr or b.stdout}")
+    log(f"[reviewers] reviewer image built: {image}")
+
+
+def slug(name: str) -> str:
+    s = re.sub(r"[^A-Za-z0-9]+", "-", (name or "").strip().lower()).strip("-")
+    return s or "reviewer"
+
+
+def roster(cfg: dict) -> list[dict]:
+    """Reviewer roster from config. Each: {rid, name, model alias, persona, temperature}."""
+    out: list[dict] = []
+    seen: dict[str, int] = {}
+    for w in (cfg.get("reviewers") or []):
+        name = (w.get("name") or "reviewer").strip()
+        rid = slug(name)
+        if rid in seen:
+            seen[rid] += 1
+            rid = f"{rid}-{seen[rid]}"
+        else:
+            seen[rid] = 1
+        out.append({
+            "rid": rid, "name": name, "model": (w.get("model") or "").strip(),
+            "persona": (w.get("persona") or "").strip(),
+            "temperature": w.get("temperature"),
+        })
+    return out
+
+
+def _container(cfg: dict, jobdir: str, rid: str, name: str, model: str, persona: str,
+               temperature, role: str, extra_mounts: str = "") -> str:
+    """docker run command for one reviewer/synthesizer container (detached, one-shot)."""
+    q = shlex.quote
+    net = serving.net_name(cfg)
+    base = serving.reviewer_proxy_base(cfg)
+    searxng = (cfg.get("searxngUrl") or "").strip()
+    persona_path = f"{jobdir}/personas/{rid}.md"
+
+    env = (
+        f"-e BM_REVIEWER_ID={q(rid)} -e BM_REVIEWER_NAME={q(name)} -e BM_ROLE={q(role)} "
+        f"-e BM_MODEL={q(model)} -e BM_LLM_BASE={q(base)} -e BM_LLM_KEY=sk-local "
+        f"-e BM_MAX_MODEL_LEN={q(str(int(cfg.get('maxModelLen', 32768))))} "
+        f"-e HOME=/home/rev "
+    )
+    if temperature is not None:
+        env += f"-e BM_TEMPERATURE={q(str(temperature))} "
+    # web_search is offered ONLY in local_services mode with a SearXNG URL.
+    if cfg.get("networkMode") == "local_services" and searxng:
+        env += f"-e BM_SEARXNG_URL={q(searxng)} "
+
+    harden = (
+        "--user 1000:1000 --security-opt no-new-privileges --cap-drop ALL "
+        "--read-only --tmpfs /tmp:size=256m --tmpfs /home/rev:size=128m "
+        "--pids-limit 256 --memory 6g --cpus 4"
+    )
+    mounts = (
+        f"-v {q(jobdir)}/docs:/docs:ro "
+        f"-v {q(jobdir)}/out:/out "
+        f"-v {q(persona_path)}:/persona/PERSONA.md:ro "
+        + extra_mounts
+    )
+    cname = f"bm-grader-{rid}"
+    return (
+        f"docker rm -f {cname} >/dev/null 2>&1; "
+        f"docker run -d --name {cname} --network {q(net)} {harden} {env} {mounts} {q(cfg['graderImage'])}"
+    )
+
+
+def _write_persona(cfg: dict, jobdir: str, rid: str, persona: str) -> None:
+    q = shlex.quote
+    sc.run(sc.head(cfg),
+           f"mkdir -p {q(jobdir)}/personas && printf '%s' {q(persona)} > {q(jobdir)}/personas/{rid}.md",
+           timeout=30)
+
+
+def run_wave_reviewers(cfg: dict, jobdir: str, panel: list[dict], rubric: str, log,
+                       wait_timeout: int = 1800) -> list[dict]:
+    """Launch every reviewer in `panel` (already filtered to this wave's models),
+    wait for them to finish, and report status. Reports land in <jobdir>/out."""
+    head = sc.head(cfg)
+    q = shlex.quote
+    # Rubric is shared; write it once into the job dir, mounted into every container.
+    sc.run(head, f"mkdir -p {q(jobdir)}/out && printf '%s' {q(rubric)} > {q(jobdir)}/RUBRIC.md", timeout=30)
+    launched = []
+    for r in panel:
+        _write_persona(cfg, jobdir, r["rid"], r["persona"])
+        cmd = _container(cfg, jobdir, r["rid"], r["name"], r["model"], r["persona"],
+                         r.get("temperature"), role="reviewer",
+                         extra_mounts=f"-v {q(jobdir)}/RUBRIC.md:/RUBRIC.md:ro ")
+        res = sc.run(head, cmd, timeout=120)
+        if res.returncode != 0:
+            log(f"[reviewers] launch {r['rid']} FAILED: {res.stderr or res.stdout}")
+            launched.append({**r, "ok": False, "error": (res.stderr or res.stdout)[:300]})
+            continue
+        log(f"[reviewers] up: {r['rid']} -> {r['model']}")
+        launched.append({**r, "ok": True, "error": ""})
+
+    # Wait for each launched container to exit (they run in parallel; waiting
+    # sequentially still finishes when the slowest does).
+    results = []
+    for r in launched:
+        if not r["ok"]:
+            results.append(r)
+            continue
+        cname = f"bm-grader-{r['rid']}"
+        w = sc.run(head, f"docker wait {cname}", timeout=wait_timeout)
+        code = (w.stdout or "").strip()
+        out_check = sc.run(head, f"test -s {q(jobdir)}/out/{q(r['rid'])}.md && echo OK || echo MISSING", timeout=30)
+        wrote = "OK" in (out_check.stdout or "")
+        log(f"[reviewers] {r['rid']} exited (code={code or '?'}), report={'written' if wrote else 'MISSING'}")
+        sc.run(head, f"docker rm -f {cname} 2>/dev/null; true", timeout=30)
+        results.append({**r, "exit": code, "report": wrote})
+    return results

orchestrator/jobs.py

@@ -0,0 +1,379 @@
+"""The Boardroom Map job runner — convenes the review panel over dropped documents.
+
+Runs as a background thread inside the FastAPI app. It does NOT run on a clock
+like Nightshift; it reacts to triggers:
+
+  * an explicit "Run Review" (drops /data/state/run_request), or
+  * autoRunOnDrop: files landing in /data/inbox, once the inbox is stable.
+
+One job at a time. A job:
+  1. extract text from the inbox locally (CPU) — only text crosses to the Sparks
+  2. rsync the text to a per-job dir on the head Spark
+  3. serve the needed models in WAVES; run the reviewers for each wave
+  4. optionally run the local lead-reviewer synthesis
+  5. pull the reports back to /data/reports/<job>, assemble latest.md
+  6. wipe the documents from the Sparks (unless disabled) and tear serving down
+
+All state (phase, current job, per-reviewer status, last report) is mirrored to
+/data/state/runtime.json so the Web UI can render it.
+"""
+from __future__ import annotations
+
+import json
+import os
+import shutil
+import threading
+import time
+import traceback
+from collections import deque
+from datetime import datetime
+
+import bm_config
+import extraction
+import preflight
+import reviewers as rev_mod
+import serving
+import spark_client as sc
+import synthesis as synth_mod
+
+DATA_DIR = os.environ.get("BM_DATA_DIR", "/data")
+INBOX = os.path.join(DATA_DIR, "inbox")
+PROCESSED = os.path.join(DATA_DIR, "processed")
+STATE_DIR = os.path.join(DATA_DIR, "state")
+JOBS_DIR = os.path.join(STATE_DIR, "jobs")
+REPORTS_DIR = os.path.join(DATA_DIR, "reports")
+RUNTIME_PATH = os.path.join(STATE_DIR, "runtime.json")
+REQUEST_PATH = os.path.join(STATE_DIR, "run_request")
+
+TICK_SECONDS = 10
+
+
+def _inbox_signature() -> tuple[int, str]:
+    """(count, signature) of supported files in the inbox, for stability checks."""
+    if not os.path.isdir(INBOX):
+        return (0, "")
+    items = []
+    for fn in sorted(os.listdir(INBOX)):
+        p = os.path.join(INBOX, fn)
+        if os.path.isfile(p) and os.path.splitext(fn)[1].lower() in extraction.SUPPORTED:
+            items.append(f"{fn}:{os.path.getsize(p)}:{int(os.path.getmtime(p))}")
+    return (len(items), "|".join(items))
+
+
+class JobRunner:
+    def __init__(self):
+        self._events = deque(maxlen=500)
+        self._lock = threading.Lock()
+        self.phase = "idle"   # idle | extracting | reviewing | synthesizing | collecting | done | error
+        self.job_id = None
+        self.message = ""
+        self.panel: list[dict] = []
+        self.waves_total = 0
+        self.wave_index = 0
+        self.last_report_path = None
+        self._thread = None
+        self._last_sig = None
+        self._stable_sig = None
+        self._last_done_sig = None
+        for d in (STATE_DIR, JOBS_DIR, REPORTS_DIR, INBOX, PROCESSED):
+            os.makedirs(d, exist_ok=True)
+        self._restore()
+
+    # ------------------------------------------------------------- logging
+    def log(self, msg: str):
+        line = f"[{datetime.now().strftime('%H:%M:%S')}] {msg}"
+        with self._lock:
+            self._events.append(line)
+        print(line, flush=True)
+        self._persist()
+
+    def events(self) -> list[str]:
+        with self._lock:
+            return list(self._events)
+
+    # ------------------------------------------------------------- persistence
+    def _persist(self):
+        try:
+            with open(RUNTIME_PATH, "w") as f:
+                json.dump(self.snapshot() | {"events": list(self._events)[-200:],
+                                             "updated": time.time()}, f)
+        except Exception:
+            pass
+
+    def _restore(self):
+        try:
+            with open(RUNTIME_PATH) as f:
+                d = json.load(f)
+            self.phase = d.get("phase", "idle")
+            self.job_id = d.get("job_id")
+            self.message = d.get("message", "")
+            self.panel = d.get("panel", [])
+            self.last_report_path = d.get("last_report_path")
+            for e in d.get("events", []):
+                self._events.append(e)
+            # A job can't survive a restart; reset a stuck running phase.
+            if self.phase in ("extracting", "reviewing", "synthesizing", "collecting"):
+                self.phase = "idle"
+        except Exception:
+            pass
+
+    def snapshot(self) -> dict:
+        return {
+            "phase": self.phase,
+            "job_id": self.job_id,
+            "message": self.message,
+            "panel": self.panel,
+            "waves_total": self.waves_total,
+            "wave_index": self.wave_index,
+            "last_report_path": self.last_report_path,
+        }
+
+    # ------------------------------------------------------------- lifecycle
+    def start(self):
+        if self._thread and self._thread.is_alive():
+            return
+        self._thread = threading.Thread(target=self._run, daemon=True)
+        self._thread.start()
+
+    def request_run(self):
+        """Public hook (used by the API) to request a review immediately."""
+        try:
+            with open(REQUEST_PATH, "w") as f:
+                f.write(str(time.time()))
+        except Exception:
+            pass
+
+    def _run(self):
+        while True:
+            try:
+                self._poll_once()
+            except Exception as e:
+                self.phase = "error"
+                self.message = str(e)[:300]
+                self.log(f"[runner] ERROR: {e}")
+                self.log(traceback.format_exc().splitlines()[-1])
+            time.sleep(TICK_SECONDS)
+
+    def _poll_once(self):
+        cfg = bm_config.load()
+        triggered = False
+        if os.path.exists(REQUEST_PATH):
+            os.remove(REQUEST_PATH)
+            triggered = True
+            self.log("[runner] review requested")
+        elif cfg.get("autoRunOnDrop"):
+            count, sig = _inbox_signature()
+            if count and sig == self._last_sig and sig != self._last_done_sig:
+                # stable across two ticks and not the batch we last processed
+                triggered = True
+                self.log("[runner] inbox stable — auto-running review")
+            self._last_sig = sig
+
+        if not triggered:
+            return
+        count, _ = _inbox_signature()
+        if not count:
+            self.log("[runner] nothing to review (inbox empty of supported files)")
+            self.phase = "idle"
+            self._persist()
+            return
+        self._run_job(cfg)
+
+    # ------------------------------------------------------------- the job
+    def _run_job(self, cfg: dict):
+        job_id = datetime.now().strftime("%Y-%m-%d_%H%M%S")
+        self.job_id = job_id
+        self.message = ""
+        self.waves_total = 0
+        self.wave_index = 0
+        self.panel = []
+        local_job = os.path.join(JOBS_DIR, job_id)
+        local_docs = os.path.join(local_job, "docs")
+        remote_job = f"{cfg['remoteWorkDir'].rstrip('/')}/jobs/{job_id}"
+        rubric = cfg.get("reviewInstructions") or bm_config.DEFAULT_RUBRIC
+        self.log(f"=== Review job {job_id} begins ===")
+
+        try:
+            # 1. Extract locally (only text crosses to the Sparks).
+            self.phase = "extracting"; self._persist()
+            manifest = extraction.extract_inbox(INBOX, local_docs, self.log)
+            ok_docs = [m for m in manifest if m["ok"]]
+            if not ok_docs:
+                raise RuntimeError("no documents could be extracted (unsupported or empty inbox)")
+            self.log(f"[runner] extracted {len(ok_docs)} document(s)")
+
+            # 2. Resolve the panel against the model catalog.
+            catalog = {m["alias"] for m in (cfg.get("models") or [])}
+            panel = rev_mod.roster(cfg)
+            valid = [r for r in panel if r["model"] in catalog]
+            invalid = [r for r in panel if r["model"] not in catalog]
+            for r in invalid:
+                self.log(f"[runner] WARNING: reviewer '{r['name']}' uses unknown model '{r['model']}' — skipped")
+            if not valid:
+                raise RuntimeError("no reviewers reference a configured model (see Configure Models/Reviewers)")
+            self.panel = [{"name": r["name"], "model": r["model"], "status": "pending"} for r in valid]
+
+            needed = {r["model"] for r in valid}
+            if cfg.get("synthesisEnabled"):
+                sm = synth_mod.pick_model(cfg)
+                if sm:
+                    needed.add(sm)
+
+            # Air-gapped mode can't route to second-Spark models (internal net).
+            if cfg.get("networkMode") == "airgapped":
+                cat = {m["alias"]: m for m in (cfg.get("models") or [])}
+                offenders = [a for a in needed if cat.get(a, {}).get("spark") == "secondary"]
+                if offenders:
+                    raise RuntimeError(
+                        "air-gapped mode requires all models on the head Spark, but these are "
+                        f"on the secondary: {', '.join(sorted(offenders))}. Move them to the "
+                        "primary Spark or switch to local-services mode.")
+
+            # 3. Ship text to the Spark + ensure infra.
+            self.phase = "reviewing"; self._persist()
+            push = sc.push_dir(sc.head(cfg), local_docs, f"{remote_job}/docs")
+            if push.returncode != 0:
+                raise RuntimeError(f"shipping documents to the Spark failed: {push.stderr}")
+            rev_mod.ensure_reviewer_image(cfg, self.log)
+            serving.ensure_network(cfg, self.log)
+            # In local_services mode, warn early if the optional web_search backend
+            # (SearXNG, self-signed HTTPS) is unreachable — non-fatal.
+            preflight.check_searxng(cfg, self.log)
+
+            # 4. Run the panel in waves (synthesis is handled separately, below).
+            review_aliases = {r["model"] for r in valid}
+            waves = serving.plan_waves(cfg, review_aliases)
+            self.waves_total = len(waves)
+            hf = bm_config.hf_token()
+            collected = []
+            for i, wave in enumerate(waves, 1):
+                self.wave_index = i
+                wave_aliases = {m["alias"] for m in wave}
+                wpanel = [r for r in valid if r["model"] in wave_aliases]
+                self.log(f"[runner] wave {i}/{len(waves)}: models={sorted(wave_aliases)} "
+                         f"reviewers={[r['name'] for r in wpanel]}")
+                serving.bring_up_wave(cfg, wave, hf, self.log)
+                self._await_serving(cfg, wave)
+                preflight.check_wave(cfg, wave, self.log)
+                res = rev_mod.run_wave_reviewers(cfg, remote_job, wpanel, rubric, self.log)
+                collected.extend(res)
+                self._mark_panel(res)
+                serving.tear_down_wave(cfg, wave, self.log)
+
+            # 5. Synthesis (its own single-model wave).
+            synth_ok = False
+            if cfg.get("synthesisEnabled"):
+                self.phase = "synthesizing"; self._persist()
+                sm = synth_mod.pick_model(cfg)
+                swave = serving.plan_waves(cfg, {sm})
+                for wave in swave:
+                    serving.bring_up_wave(cfg, wave, hf, self.log)
+                    self._await_serving(cfg, wave)
+                    preflight.check_wave(cfg, wave, self.log)
+                    sres = synth_mod.run_synthesis(cfg, remote_job, rubric, self.log)
+                    synth_ok = bool(sres.get("report"))
+                    serving.tear_down_wave(cfg, wave, self.log)
+
+            # 6. Collect reports + assemble.
+            self.phase = "collecting"; self._persist()
+            self._collect(cfg, job_id, remote_job, local_job, valid, manifest, synth_ok)
+
+            # 7. Confidentiality: wipe the documents from the Spark.
+            if cfg.get("wipeRemoteDocs", True):
+                sc.run(sc.head(cfg), f"rm -rf {remote_job}", timeout=60)
+                self.log("[runner] wiped document text from the Spark")
+            serving.tear_down_all(cfg, self.log)
+
+            # 8. Clear the inbox (move originals aside so they aren't re-reviewed).
+            self._drain_inbox(job_id)
+            self._last_done_sig = _inbox_signature()[1]
+            self.phase = "done"
+            self.message = f"Reviewed {len(ok_docs)} document(s) with {len(valid)} reviewer(s)."
+            self.log(f"=== Review job {job_id} complete ===")
+            self._persist()
+        except Exception as e:
+            self.phase = "error"
+            self.message = f"Review failed: {e}"
+            self.log(f"[runner] JOB FAILED — {e}")
+            self.log(traceback.format_exc().splitlines()[-1])
+            try:
+                serving.tear_down_all(cfg, self.log)
+            except Exception:
+                pass
+            self._persist()
+
+    # ------------------------------------------------------------- helpers
+    def _await_serving(self, cfg, wave, timeout=900):
+        self.log("[runner] waiting for wave serving to come online…")
+        deadline = time.time() + timeout
+        want = len(wave) + 1  # vLLMs + proxy
+        while time.time() < deadline:
+            running = serving.health(cfg).get("running", [])
+            if sum(1 for r in running if "Up" in r) >= want:
+                self.log("[runner] wave serving online")
+                return
+            time.sleep(15)
+        self.log("[runner] WARNING: wave serving not fully confirmed; continuing")
+
+    def _mark_panel(self, res: list[dict]):
+        by_name = {r["name"]: r for r in res}
+        for p in self.panel:
+            r = by_name.get(p["name"])
+            if not r:
+                continue
+            if not r.get("ok", True):
+                p["status"] = "launch-failed"
+            elif r.get("report"):
+                p["status"] = "done"
+            else:
+                p["status"] = "no-report"
+        self._persist()
+
+    def _collect(self, cfg, job_id, remote_job, local_job, valid, manifest, synth_ok):
+        out_local = os.path.join(REPORTS_DIR, job_id)
+        os.makedirs(out_local, exist_ok=True)
+        sc.pull_dir(sc.head(cfg), f"{remote_job}/out", os.path.join(out_local, "reviewers"))
+        if synth_ok:
+            sc.pull_dir(sc.head(cfg), f"{remote_job}/synth-out", os.path.join(out_local, "synthesis"))
+
+        # Assemble a single latest.md: the consolidated report if present, else a
+        # concatenation of the individual reports.
+        parts = [f"# Boardroom Map review — {job_id}\n",
+                 "Documents reviewed: " + ", ".join(m["source"] for m in manifest if m["ok"]) + "\n",
+                 "Panel: " + ", ".join(f"{r['name']} ({r['model']})" for r in valid) + "\n"]
+        consolidated = os.path.join(out_local, "synthesis", "CONSOLIDATED_REPORT.md")
+        if synth_ok and os.path.exists(consolidated):
+            parts.append("\n---\n\n## Consolidated report (lead reviewer)\n\n")
+            parts.append(open(consolidated, errors="replace").read())
+            parts.append("\n\n---\n")
+        parts.append("\n## Individual reviewer reports\n")
+        rev_local = os.path.join(out_local, "reviewers")
+        if os.path.isdir(rev_local):
+            for fn in sorted(os.listdir(rev_local)):
+                if fn.endswith(".md"):
+                    parts.append(f"\n### {fn[:-3]}\n\n")
+                    parts.append(open(os.path.join(rev_local, fn), errors="replace").read())
+                    parts.append("\n")
+        assembled = "".join(parts)
+        with open(os.path.join(out_local, "report.md"), "w") as f:
+            f.write(assembled)
+        with open(os.path.join(REPORTS_DIR, "latest.md"), "w") as f:
+            f.write(assembled)
+        self.last_report_path = os.path.join(out_local, "report.md")
+        self.log(f"[runner] reports saved to {out_local}")
+
+    def _drain_inbox(self, job_id):
+        dest = os.path.join(PROCESSED, job_id)
+        os.makedirs(dest, exist_ok=True)
+        for fn in os.listdir(INBOX):
+            src = os.path.join(INBOX, fn)
+            if os.path.isfile(src):
+                try:
+                    shutil.move(src, os.path.join(dest, fn))
+                except Exception:
+                    pass
+        self.log(f"[runner] inbox cleared (originals moved to processed/{job_id})")
+
+
+# Module-level singleton used by app.py
+runner = JobRunner()

orchestrator/preflight.py

@@ -0,0 +1,92 @@
+"""Preflight checks — fail LOUD before launching reviewers, so a dead model
+endpoint is caught immediately instead of after a container spins fruitlessly.
+
+The model proxy has no published host port (it lives on the per-job Docker
+network so that air-gapped reviewers can reach it without the host exposing
+anything). So we probe it the same way a reviewer would: from a throwaway
+container attached to the same network.
+"""
+from __future__ import annotations
+
+import shlex
+
+import spark_client as sc
+import serving
+
+
+def _probe_in_net(cfg: dict, inner_cmd: str, timeout: int) -> sc.subprocess.CompletedProcess:
+    """Run a shell command inside a throwaway container on the per-job network,
+    using the reviewer image (it has curl)."""
+    q = shlex.quote
+    net = serving.net_name(cfg)
+    image = cfg["graderImage"]
+    cmd = (
+        f"docker run --rm --network {q(net)} --entrypoint sh {q(image)} "
+        f"-c {q(inner_cmd)}"
+    )
+    return sc.run(sc.head(cfg), cmd, timeout=timeout)
+
+
+def check_wave(cfg: dict, wave: list[dict], log) -> None:
+    """The proxy answers AND each model alias in the wave returns a completion."""
+    base = serving.reviewer_proxy_base(cfg).rstrip("/")  # http://boardroom-proxy:PORT/v1
+
+    # 1. Proxy reachable at all.
+    reach = f"curl -sf -m 8 {shlex.quote(base + '/models')} -o /dev/null && echo OK || echo FAIL"
+    r = _probe_in_net(cfg, reach, timeout=40)
+    if "OK" not in (r.stdout or ""):
+        raise RuntimeError(
+            f"model proxy not answering at {base} from inside the network. "
+            f"Check that the router container ({serving.PROXY_NAME}) came up.")
+    log("[preflight] model proxy answering")
+
+    # 2. Each alias must actually return a completion.
+    dead = []
+    for m in wave:
+        alias = m["alias"]
+        payload = '{"model":"%s","messages":[{"role":"user","content":"ping"}],"max_tokens":1}' % alias
+        probe = (
+            f"curl -sf -m 60 -X POST {shlex.quote(base + '/chat/completions')} "
+            f"-H 'content-type: application/json' -H 'authorization: Bearer sk-local' "
+            f"-d {shlex.quote(payload)} 2>/dev/null | head -c 600"
+        )
+        rr = _probe_in_net(cfg, probe, timeout=90)
+        out = rr.stdout or ""
+        if '"choices"' not in out and '"content"' not in out:
+            dead.append(alias)
+        else:
+            log(f"[preflight] model {alias} responded")
+    if dead:
+        raise RuntimeError(
+            "These models are not responding through the proxy: " + ", ".join(dead) +
+            ". Check the corresponding vLLM container(s) on the Spark — they may have "
+            "failed to load, OOM'd, or (in air-gapped mode) the model isn't in the HF cache.")
+
+
+def check_searxng(cfg: dict, log) -> None:
+    """In local_services mode, verify the reviewers' optional web_search backend.
+
+    StartOS exposes SearXNG only over HTTPS with a SELF-SIGNED cert (the lesson
+    from Nightshift's first deploy), so the probe uses `curl -sfk`, and it asserts
+    the JSON format is enabled (HTML back == json format off). This is NON-fatal:
+    web_search is an optional enhancement in Boardroom Map, so a broken SearXNG only
+    warns rather than failing the whole review (unlike Nightshift, where research
+    depended on it)."""
+    if cfg.get("networkMode") != "local_services":
+        return
+    sx = (cfg.get("searxngUrl") or "").strip().rstrip("/")
+    if not sx:
+        return
+    head = sc.head(cfg)
+    probe = f"curl -sfk -m 12 {shlex.quote(sx + '/search?q=boardroom&format=json')} 2>/dev/null | head -c 400"
+    r = sc.run(head, probe, timeout=25)
+    out = (r.stdout or "").strip()
+    if not out:
+        log(f"[preflight] WARNING: SearXNG not reachable from the head Spark at {sx} — "
+            "reviewers' web_search will be unavailable (reviews still run).")
+        return
+    if not out.lstrip().startswith("{") and '"results"' not in out:
+        log(f"[preflight] WARNING: SearXNG at {sx} did not return JSON (enable the json "
+            "format in settings.yml) — web_search may not work.")
+        return
+    log("[preflight] SearXNG reachable with JSON enabled")

orchestrator/requirements.txt

@@ -0,0 +1,7 @@
+fastapi==0.115.6
+uvicorn[standard]==0.34.0
+jinja2==3.1.5
+python-multipart==0.0.20
+pyyaml==6.0.2
+pypdf==5.1.0
+python-docx==1.1.2

orchestrator/serving.py

@@ -0,0 +1,211 @@
+"""Bring up model serving on the Sparks over SSH, in WAVES.
+
+Boardroom Map serves a user-defined catalog of local models. Two Sparks can't hold
+unlimited distinct models, so the job runner loads them in waves: each wave brings
+up the vLLM container(s) that fit (<= maxConcurrentModels on the head Spark),
+points a LiteLLM router at them, lets the reviewers assigned to those models run,
+then tears the wave down and loads the next.
+
+Network topology (the confidentiality boundary):
+  * A per-job user-defined Docker network on the head Spark (default
+    "boardroom-net"). In `airgapped` mode it is created with --internal, so the
+    reviewer containers attached to it can reach the model proxy but have ZERO
+    internet egress. In `local_services` mode it is a normal bridge (egress
+    possible) so reviewers can also reach LAN services / the second Spark.
+  * Head-Spark vLLMs join this network; the proxy reaches them by container name
+    (bm-vllm-<alias>). Reviewers reach the proxy by name (boardroom-proxy).
+  * Second-Spark vLLMs publish a host port; the proxy reaches them over the LAN.
+    This only works in local_services mode (an --internal network can't route to
+    the LAN), so airgapped jobs must keep all models on the head Spark — enforced
+    in preflight.
+
+Models are served from a pre-populated HF cache mounted from the Spark work dir,
+so airgapped serving needs no live download.
+"""
+from __future__ import annotations
+
+import json
+import shlex
+
+import spark_client as sc
+
+PROXY_NAME = "boardroom-proxy"
+
+
+def net_name(cfg: dict) -> str:
+    return cfg.get("networkName") or "boardroom-net"
+
+
+def _vllm_name(alias: str) -> str:
+    return f"bm-vllm-{alias}"
+
+
+def _hf_cache(cfg: dict) -> str:
+    return f"{cfg['remoteWorkDir'].rstrip('/')}/hf-cache"
+
+
+# ------------------------------------------------------------------ network
+def ensure_network(cfg: dict, log) -> None:
+    head = sc.head(cfg)
+    name = net_name(cfg)
+    internal = "--internal " if cfg.get("networkMode") == "airgapped" else ""
+    cmd = (
+        f"docker network inspect {shlex.quote(name)} >/dev/null 2>&1 && echo EXISTS || "
+        f"docker network create {internal}{shlex.quote(name)}"
+    )
+    r = sc.run(head, cmd, timeout=60)
+    if r.returncode != 0:
+        raise RuntimeError(f"could not create docker network {name}: {r.stderr or r.stdout}")
+    mode = "airgapped (--internal)" if internal else "local_services"
+    log(f"[serving] network {name} ready ({mode})")
+
+
+def remove_network(cfg: dict, log) -> None:
+    head = sc.head(cfg)
+    sc.run(head, f"docker network rm {shlex.quote(net_name(cfg))} 2>/dev/null; true", timeout=30)
+
+
+# ------------------------------------------------------------------ vLLM
+def _vllm_run(cfg: dict, model: dict, hf_token: str | None) -> tuple[sc.Spark, str]:
+    """Build the docker run command for one model on its assigned Spark."""
+    alias, hf, port = model["alias"], model["hfModel"], int(model["port"])
+    image = cfg["servingImage"]
+    gpu_util = cfg["gpuMemoryUtilization"]
+    max_len = int(cfg["maxModelLen"])
+    parser = cfg.get("toolCallParser", "hermes")
+    tools = (f"--enable-auto-tool-choice --tool-call-parser {shlex.quote(parser)} " if parser else "")
+    env = f"-e HF_TOKEN={shlex.quote(hf_token)} " if hf_token else ""
+    cache = _hf_cache(cfg)
+    name = _vllm_name(alias)
+
+    if model.get("spark") == "secondary":
+        target = sc.by_role(cfg, "secondary")
+        # 2nd Spark: publish the port so the head's proxy can reach it over the LAN.
+        net = f"-p {port}:{port}"
+    else:
+        target = sc.head(cfg)
+        # Head Spark: attach to the per-job network; proxy reaches it by name.
+        net = f"--network {shlex.quote(net_name(cfg))}"
+
+    cmd = (
+        f"mkdir -p {shlex.quote(cache)}; "
+        f"docker rm -f {name} >/dev/null 2>&1; "
+        f"docker run -d --name {name} --gpus all --ipc=host --shm-size=16g "
+        f"--restart unless-stopped {net} {env}"
+        f"-v {shlex.quote(cache)}:/root/.cache/huggingface "
+        f"{shlex.quote(image)} "
+        f"vllm serve {shlex.quote(hf)} --host 0.0.0.0 --port {port} "
+        f"--gpu-memory-utilization {gpu_util} --max-model-len {max_len} {tools}"
+        f"--served-model-name {shlex.quote(alias)}"
+    )
+    return target, cmd
+
+
+def _litellm_config(cfg: dict, wave: list[dict]) -> dict:
+    """Router config exposing each model alias in the wave on one endpoint."""
+    model_list = []
+    for m in wave:
+        alias, port = m["alias"], int(m["port"])
+        if m.get("spark") == "secondary":
+            api_base = f"http://{cfg['secondarySparkHost']}:{port}/v1"
+        else:
+            api_base = f"http://{_vllm_name(alias)}:{port}/v1"
+        model_list.append({
+            "model_name": alias,
+            "litellm_params": {
+                "model": f"openai/{alias}",
+                "api_base": api_base,
+                "api_key": "sk-local",
+            },
+        })
+    return {"model_list": model_list, "general_settings": {"master_key": "sk-local"}}
+
+
+def bring_up_wave(cfg: dict, wave: list[dict], hf_token: str | None, log) -> None:
+    """Start the vLLMs for `wave` and a router that exposes them. Idempotent."""
+    head = sc.head(cfg)
+    for m in wave:
+        target, cmd = _vllm_run(cfg, m, hf_token)
+        log(f"[serving] launching {m['alias']} ({m['hfModel']}) on {target.host}:{m['port']}")
+        r = sc.run(target, cmd, timeout=240)
+        if r.returncode != 0:
+            raise RuntimeError(f"vLLM launch for {m['alias']} failed: {r.stderr or r.stdout}")
+
+    # LiteLLM router on the head Spark, attached to the per-job network so the
+    # reviewers reach it by name (boardroom-proxy). No published port — preflight
+    # probes it from inside the network.
+    cfg_json = json.dumps(_litellm_config(cfg, wave))
+    workdir = cfg["remoteWorkDir"]
+    remote_cfg = f"{workdir}/litellm.config.json"
+    proxy_port = int(cfg["proxyPort"])
+    log(f"[serving] launching router {PROXY_NAME} on network {net_name(cfg)}:{proxy_port}")
+    setup = (
+        f"mkdir -p {shlex.quote(workdir)} && "
+        f"printf '%s' {shlex.quote(cfg_json)} > {shlex.quote(remote_cfg)} && "
+        f"docker rm -f {PROXY_NAME} >/dev/null 2>&1; "
+        f"docker run -d --name {PROXY_NAME} --restart unless-stopped "
+        f"--network {shlex.quote(net_name(cfg))} "
+        f"-v {shlex.quote(remote_cfg)}:/app/config.json "
+        f"ghcr.io/berriai/litellm:main-stable "
+        f"--config /app/config.json --port {proxy_port} --host 0.0.0.0"
+    )
+    r = sc.run(head, setup, timeout=180)
+    if r.returncode != 0:
+        raise RuntimeError(f"LiteLLM router launch failed: {r.stderr or r.stdout}")
+
+
+def tear_down_wave(cfg: dict, wave: list[dict], log) -> None:
+    names = " ".join(_vllm_name(m["alias"]) for m in wave)
+    # vLLMs may be split across both Sparks; clear the names on each.
+    for sp in sc.sparks(cfg):
+        sc.run(sp, f"docker rm -f {names} 2>/dev/null; true", timeout=120)
+    sc.run(sc.head(cfg), f"docker rm -f {PROXY_NAME} 2>/dev/null; true", timeout=60)
+    log(f"[serving] wave torn down ({names})")
+
+
+def tear_down_all(cfg: dict, log) -> None:
+    """Best-effort: remove every Boardroom Map serving container + the network."""
+    for sp in sc.sparks(cfg):
+        sc.run(sp, "docker ps -aq --filter name=bm-vllm- | xargs -r docker rm -f; "
+                   f"docker rm -f {PROXY_NAME} 2>/dev/null; true", timeout=120)
+    remove_network(cfg, log)
+    log("[serving] all serving torn down")
+
+
+def plan_waves(cfg: dict, needed_aliases: set[str]) -> list[list[dict]]:
+    """Group the needed models into waves that respect per-Spark concurrency.
+
+    Head-Spark models are chunked into groups of `maxConcurrentModels`; secondary
+    models likewise. Wave i runs head-group-i and secondary-group-i together (they
+    sit on different GPUs)."""
+    catalog = {m["alias"]: m for m in (cfg.get("models") or [])}
+    cap = max(1, int(cfg.get("maxConcurrentModels", 1)))
+    primary, secondary = [], []
+    for alias in sorted(needed_aliases):
+        m = catalog.get(alias)
+        if not m:
+            continue
+        (secondary if m.get("spark") == "secondary" else primary).append(m)
+
+    def chunk(lst):
+        return [lst[i:i + cap] for i in range(0, len(lst), cap)]
+
+    pg, sg = chunk(primary), chunk(secondary)
+    waves = []
+    for i in range(max(len(pg), len(sg))):
+        wave = (pg[i] if i < len(pg) else []) + (sg[i] if i < len(sg) else [])
+        if wave:
+            waves.append(wave)
+    return waves
+
+
+def reviewer_proxy_base(cfg: dict) -> str:
+    """The OpenAI-compatible base URL reviewers use (by container name on the net)."""
+    return f"http://{PROXY_NAME}:{int(cfg['proxyPort'])}/v1"
+
+
+def health(cfg: dict) -> dict:
+    head = sc.head(cfg)
+    r = sc.run(head, "docker ps --filter name=bm-vllm- --filter name=boardroom-proxy "
+                     "--format '{{.Names}} {{.Status}}'", timeout=30)
+    return {"running": (r.stdout or "").strip().splitlines()}

orchestrator/spark_client.py

@@ -0,0 +1,163 @@
+"""SSH/rsync helpers for driving the Sparks from the Boardroom Map control plane.
+
+Shells out to the system `ssh`/`rsync` (installed in the image) rather than a
+Python SSH lib, so we get `docker logs -f` streaming for free and the exact same
+behavior a human would get from a shell. Also usable as a CLI:
+
+    python spark_client.py test     # probe GPU + images on the configured Spark(s)
+
+This mirrors the LLaMA-Factory / Nightshift services' spark_client.py so the SSH
+logic lives in one place and behaves identically across services.
+"""
+from __future__ import annotations
+
+import json
+import os
+import shlex
+import subprocess
+import sys
+from dataclasses import dataclass
+
+DATA_DIR = os.environ.get("BM_DATA_DIR", "/data")
+CONFIG_PATH = os.path.join(DATA_DIR, "config.json")
+KEY_PATH = os.path.join(DATA_DIR, "ssh", "id_spark")
+
+
+def load_config() -> dict:
+    with open(CONFIG_PATH) as f:
+        return json.load(f)
+
+
+def _ensure_key_perms() -> str:
+    """SSH refuses world-readable keys. Copy to a private 600 path at runtime."""
+    safe = "/tmp/id_spark"
+    if not os.path.exists(KEY_PATH):
+        raise FileNotFoundError(
+            f"SSH key not found at {KEY_PATH}. Run the 'Configure Sparks' action first."
+        )
+    with open(KEY_PATH, "rb") as src, open(safe, "wb") as dst:
+        dst.write(src.read())
+    os.chmod(safe, 0o600)
+    return safe
+
+
+@dataclass
+class Spark:
+    host: str
+    user: str
+    port: int
+    role: str = "primary"  # "primary" (head) or "secondary"
+
+    def ssh_base(self) -> list[str]:
+        key = _ensure_key_perms()
+        return [
+            "ssh", "-i", key, "-p", str(self.port),
+            "-o", "StrictHostKeyChecking=accept-new",
+            "-o", "BatchMode=yes",
+            "-o", "ConnectTimeout=15",
+            f"{self.user}@{self.host}",
+        ]
+
+
+def sparks(cfg: dict | None = None) -> list[Spark]:
+    cfg = cfg or load_config()
+    user = cfg.get("primarySparkUser", "nvidia")
+    port = int(cfg.get("sshPort", 22))
+    out = [Spark(cfg["primarySparkHost"], user, port, role="primary")]
+    if cfg.get("useBothSparks") and cfg.get("secondarySparkHost"):
+        out.append(Spark(cfg["secondarySparkHost"], user, port, role="secondary"))
+    return out
+
+
+def head(cfg: dict | None = None) -> Spark:
+    """The head Spark: hosts the model proxy, the network, and the reviewer panel."""
+    return sparks(cfg)[0]
+
+
+def by_role(cfg: dict, role: str) -> Spark:
+    """Return the Spark serving a given role ('primary'|'secondary'); falls back
+    to the head if the secondary isn't configured."""
+    for sp in sparks(cfg):
+        if sp.role == role:
+            return sp
+    return head(cfg)
+
+
+def run(spark: Spark, remote_cmd: str, timeout: int | None = None) -> subprocess.CompletedProcess:
+    """Run a shell command on the Spark, capturing output."""
+    return subprocess.run(
+        spark.ssh_base() + [remote_cmd],
+        capture_output=True, text=True, timeout=timeout,
+    )
+
+
+def stream(spark: Spark, remote_cmd: str):
+    """Yield stdout lines from a long-running remote command (e.g. docker logs -f)."""
+    proc = subprocess.Popen(
+        spark.ssh_base() + [remote_cmd],
+        stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1,
+    )
+    try:
+        assert proc.stdout is not None
+        for line in proc.stdout:
+            yield line
+    finally:
+        proc.terminate()
+
+
+def push_dir(spark: Spark, local_dir: str, remote_dir: str) -> subprocess.CompletedProcess:
+    """rsync a local dir up to the Spark."""
+    key = _ensure_key_perms()
+    ssh = f"ssh -i {key} -p {spark.port} -o StrictHostKeyChecking=accept-new -o BatchMode=yes"
+    run(spark, f"mkdir -p {shlex.quote(remote_dir)}")
+    return subprocess.run(
+        ["rsync", "-az", "-e", ssh,
+         local_dir.rstrip("/") + "/", f"{spark.user}@{spark.host}:{remote_dir.rstrip('/')}/"],
+        capture_output=True, text=True,
+    )
+
+
+def pull_dir(spark: Spark, remote_dir: str, local_dir: str) -> subprocess.CompletedProcess:
+    """rsync a remote dir back down to the StartOS volume."""
+    key = _ensure_key_perms()
+    ssh = f"ssh -i {key} -p {spark.port} -o StrictHostKeyChecking=accept-new -o BatchMode=yes"
+    os.makedirs(local_dir, exist_ok=True)
+    return subprocess.run(
+        ["rsync", "-az", "-e", ssh,
+         f"{spark.user}@{spark.host}:{remote_dir.rstrip('/')}/", local_dir.rstrip("/") + "/"],
+        capture_output=True, text=True,
+    )
+
+
+def test_cli() -> int:
+    cfg = load_config()
+    if not cfg.get("primarySparkHost"):
+        print("No Spark configured. Run the 'Configure Sparks' action first.")
+        return 1
+    serving = cfg.get("servingImage", "boardroom-vllm:latest")
+    reviewer = cfg.get("graderImage", "boardroom-grader:latest")
+    rc_all = 0
+    for sp in sparks(cfg):
+        print(f"== {sp.user}@{sp.host}:{sp.port} ({sp.role}) ==")
+        probe = (
+            "nvidia-smi -L && echo '---' && "
+            f"(docker image inspect {shlex.quote(serving)} >/dev/null 2>&1 "
+            f"&& echo 'vLLM image present: {serving}' || echo 'vLLM image MISSING') && "
+            f"(docker image inspect {shlex.quote(reviewer)} >/dev/null 2>&1 "
+            f"&& echo 'reviewer image present: {reviewer}' || echo 'reviewer image MISSING (build sandbox/ on the head Spark)')"
+        )
+        r = run(sp, probe, timeout=40)
+        print(r.stdout.strip() or "(no output)")
+        if r.returncode != 0:
+            rc_all = r.returncode
+            print(f"[error rc={r.returncode}] {r.stderr.strip()}")
+        print()
+    return rc_all
+
+
+if __name__ == "__main__":
+    cmd = sys.argv[1] if len(sys.argv) > 1 else "test"
+    if cmd == "test":
+        sys.exit(test_cli())
+    print(f"unknown command: {cmd}")
+    sys.exit(2)

orchestrator/templates/index.html

@@ -0,0 +1,155 @@
+<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="utf-8"/>
+  <meta name="viewport" content="width=device-width, initial-scale=1"/>
+  <title>Boardroom Map</title>
+  <style>
+    :root { --bg:#10131a; --panel:#181d27; --edge:#2a313f; --ink:#e9edf5; --dim:#97a1b5;
+            --accent:#c9a24b; --ok:#5fd08a; --warn:#ffcf66; --bad:#ff7a7a; }
+    * { box-sizing:border-box; }
+    body { margin:0; font:14px/1.5 system-ui,-apple-system,Segoe UI,Roboto,sans-serif;
+           background:var(--bg); color:var(--ink); }
+    header { display:flex; align-items:center; gap:12px; padding:18px 22px; border-bottom:1px solid var(--edge); }
+    header .seal { width:26px;height:26px;border-radius:50%;border:2px solid var(--accent);
+      display:flex;align-items:center;justify-content:center;color:var(--accent);font-weight:700;font-size:13px; }
+    h1 { font-size:18px; margin:0; letter-spacing:2px; }
+    .wrap { max-width:1100px; margin:0 auto; padding:22px; display:grid; gap:18px; grid-template-columns:1fr 1fr; }
+    .card { background:var(--panel); border:1px solid var(--edge); border-radius:14px; padding:16px 18px; }
+    .card.full { grid-column:1 / -1; }
+    .card h2 { margin:0 0 10px; font-size:13px; text-transform:uppercase; letter-spacing:1px; color:var(--dim); }
+    .pill { display:inline-block; padding:2px 10px; border-radius:999px; font-size:12px; font-weight:600; }
+    .pill.idle,.pill.done{background:#26303f;color:var(--dim)}
+    .pill.reviewing,.pill.synthesizing,.pill.collecting,.pill.extracting{background:#3b3414;color:var(--warn)}
+    .pill.error{background:#3b1414;color:var(--bad)}
+    .kv { display:flex; justify-content:space-between; padding:4px 0; border-bottom:1px dashed var(--edge); }
+    .kv:last-child{border:0} .kv .k{color:var(--dim)}
+    .dot{display:inline-block;width:8px;height:8px;border-radius:50%;margin-right:6px}
+    .dot.on{background:var(--ok)} .dot.off{background:#55608f}
+    pre { background:#0c0f16; border:1px solid var(--edge); border-radius:10px; padding:12px; margin:0;
+          max-height:340px; overflow:auto; font:12px/1.5 ui-monospace,Menlo,monospace; color:#cdd6ff; white-space:pre-wrap; }
+    .row{display:flex;gap:8px;flex-wrap:wrap;margin-top:10px}
+    button{background:#222a3a;color:var(--ink);border:1px solid var(--edge);border-radius:9px;
+      padding:8px 12px;font-size:13px;cursor:pointer} button:hover{background:#2c3650}
+    button.primary{background:#3a2f12;border-color:#6b551f;color:#ffdf9a}
+    .drop{border:1.5px dashed var(--edge);border-radius:12px;padding:18px;text-align:center;color:var(--dim);cursor:pointer}
+    .drop.hot{border-color:var(--accent);color:var(--ink)}
+    .badge{font-size:11px;color:var(--dim)}
+    .muted{color:var(--dim);font-size:12px}
+    a{color:#9ab8ff}
+  </style>
+</head>
+<body>
+  <header>
+    <div class="seal">§</div>
+    <h1>BOARDROOM MAP</h1>
+    <span id="phase" class="pill idle">idle</span>
+    <span class="muted" id="netmode"></span>
+  </header>
+
+  <div class="wrap">
+    <div class="card">
+      <h2>Documents</h2>
+      <div id="drop" class="drop">Drop confidential documents here, or click to choose<br>
+        <span class="badge">PDF · DOCX · TXT · MD</span></div>
+      <input id="file" type="file" multiple style="display:none"/>
+      <div id="inbox" style="margin-top:10px"></div>
+      <div class="row">
+        <button class="primary" onclick="runReview()">Run Review</button>
+        <button onclick="act('/api/inbox/clear','POST')">Clear Inbox</button>
+        <button onclick="refresh()">Refresh</button>
+      </div>
+    </div>
+
+    <div class="card">
+      <h2>Setup</h2>
+      <div id="ready"></div>
+    </div>
+
+    <div class="card">
+      <h2>Panel</h2>
+      <div id="panel"><div class="muted">No reviewers configured.</div></div>
+    </div>
+
+    <div class="card">
+      <h2>Serving / Job</h2>
+      <div id="job"><div class="muted">—</div></div>
+      <div class="row">
+        <button onclick="act('/api/reviewer/build-image','POST')">Build reviewer image</button>
+        <button onclick="act('/api/stop','POST')">Stop serving</button>
+      </div>
+    </div>
+
+    <div class="card full">
+      <h2>Activity log</h2>
+      <pre id="log">loading…</pre>
+    </div>
+
+    <div class="card full">
+      <h2>Latest report</h2>
+      <pre id="report">—</pre>
+    </div>
+  </div>
+
+<script>
+async function getJSON(u){ const r=await fetch(u); if(!r.ok) throw new Error(await r.text()); return r.json(); }
+async function act(u,m){ try{ const r=await fetch(u,{method:m}); const t=await r.text();
+  alert(r.ok? 'OK: '+t.slice(0,400) : 'Error: '+t.slice(0,400)); refresh(); }catch(e){ alert(e); } }
+function dot(on,label){ return `<div><span class="dot ${on?'on':'off'}"></span>${label}</div>`; }
+async function runReview(){ act('/api/run','POST'); }
+
+const drop=document.getElementById('drop'), file=document.getElementById('file');
+drop.onclick=()=>file.click();
+file.onchange=()=>upload(file.files);
+['dragover','dragenter'].forEach(e=>drop.addEventListener(e,ev=>{ev.preventDefault();drop.classList.add('hot');}));
+['dragleave','drop'].forEach(e=>drop.addEventListener(e,ev=>{ev.preventDefault();drop.classList.remove('hot');}));
+drop.addEventListener('drop',ev=>{ if(ev.dataTransfer.files.length) upload(ev.dataTransfer.files); });
+async function upload(files){
+  const fd=new FormData(); for(const f of files) fd.append('files',f);
+  try{ const r=await fetch('/api/upload',{method:'POST',body:fd});
+    if(!r.ok) alert('Upload failed: '+(await r.text()).slice(0,300)); refresh(); }catch(e){ alert(e); }
+}
+
+async function refresh(){
+  try{
+    const s = await getJSON('/api/status');
+    const rt = s.runtime||{};
+    const ph = document.getElementById('phase');
+    ph.textContent = rt.phase||'idle'; ph.className = 'pill '+(rt.phase||'idle');
+    document.getElementById('netmode').textContent =
+      (s.networkMode==='airgapped'?'air-gapped':'local-services')
+      + (s.synthesis?' · synthesis on':'') + (s.autoRunOnDrop?' · auto-run':'');
+
+    document.getElementById('inbox').innerHTML = (s.inbox||[]).length
+      ? (s.inbox||[]).map(f=>`<div class="kv"><span>${f.name} ${f.supported?'':'<span class="badge">unsupported</span>'}</span><span class="muted">${(f.bytes/1024).toFixed(0)} KB</span></div>`).join('')
+      : '<div class="muted">Inbox empty.</div>';
+
+    document.getElementById('ready').innerHTML =
+      dot(s.configured.sparks,'Sparks configured') +
+      dot(s.configured.models>0,`${s.configured.models} model(s)`) +
+      dot(s.configured.reviewers>0,`${s.configured.reviewers} reviewer(s)`) +
+      `<div class="kv"><span class="k">network mode</span><span>${s.networkMode}</span></div>` +
+      `<div class="kv"><span class="k">wipe docs after</span><span>${s.wipeRemoteDocs?'yes':'no'}</span></div>` +
+      (s.models||[]).map(m=>`<div class="kv"><span>${m.alias}</span><span class="muted">${m.hfModel} · ${m.spark}</span></div>`).join('');
+
+    document.getElementById('panel').innerHTML = (s.panel||[]).length
+      ? (s.panel||[]).map(r=>`<div class="kv"><span>${r.name} ${r.known?'':'<span class="badge">unknown model</span>'}</span><span class="muted">${r.model}${r.persona?' · persona ✓':''}</span></div>`).join('')
+      : '<div class="muted">No reviewers configured.</div>';
+
+    const panelRt=(rt.panel||[]);
+    document.getElementById('job').innerHTML =
+      `<div class="kv"><span class="k">job</span><span>${rt.job_id||'—'}</span></div>` +
+      (rt.waves_total?`<div class="kv"><span class="k">wave</span><span>${rt.wave_index}/${rt.waves_total}</span></div>`:'') +
+      (rt.message?`<div class="muted" style="margin-top:6px">${rt.message}</div>`:'') +
+      panelRt.map(p=>`<div class="kv"><span>${p.name}</span><span class="muted">${p.status||''}</span></div>`).join('');
+
+    const ev = await getJSON('/api/events');
+    document.getElementById('log').textContent = (ev.events||[]).slice(-200).join('\n') || '(no activity yet)';
+    try{ const rep = await (await fetch('/api/report')).text();
+      document.getElementById('report').textContent = rep; }catch(e){}
+  }catch(e){ document.getElementById('log').textContent = 'status error: '+e; }
+}
+refresh(); setInterval(refresh, 6000);
+</script>
+</body>
+</html>