Scaffold: fork of Chambers architecture, renamed to Boardroom Map

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-06 13:10:25 -05:00
commit 1dde915540
48 changed files with 4025 additions and 0 deletions
@@ -0,0 +1,92 @@
+"""Preflight checks — fail LOUD before launching reviewers, so a dead model
+endpoint is caught immediately instead of after a container spins fruitlessly.
+
+The model proxy has no published host port (it lives on the per-job Docker
+network so that air-gapped reviewers can reach it without the host exposing
+anything). So we probe it the same way a reviewer would: from a throwaway
+container attached to the same network.
+"""
+from __future__ import annotations
+
+import shlex
+
+import spark_client as sc
+import serving
+
+
+def _probe_in_net(cfg: dict, inner_cmd: str, timeout: int) -> sc.subprocess.CompletedProcess:
+    """Run a shell command inside a throwaway container on the per-job network,
+    using the reviewer image (it has curl)."""
+    q = shlex.quote
+    net = serving.net_name(cfg)
+    image = cfg["graderImage"]
+    cmd = (
+        f"docker run --rm --network {q(net)} --entrypoint sh {q(image)} "
+        f"-c {q(inner_cmd)}"
+    )
+    return sc.run(sc.head(cfg), cmd, timeout=timeout)
+
+
+def check_wave(cfg: dict, wave: list[dict], log) -> None:
+    """The proxy answers AND each model alias in the wave returns a completion."""
+    base = serving.reviewer_proxy_base(cfg).rstrip("/")  # http://boardroom-proxy:PORT/v1
+
+    # 1. Proxy reachable at all.
+    reach = f"curl -sf -m 8 {shlex.quote(base + '/models')} -o /dev/null && echo OK || echo FAIL"
+    r = _probe_in_net(cfg, reach, timeout=40)
+    if "OK" not in (r.stdout or ""):
+        raise RuntimeError(
+            f"model proxy not answering at {base} from inside the network. "
+            f"Check that the router container ({serving.PROXY_NAME}) came up.")
+    log("[preflight] model proxy answering")
+
+    # 2. Each alias must actually return a completion.
+    dead = []
+    for m in wave:
+        alias = m["alias"]
+        payload = '{"model":"%s","messages":[{"role":"user","content":"ping"}],"max_tokens":1}' % alias
+        probe = (
+            f"curl -sf -m 60 -X POST {shlex.quote(base + '/chat/completions')} "
+            f"-H 'content-type: application/json' -H 'authorization: Bearer sk-local' "
+            f"-d {shlex.quote(payload)} 2>/dev/null | head -c 600"
+        )
+        rr = _probe_in_net(cfg, probe, timeout=90)
+        out = rr.stdout or ""
+        if '"choices"' not in out and '"content"' not in out:
+            dead.append(alias)
+        else:
+            log(f"[preflight] model {alias} responded")
+    if dead:
+        raise RuntimeError(
+            "These models are not responding through the proxy: " + ", ".join(dead) +
+            ". Check the corresponding vLLM container(s) on the Spark — they may have "
+            "failed to load, OOM'd, or (in air-gapped mode) the model isn't in the HF cache.")
+
+
+def check_searxng(cfg: dict, log) -> None:
+    """In local_services mode, verify the reviewers' optional web_search backend.
+
+    StartOS exposes SearXNG only over HTTPS with a SELF-SIGNED cert (the lesson
+    from Nightshift's first deploy), so the probe uses `curl -sfk`, and it asserts
+    the JSON format is enabled (HTML back == json format off). This is NON-fatal:
+    web_search is an optional enhancement in Boardroom Map, so a broken SearXNG only
+    warns rather than failing the whole review (unlike Nightshift, where research
+    depended on it)."""
+    if cfg.get("networkMode") != "local_services":
+        return
+    sx = (cfg.get("searxngUrl") or "").strip().rstrip("/")
+    if not sx:
+        return
+    head = sc.head(cfg)
+    probe = f"curl -sfk -m 12 {shlex.quote(sx + '/search?q=boardroom&format=json')} 2>/dev/null | head -c 400"
+    r = sc.run(head, probe, timeout=25)
+    out = (r.stdout or "").strip()
+    if not out:
+        log(f"[preflight] WARNING: SearXNG not reachable from the head Spark at {sx} — "
+            "reviewers' web_search will be unavailable (reviews still run).")
+        return
+    if not out.lstrip().startswith("{") and '"results"' not in out:
+        log(f"[preflight] WARNING: SearXNG at {sx} did not return JSON (enable the json "
+            "format in settings.yml) — web_search may not work.")
+        return
+    log("[preflight] SearXNG reachable with JSON enabled")