"""Preflight checks — fail LOUD before launching reviewers, so a dead model endpoint is caught immediately instead of after a container spins fruitlessly. The model proxy has no published host port (it lives on the per-job Docker network so that air-gapped reviewers can reach it without the host exposing anything). So we probe it the same way a reviewer would: from a throwaway container attached to the same network. """ from __future__ import annotations import shlex import spark_client as sc import serving def _probe_in_net(cfg: dict, inner_cmd: str, timeout: int) -> sc.subprocess.CompletedProcess: """Run a shell command inside a throwaway container on the per-job network, using the reviewer image (it has curl).""" q = shlex.quote net = serving.net_name(cfg) image = cfg["graderImage"] cmd = ( f"docker run --rm --network {q(net)} --entrypoint sh {q(image)} " f"-c {q(inner_cmd)}" ) return sc.run(sc.head(cfg), cmd, timeout=timeout) def check_wave(cfg: dict, wave: list[dict], log) -> None: """The proxy answers AND each model alias in the wave returns a completion.""" base = serving.reviewer_proxy_base(cfg).rstrip("/") # http://boardroom-proxy:PORT/v1 # 1. Proxy reachable at all. reach = f"curl -sf -m 8 {shlex.quote(base + '/models')} -o /dev/null && echo OK || echo FAIL" r = _probe_in_net(cfg, reach, timeout=40) if "OK" not in (r.stdout or ""): raise RuntimeError( f"model proxy not answering at {base} from inside the network. " f"Check that the router container ({serving.PROXY_NAME}) came up.") log("[preflight] model proxy answering") # 2. Each alias must actually return a completion. dead = [] for m in wave: alias = m["alias"] payload = '{"model":"%s","messages":[{"role":"user","content":"ping"}],"max_tokens":1}' % alias probe = ( f"curl -sf -m 60 -X POST {shlex.quote(base + '/chat/completions')} " f"-H 'content-type: application/json' -H 'authorization: Bearer sk-local' " f"-d {shlex.quote(payload)} 2>/dev/null | head -c 600" ) rr = _probe_in_net(cfg, probe, timeout=90) out = rr.stdout or "" if '"choices"' not in out and '"content"' not in out: dead.append(alias) else: log(f"[preflight] model {alias} responded") if dead: raise RuntimeError( "These models are not responding through the proxy: " + ", ".join(dead) + ". Check the corresponding vLLM container(s) on the Spark — they may have " "failed to load, OOM'd, or (in air-gapped mode) the model isn't in the HF cache.") def check_searxng(cfg: dict, log) -> None: """In local_services mode, verify the reviewers' optional web_search backend. StartOS exposes SearXNG only over HTTPS with a SELF-SIGNED cert (the lesson from Nightshift's first deploy), so the probe uses `curl -sfk`, and it asserts the JSON format is enabled (HTML back == json format off). This is NON-fatal: web_search is an optional enhancement in Boardroom Map, so a broken SearXNG only warns rather than failing the whole review (unlike Nightshift, where research depended on it).""" if cfg.get("networkMode") != "local_services": return sx = (cfg.get("searxngUrl") or "").strip().rstrip("/") if not sx: return head = sc.head(cfg) probe = f"curl -sfk -m 12 {shlex.quote(sx + '/search?q=boardroom&format=json')} 2>/dev/null | head -c 400" r = sc.run(head, probe, timeout=25) out = (r.stdout or "").strip() if not out: log(f"[preflight] WARNING: SearXNG not reachable from the head Spark at {sx} — " "reviewers' web_search will be unavailable (reviews still run).") return if not out.lstrip().startswith("{") and '"results"' not in out: log(f"[preflight] WARNING: SearXNG at {sx} did not return JSON (enable the json " "format in settings.yml) — web_search may not work.") return log("[preflight] SearXNG reachable with JSON enabled")