boardroom-map/orchestrator/preflight.py

"""Preflight checks — fail LOUD before launching reviewers, so a dead model
endpoint is caught immediately instead of after a container spins fruitlessly.

The model proxy has no published host port (it lives on the per-job Docker
network so that air-gapped reviewers can reach it without the host exposing
anything). So we probe it the same way a reviewer would: from a throwaway
container attached to the same network.
"""
from __future__ import annotations

import shlex
import time

import spark_client as sc
import serving


def _probe_in_net(cfg: dict, inner_cmd: str, timeout: int) -> sc.subprocess.CompletedProcess:
    """Run a shell command inside a throwaway container on the per-job network,
    using the reviewer image (it has curl)."""
    q = shlex.quote
    net = serving.net_name(cfg)
    image = cfg["graderImage"]
    cmd = (
        f"docker run --rm --network {q(net)} --entrypoint sh {q(image)} "
        f"-c {q(inner_cmd)}"
    )
    return sc.run(sc.head(cfg), cmd, timeout=timeout)


def _container_running(cfg: dict, name: str) -> bool:
    """True only for a stably running container — a crash-looping one under
    `--restart unless-stopped` reports Status "running"/"restarting" between
    crashes, so treat repeated restarts as dead."""
    cmd = ("docker inspect -f '{{.State.Status}} {{.RestartCount}}' "
           + shlex.quote(name) + " 2>/dev/null")
    r = sc.run(sc.head(cfg), cmd, timeout=30)
    parts = (r.stdout or "").strip().split()
    if len(parts) != 2:
        return False
    status, restarts = parts[0], parts[1]
    if status != "running":
        return False
    try:
        return int(restarts) < 3
    except ValueError:
        return False


def _container_logs(cfg: dict, name: str, tail: int = 30) -> str:
    cmd = f"docker logs --tail {tail} " + shlex.quote(name) + " 2>&1"
    r = sc.run(sc.head(cfg), cmd, timeout=30)
    return (r.stdout or "").strip()


def check_wave(cfg: dict, wave: list[dict], log, timeout: int = 900) -> None:
    """The proxy answers AND each model alias in the wave returns a completion.

    Both are POLLED up to `timeout` seconds: `docker run -d` returns long before
    LiteLLM has booted (~10-30s) or vLLM has loaded a 31B model into GPU memory
    (minutes). A crashed container fails fast with its log tail instead of
    burning the whole timeout. The proxy requires its master key even on
    /models, so every probe authenticates."""
    base = serving.reviewer_proxy_base(cfg).rstrip("/")  # http://boardroom-proxy:PORT/v1
    deadline = time.time() + timeout

    # 1. Proxy reachable (LiteLLM boot).
    reach = (
        f"curl -sf -m 8 -H 'authorization: Bearer sk-local' "
        f"{shlex.quote(base + '/models')} -o /dev/null && echo OK || echo FAIL"
    )
    while True:
        r = _probe_in_net(cfg, reach, timeout=40)
        if "OK" in (r.stdout or ""):
            log("[preflight] model proxy answering")
            break
        if not _container_running(cfg, serving.PROXY_NAME):
            raise RuntimeError(
                f"router container {serving.PROXY_NAME} is not running. Last log lines:\n"
                + _container_logs(cfg, serving.PROXY_NAME))
        if time.time() > deadline:
            raise RuntimeError(
                f"model proxy did not answer at {base} within {timeout}s "
                f"(container is up — check its logs on the Spark).")
        log("[preflight] proxy still booting…")
        time.sleep(10)

    # 2. Each alias must actually return a completion (this is what waits out
    #    the multi-minute vLLM model load).
    for m in wave:
        alias = m["alias"]
        vllm_name = f"bm-vllm-{alias}"
        payload = '{"model":"%s","messages":[{"role":"user","content":"ping"}],"max_tokens":1}' % alias
        probe = (
            f"curl -sf -m 60 -X POST {shlex.quote(base + '/chat/completions')} "
            f"-H 'content-type: application/json' -H 'authorization: Bearer sk-local' "
            f"-d {shlex.quote(payload)} 2>/dev/null | head -c 600"
        )
        attempt = 0
        while True:
            rr = _probe_in_net(cfg, probe, timeout=90)
            out = rr.stdout or ""
            if '"choices"' in out or '"content"' in out:
                log(f"[preflight] model {alias} responded")
                break
            if not _container_running(cfg, vllm_name):
                raise RuntimeError(
                    f"vLLM container {vllm_name} is not running — it likely crashed "
                    f"during startup or model load. Last log lines:\n"
                    + _container_logs(cfg, vllm_name))
            if time.time() > deadline:
                raise RuntimeError(
                    f"model {alias} did not answer through the proxy within {timeout}s. "
                    f"Last probe output: {out[:200] or '(empty)'}")
            attempt += 1
            if attempt % 4 == 1:
                log(f"[preflight] {alias} still loading…")
            time.sleep(15)


def check_searxng(cfg: dict, log) -> None:
    """In local_services mode, verify the reviewers' optional web_search backend.

    StartOS exposes SearXNG only over HTTPS with a SELF-SIGNED cert (the lesson
    from Nightshift's first deploy), so the probe uses `curl -sfk`, and it asserts
    the JSON format is enabled (HTML back == json format off). This is NON-fatal:
    web_search is an optional enhancement in Boardroom Map, so a broken SearXNG only
    warns rather than failing the whole review (unlike Nightshift, where research
    depended on it)."""
    if cfg.get("networkMode") != "local_services":
        return
    sx = (cfg.get("searxngUrl") or "").strip().rstrip("/")
    if not sx:
        return
    head = sc.head(cfg)
    probe = f"curl -sfk -m 12 {shlex.quote(sx + '/search?q=boardroom&format=json')} 2>/dev/null | head -c 400"
    r = sc.run(head, probe, timeout=25)
    out = (r.stdout or "").strip()
    if not out:
        log(f"[preflight] WARNING: SearXNG not reachable from the head Spark at {sx} — "
            "reviewers' web_search will be unavailable (reviews still run).")
        return
    if not out.lstrip().startswith("{") and '"results"' not in out:
        log(f"[preflight] WARNING: SearXNG at {sx} did not return JSON (enable the json "
            "format in settings.yml) — web_search may not work.")
        return
    log("[preflight] SearXNG reachable with JSON enabled")