boardroom-map/orchestrator/graders.py

"""Launch the reviewer panel on the head Spark over SSH.

Each reviewer is a ONE-SHOT, hardened, read-only container: it reads the document
text mounted at /docs, runs its model (through the on-Spark proxy) under its
persona + the shared rubric, writes a single report to /out/<id>.md, and exits.
There is no shared writable workspace and no git — reviewers cannot alter the
documents or each other's reports.

Sandbox (per the operator's confidentiality requirement):
  * non-root, --cap-drop ALL, --security-opt no-new-privileges
  * read-only rootfs + small writable tmpfs; only /docs (ro), /persona (ro), and
    /out (rw, this reviewer's report dir) are mounted
  * NO docker socket, cpu/mem/pid caps
  * attached to the per-job network — in airgapped mode that network is
    --internal, so the reviewer can reach ONLY the model proxy, never the internet

Reviewers hold no credentials beyond a dummy proxy key.
"""
from __future__ import annotations

import os
import re
import shlex

import spark_client as sc
import serving

SANDBOX_SRC = os.path.join(os.path.dirname(os.path.abspath(__file__)), "sandbox")


def ensure_reviewer_image(cfg: dict, log) -> None:
    """Build the reviewer image on the head Spark if missing (aarch64, native)."""
    head = sc.head(cfg)
    image = cfg["graderImage"]
    q = shlex.quote
    r = sc.run(head, f"docker image inspect {q(image)} >/dev/null 2>&1 && echo PRESENT || echo MISSING",
               timeout=30)
    if "PRESENT" in (r.stdout or ""):
        log(f"[reviewers] image {image} already present on {head.host}")
        return
    if not os.path.isdir(SANDBOX_SRC):
        raise RuntimeError(f"reviewer build context missing at {SANDBOX_SRC} (image not baked in?)")
    remote_dir = f"{cfg['remoteWorkDir']}/sandbox-build"
    log(f"[reviewers] building reviewer image {image} on {head.host} (first run; a few minutes)…")
    push = sc.push_dir(head, SANDBOX_SRC, remote_dir)
    if push.returncode != 0:
        raise RuntimeError(f"rsync reviewer build context to {head.host} failed: {push.stderr}")
    b = sc.run(head, f"cd {q(remote_dir)} && IMAGE={q(image)} bash build.sh", timeout=1800)
    if b.returncode != 0:
        raise RuntimeError(f"reviewer image build failed on {head.host}: {b.stderr or b.stdout}")
    log(f"[reviewers] reviewer image built: {image}")


def slug(name: str) -> str:
    s = re.sub(r"[^A-Za-z0-9]+", "-", (name or "").strip().lower()).strip("-")
    return s or "reviewer"


def roster(cfg: dict) -> list[dict]:
    """Reviewer roster from config. Each: {rid, name, model alias, persona, temperature}."""
    out: list[dict] = []
    seen: dict[str, int] = {}
    for w in (cfg.get("reviewers") or []):
        name = (w.get("name") or "reviewer").strip()
        rid = slug(name)
        if rid in seen:
            seen[rid] += 1
            rid = f"{rid}-{seen[rid]}"
        else:
            seen[rid] = 1
        out.append({
            "rid": rid, "name": name, "model": (w.get("model") or "").strip(),
            "persona": (w.get("persona") or "").strip(),
            "temperature": w.get("temperature"),
        })
    return out


def _container(cfg: dict, jobdir: str, rid: str, name: str, model: str, persona: str,
               temperature, role: str, extra_mounts: str = "") -> str:
    """docker run command for one reviewer/synthesizer container (detached, one-shot)."""
    q = shlex.quote
    net = serving.net_name(cfg)
    base = serving.reviewer_proxy_base(cfg)
    searxng = (cfg.get("searxngUrl") or "").strip()
    persona_path = f"{jobdir}/personas/{rid}.md"

    env = (
        f"-e BM_REVIEWER_ID={q(rid)} -e BM_REVIEWER_NAME={q(name)} -e BM_ROLE={q(role)} "
        f"-e BM_MODEL={q(model)} -e BM_LLM_BASE={q(base)} -e BM_LLM_KEY=sk-local "
        f"-e BM_MAX_MODEL_LEN={q(str(int(cfg.get('maxModelLen', 32768))))} "
        f"-e HOME=/home/rev "
    )
    if temperature is not None:
        env += f"-e BM_TEMPERATURE={q(str(temperature))} "
    # web_search is offered ONLY in local_services mode with a SearXNG URL.
    if cfg.get("networkMode") == "local_services" and searxng:
        env += f"-e BM_SEARXNG_URL={q(searxng)} "

    harden = (
        "--user 1000:1000 --security-opt no-new-privileges --cap-drop ALL "
        "--read-only --tmpfs /tmp:size=256m --tmpfs /home/rev:size=128m "
        "--pids-limit 256 --memory 6g --cpus 4"
    )
    mounts = (
        f"-v {q(jobdir)}/docs:/docs:ro "
        f"-v {q(jobdir)}/out:/out "
        f"-v {q(persona_path)}:/persona/PERSONA.md:ro "
        + extra_mounts
    )
    cname = f"bm-grader-{rid}"
    return (
        f"docker rm -f {cname} >/dev/null 2>&1; "
        f"docker run -d --name {cname} --network {q(net)} {harden} {env} {mounts} {q(cfg['graderImage'])}"
    )


def _write_persona(cfg: dict, jobdir: str, rid: str, persona: str) -> None:
    q = shlex.quote
    sc.run(sc.head(cfg),
           f"mkdir -p {q(jobdir)}/personas && printf '%s' {q(persona)} > {q(jobdir)}/personas/{rid}.md",
           timeout=30)


def run_wave_reviewers(cfg: dict, jobdir: str, panel: list[dict], rubric: str, log,
                       wait_timeout: int = 1800) -> list[dict]:
    """Launch every reviewer in `panel` (already filtered to this wave's models),
    wait for them to finish, and report status. Reports land in <jobdir>/out."""
    head = sc.head(cfg)
    q = shlex.quote
    # Rubric is shared; write it once into the job dir, mounted into every container.
    sc.run(head, f"mkdir -p {q(jobdir)}/out && printf '%s' {q(rubric)} > {q(jobdir)}/RUBRIC.md", timeout=30)
    launched = []
    for r in panel:
        _write_persona(cfg, jobdir, r["rid"], r["persona"])
        cmd = _container(cfg, jobdir, r["rid"], r["name"], r["model"], r["persona"],
                         r.get("temperature"), role="reviewer",
                         extra_mounts=f"-v {q(jobdir)}/RUBRIC.md:/RUBRIC.md:ro ")
        res = sc.run(head, cmd, timeout=120)
        if res.returncode != 0:
            log(f"[reviewers] launch {r['rid']} FAILED: {res.stderr or res.stdout}")
            launched.append({**r, "ok": False, "error": (res.stderr or res.stdout)[:300]})
            continue
        log(f"[reviewers] up: {r['rid']} -> {r['model']}")
        launched.append({**r, "ok": True, "error": ""})

    # Wait for each launched container to exit (they run in parallel; waiting
    # sequentially still finishes when the slowest does).
    results = []
    for r in launched:
        if not r["ok"]:
            results.append(r)
            continue
        cname = f"bm-grader-{r['rid']}"
        w = sc.run(head, f"docker wait {cname}", timeout=wait_timeout)
        code = (w.stdout or "").strip()
        out_check = sc.run(head, f"test -s {q(jobdir)}/out/{q(r['rid'])}.md && echo OK || echo MISSING", timeout=30)
        wrote = "OK" in (out_check.stdout or "")
        log(f"[reviewers] {r['rid']} exited (code={code or '?'}), report={'written' if wrote else 'MISSING'}")
        sc.run(head, f"docker rm -f {cname} 2>/dev/null; true", timeout=30)
        results.append({**r, "exit": code, "report": wrote})
    return results