"""Launch the reviewer panel on the head Spark over SSH. Each reviewer is a ONE-SHOT, hardened, read-only container: it reads the document text mounted at /docs, runs its model (through the on-Spark proxy) under its persona + the shared rubric, writes a single report to /out/.md, and exits. There is no shared writable workspace and no git — reviewers cannot alter the documents or each other's reports. Sandbox (per the operator's confidentiality requirement): * non-root, --cap-drop ALL, --security-opt no-new-privileges * read-only rootfs + small writable tmpfs; only /docs (ro), /persona (ro), and /out (rw, this reviewer's report dir) are mounted * NO docker socket, cpu/mem/pid caps * attached to the per-job network — in airgapped mode that network is --internal, so the reviewer can reach ONLY the model proxy, never the internet Reviewers hold no credentials beyond a dummy proxy key. """ from __future__ import annotations import os import re import shlex import spark_client as sc import serving SANDBOX_SRC = os.path.join(os.path.dirname(os.path.abspath(__file__)), "sandbox") def ensure_reviewer_image(cfg: dict, log) -> None: """Build the reviewer image on the head Spark if missing (aarch64, native).""" head = sc.head(cfg) image = cfg["graderImage"] q = shlex.quote r = sc.run(head, f"docker image inspect {q(image)} >/dev/null 2>&1 && echo PRESENT || echo MISSING", timeout=30) if "PRESENT" in (r.stdout or ""): log(f"[reviewers] image {image} already present on {head.host}") return if not os.path.isdir(SANDBOX_SRC): raise RuntimeError(f"reviewer build context missing at {SANDBOX_SRC} (image not baked in?)") remote_dir = f"{cfg['remoteWorkDir']}/sandbox-build" log(f"[reviewers] building reviewer image {image} on {head.host} (first run; a few minutes)…") push = sc.push_dir(head, SANDBOX_SRC, remote_dir) if push.returncode != 0: raise RuntimeError(f"rsync reviewer build context to {head.host} failed: {push.stderr}") b = sc.run(head, f"cd {q(remote_dir)} && IMAGE={q(image)} bash build.sh", timeout=1800) if b.returncode != 0: raise RuntimeError(f"reviewer image build failed on {head.host}: {b.stderr or b.stdout}") log(f"[reviewers] reviewer image built: {image}") def slug(name: str) -> str: s = re.sub(r"[^A-Za-z0-9]+", "-", (name or "").strip().lower()).strip("-") return s or "reviewer" def roster(cfg: dict) -> list[dict]: """Reviewer roster from config. Each: {rid, name, model alias, persona, temperature}.""" out: list[dict] = [] seen: dict[str, int] = {} for w in (cfg.get("reviewers") or []): name = (w.get("name") or "reviewer").strip() rid = slug(name) if rid in seen: seen[rid] += 1 rid = f"{rid}-{seen[rid]}" else: seen[rid] = 1 out.append({ "rid": rid, "name": name, "model": (w.get("model") or "").strip(), "persona": (w.get("persona") or "").strip(), "temperature": w.get("temperature"), }) return out def _container(cfg: dict, jobdir: str, rid: str, name: str, model: str, persona: str, temperature, role: str, extra_mounts: str = "") -> str: """docker run command for one reviewer/synthesizer container (detached, one-shot).""" q = shlex.quote net = serving.net_name(cfg) base = serving.reviewer_proxy_base(cfg) searxng = (cfg.get("searxngUrl") or "").strip() persona_path = f"{jobdir}/personas/{rid}.md" env = ( f"-e BM_REVIEWER_ID={q(rid)} -e BM_REVIEWER_NAME={q(name)} -e BM_ROLE={q(role)} " f"-e BM_MODEL={q(model)} -e BM_LLM_BASE={q(base)} -e BM_LLM_KEY=sk-local " f"-e BM_MAX_MODEL_LEN={q(str(int(cfg.get('maxModelLen', 32768))))} " f"-e HOME=/home/rev " ) if temperature is not None: env += f"-e BM_TEMPERATURE={q(str(temperature))} " # web_search is offered ONLY in local_services mode with a SearXNG URL. if cfg.get("networkMode") == "local_services" and searxng: env += f"-e BM_SEARXNG_URL={q(searxng)} " harden = ( "--user 1000:1000 --security-opt no-new-privileges --cap-drop ALL " "--read-only --tmpfs /tmp:size=256m --tmpfs /home/rev:size=128m " "--pids-limit 256 --memory 6g --cpus 4" ) mounts = ( f"-v {q(jobdir)}/docs:/docs:ro " f"-v {q(jobdir)}/out:/out " f"-v {q(persona_path)}:/persona/PERSONA.md:ro " + extra_mounts ) cname = f"bm-grader-{rid}" return ( f"docker rm -f {cname} >/dev/null 2>&1; " f"docker run -d --name {cname} --network {q(net)} {harden} {env} {mounts} {q(cfg['graderImage'])}" ) def _write_persona(cfg: dict, jobdir: str, rid: str, persona: str) -> None: q = shlex.quote sc.run(sc.head(cfg), f"mkdir -p {q(jobdir)}/personas && printf '%s' {q(persona)} > {q(jobdir)}/personas/{rid}.md", timeout=30) def run_wave_reviewers(cfg: dict, jobdir: str, panel: list[dict], rubric: str, log, wait_timeout: int = 1800) -> list[dict]: """Launch every reviewer in `panel` (already filtered to this wave's models), wait for them to finish, and report status. Reports land in /out.""" head = sc.head(cfg) q = shlex.quote # Rubric is shared; write it once into the job dir, mounted into every container. sc.run(head, f"mkdir -p {q(jobdir)}/out && printf '%s' {q(rubric)} > {q(jobdir)}/RUBRIC.md", timeout=30) launched = [] for r in panel: _write_persona(cfg, jobdir, r["rid"], r["persona"]) cmd = _container(cfg, jobdir, r["rid"], r["name"], r["model"], r["persona"], r.get("temperature"), role="reviewer", extra_mounts=f"-v {q(jobdir)}/RUBRIC.md:/RUBRIC.md:ro ") res = sc.run(head, cmd, timeout=120) if res.returncode != 0: log(f"[reviewers] launch {r['rid']} FAILED: {res.stderr or res.stdout}") launched.append({**r, "ok": False, "error": (res.stderr or res.stdout)[:300]}) continue log(f"[reviewers] up: {r['rid']} -> {r['model']}") launched.append({**r, "ok": True, "error": ""}) # Wait for each launched container to exit (they run in parallel; waiting # sequentially still finishes when the slowest does). results = [] for r in launched: if not r["ok"]: results.append(r) continue cname = f"bm-grader-{r['rid']}" w = sc.run(head, f"docker wait {cname}", timeout=wait_timeout) code = (w.stdout or "").strip() out_check = sc.run(head, f"test -s {q(jobdir)}/out/{q(r['rid'])}.md && echo OK || echo MISSING", timeout=30) wrote = "OK" in (out_check.stdout or "") log(f"[reviewers] {r['rid']} exited (code={code or '?'}), report={'written' if wrote else 'MISSING'}") sc.run(head, f"docker rm -f {cname} 2>/dev/null; true", timeout=30) results.append({**r, "exit": code, "report": wrote}) return results