Implement BDEF v1.1 grading: scoring core, per-deck pipeline, ledger, dashboard, StartOS layer
- Deterministic scoring.py (quant 60 / qual 40 / flags -15, profitability heaviest) - Per-company JSON ledger with forecast-target chaining deck N-1 -> N - Single-shot sandbox agent with guided-JSON fallback ladder (no tool loop) - Portfolio dashboard with sparklines, KPI hit rates, BDEF category bars - 48 unit tests green; endpoints smoke-tested; npm check+build green Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Jonathan Kirkwood
co-authored by
Claude Fable 5
Claude Fable 5
parent
1dde915540
commit
b1d7aed9f4
+158
-82
@@ -1,67 +1,87 @@
|
||||
"""Launch the reviewer panel on the head Spark over SSH.
|
||||
"""Launch the grading panel (and the stage-1 extractor) on the head Spark.
|
||||
|
||||
Each reviewer is a ONE-SHOT, hardened, read-only container: it reads the document
|
||||
text mounted at /docs, runs its model (through the on-Spark proxy) under its
|
||||
persona + the shared rubric, writes a single report to /out/<id>.md, and exits.
|
||||
There is no shared writable workspace and no git — reviewers cannot alter the
|
||||
documents or each other's reports.
|
||||
Each role agent is a ONE-SHOT, hardened, read-only container running
|
||||
sandbox/grader_agent.py: it reads the deck text mounted at /docs, runs its model
|
||||
(through the on-Spark proxy) under its persona + the BDEF rubric, writes exactly
|
||||
one output file to /out, and exits. There is no tool loop and no shared writable
|
||||
workspace — agents cannot alter the deck text or each other's reports.
|
||||
|
||||
Per-deck mount contract (remote paths under {remoteWorkDir}/jobs/<job>/<company>/<deck>):
|
||||
docs/ -> /docs:ro
|
||||
BDEF.md -> /BDEF.md:ro
|
||||
schemas/<role>.schema.json -> /schema.json:ro (extractor|grades)
|
||||
personas/<rid>.md -> /persona/PERSONA.md:ro
|
||||
out/ -> /out:rw
|
||||
|
||||
Sandbox (per the operator's confidentiality requirement):
|
||||
* non-root, --cap-drop ALL, --security-opt no-new-privileges
|
||||
* read-only rootfs + small writable tmpfs; only /docs (ro), /persona (ro), and
|
||||
/out (rw, this reviewer's report dir) are mounted
|
||||
* read-only rootfs + small writable tmpfs (/tmp, /home/rev)
|
||||
* NO docker socket, cpu/mem/pid caps
|
||||
* attached to the per-job network — in airgapped mode that network is
|
||||
--internal, so the reviewer can reach ONLY the model proxy, never the internet
|
||||
--internal, so the agent can reach ONLY the model proxy, never the internet
|
||||
|
||||
Reviewers hold no credentials beyond a dummy proxy key.
|
||||
Agents hold no credentials beyond a dummy proxy key.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
import shutil
|
||||
|
||||
import spark_client as sc
|
||||
import bm_config
|
||||
import prompts
|
||||
import serving
|
||||
import spark_client as sc
|
||||
|
||||
SANDBOX_SRC = os.path.join(os.path.dirname(os.path.abspath(__file__)), "sandbox")
|
||||
ORCH_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
SANDBOX_SRC = os.path.join(ORCH_DIR, "sandbox")
|
||||
SCHEMAS_SRC = os.path.join(ORCH_DIR, "schemas")
|
||||
SCHEMA_FILES = ("extraction.schema.json", "grades.schema.json")
|
||||
|
||||
HARDEN = (
|
||||
"--user 1000:1000 --security-opt no-new-privileges --cap-drop ALL "
|
||||
"--read-only --tmpfs /tmp:size=256m --tmpfs /home/rev:size=128m "
|
||||
"--pids-limit 256 --memory 6g --cpus 4"
|
||||
)
|
||||
|
||||
|
||||
def ensure_reviewer_image(cfg: dict, log) -> None:
|
||||
"""Build the reviewer image on the head Spark if missing (aarch64, native)."""
|
||||
# ------------------------------------------------------------------ image
|
||||
def ensure_grader_image(cfg: dict, log) -> None:
|
||||
"""Build the grader image on the head Spark if missing (aarch64, native)."""
|
||||
head = sc.head(cfg)
|
||||
image = cfg["graderImage"]
|
||||
q = shlex.quote
|
||||
r = sc.run(head, f"docker image inspect {q(image)} >/dev/null 2>&1 && echo PRESENT || echo MISSING",
|
||||
timeout=30)
|
||||
if "PRESENT" in (r.stdout or ""):
|
||||
log(f"[reviewers] image {image} already present on {head.host}")
|
||||
log(f"[graders] image {image} already present on {head.host}")
|
||||
return
|
||||
if not os.path.isdir(SANDBOX_SRC):
|
||||
raise RuntimeError(f"reviewer build context missing at {SANDBOX_SRC} (image not baked in?)")
|
||||
raise RuntimeError(f"grader build context missing at {SANDBOX_SRC} (image not baked in?)")
|
||||
remote_dir = f"{cfg['remoteWorkDir']}/sandbox-build"
|
||||
log(f"[reviewers] building reviewer image {image} on {head.host} (first run; a few minutes)…")
|
||||
log(f"[graders] building grader image {image} on {head.host} (first run; a few minutes)…")
|
||||
push = sc.push_dir(head, SANDBOX_SRC, remote_dir)
|
||||
if push.returncode != 0:
|
||||
raise RuntimeError(f"rsync reviewer build context to {head.host} failed: {push.stderr}")
|
||||
raise RuntimeError(f"rsync grader build context to {head.host} failed: {push.stderr}")
|
||||
b = sc.run(head, f"cd {q(remote_dir)} && IMAGE={q(image)} bash build.sh", timeout=1800)
|
||||
if b.returncode != 0:
|
||||
raise RuntimeError(f"reviewer image build failed on {head.host}: {b.stderr or b.stdout}")
|
||||
log(f"[reviewers] reviewer image built: {image}")
|
||||
raise RuntimeError(f"grader image build failed on {head.host}: {b.stderr or b.stdout}")
|
||||
log(f"[graders] grader image built: {image}")
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ roster
|
||||
def slug(name: str) -> str:
|
||||
s = re.sub(r"[^A-Za-z0-9]+", "-", (name or "").strip().lower()).strip("-")
|
||||
return s or "reviewer"
|
||||
return s or "grader"
|
||||
|
||||
|
||||
def roster(cfg: dict) -> list[dict]:
|
||||
"""Reviewer roster from config. Each: {rid, name, model alias, persona, temperature}."""
|
||||
"""Grader roster from config. Each: {rid, name, model alias, persona, temperature}."""
|
||||
out: list[dict] = []
|
||||
seen: dict[str, int] = {}
|
||||
for w in (cfg.get("reviewers") or []):
|
||||
name = (w.get("name") or "reviewer").strip()
|
||||
for w in (cfg.get("graders") or []):
|
||||
name = (w.get("name") or "grader").strip()
|
||||
rid = slug(name)
|
||||
if rid in seen:
|
||||
seen[rid] += 1
|
||||
@@ -76,87 +96,143 @@ def roster(cfg: dict) -> list[dict]:
|
||||
return out
|
||||
|
||||
|
||||
def _container(cfg: dict, jobdir: str, rid: str, name: str, model: str, persona: str,
|
||||
temperature, role: str, extra_mounts: str = "") -> str:
|
||||
"""docker run command for one reviewer/synthesizer container (detached, one-shot)."""
|
||||
q = shlex.quote
|
||||
net = serving.net_name(cfg)
|
||||
base = serving.reviewer_proxy_base(cfg)
|
||||
searxng = (cfg.get("searxngUrl") or "").strip()
|
||||
persona_path = f"{jobdir}/personas/{rid}.md"
|
||||
# ------------------------------------------------------------------ staging
|
||||
def stage_deck_files(cfg: dict, local_deck_dir: str, panel: list[dict]) -> None:
|
||||
"""Write everything the role containers need into the LOCAL per-deck staging
|
||||
dir (jobs.py rsyncs the whole dir to the Spark afterwards): the BDEF rubric,
|
||||
both schemas, and one persona file per role agent (extractor + each grader +
|
||||
adjudicator). Also pre-creates out/ and adjudicator-out/ so the rsync creates
|
||||
them remotely with the SSH user's ownership (uid 1000 = the container user)."""
|
||||
for sub in ("personas", "schemas", "out", "adjudicator-out"):
|
||||
os.makedirs(os.path.join(local_deck_dir, sub), exist_ok=True)
|
||||
with open(os.path.join(local_deck_dir, "BDEF.md"), "w") as f:
|
||||
f.write(bm_config.bdef_text(cfg))
|
||||
for fn in SCHEMA_FILES:
|
||||
shutil.copyfile(os.path.join(SCHEMAS_SRC, fn),
|
||||
os.path.join(local_deck_dir, "schemas", fn))
|
||||
|
||||
def persona(rid: str, text: str) -> None:
|
||||
with open(os.path.join(local_deck_dir, "personas", f"{rid}.md"), "w") as f:
|
||||
f.write((text or "").strip() + "\n")
|
||||
|
||||
persona("extractor", prompts.extractor_persona())
|
||||
for r in panel:
|
||||
persona(r["rid"], r["persona"] or prompts.default_grader_persona(r["name"]))
|
||||
persona("adjudicator",
|
||||
(cfg.get("adjudicatorPersona") or "").strip() or prompts.adjudicator_persona())
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ containers
|
||||
def container_suffix(deckdir: str) -> str:
|
||||
"""Short, docker-safe name suffix from the deck dir (…/<company>/<deck_id>)."""
|
||||
parts = deckdir.rstrip("/").split("/")
|
||||
tail = "-".join(parts[-2:]) if len(parts) >= 2 else parts[-1]
|
||||
return slug(tail)[:48] or "deck"
|
||||
|
||||
|
||||
def base_env(cfg: dict, rid: str, name: str, role: str, model: str, temperature) -> str:
|
||||
"""The env-var block shared by every role container (also used by adjudicator.py)."""
|
||||
q = shlex.quote
|
||||
base = serving.reviewer_proxy_base(cfg)
|
||||
env = (
|
||||
f"-e BM_REVIEWER_ID={q(rid)} -e BM_REVIEWER_NAME={q(name)} -e BM_ROLE={q(role)} "
|
||||
f"-e BM_ROLE={q(role)} -e BM_GRADER_ID={q(rid)} -e BM_GRADER_NAME={q(name)} "
|
||||
f"-e BM_MODEL={q(model)} -e BM_LLM_BASE={q(base)} -e BM_LLM_KEY=sk-local "
|
||||
f"-e BM_MAX_MODEL_LEN={q(str(int(cfg.get('maxModelLen', 32768))))} "
|
||||
f"-e HOME=/home/rev "
|
||||
)
|
||||
if temperature is not None:
|
||||
if role == "extractor":
|
||||
env += "-e BM_TEMPERATURE=0.0 "
|
||||
elif temperature is not None:
|
||||
env += f"-e BM_TEMPERATURE={q(str(temperature))} "
|
||||
# web_search is offered ONLY in local_services mode with a SearXNG URL.
|
||||
if cfg.get("networkMode") == "local_services" and searxng:
|
||||
env += f"-e BM_SEARXNG_URL={q(searxng)} "
|
||||
|
||||
harden = (
|
||||
"--user 1000:1000 --security-opt no-new-privileges --cap-drop ALL "
|
||||
"--read-only --tmpfs /tmp:size=256m --tmpfs /home/rev:size=128m "
|
||||
"--pids-limit 256 --memory 6g --cpus 4"
|
||||
)
|
||||
mounts = (
|
||||
f"-v {q(jobdir)}/docs:/docs:ro "
|
||||
f"-v {q(jobdir)}/out:/out "
|
||||
f"-v {q(persona_path)}:/persona/PERSONA.md:ro "
|
||||
+ extra_mounts
|
||||
)
|
||||
cname = f"bm-grader-{rid}"
|
||||
return (
|
||||
f"docker rm -f {cname} >/dev/null 2>&1; "
|
||||
f"docker run -d --name {cname} --network {q(net)} {harden} {env} {mounts} {q(cfg['graderImage'])}"
|
||||
)
|
||||
return env
|
||||
|
||||
|
||||
def _write_persona(cfg: dict, jobdir: str, rid: str, persona: str) -> None:
|
||||
def _container(cfg: dict, deckdir: str, rid: str, name: str, model: str,
|
||||
temperature, role: str) -> tuple[str, str]:
|
||||
"""(container name, docker run command) for one extractor/grader container."""
|
||||
q = shlex.quote
|
||||
sc.run(sc.head(cfg),
|
||||
f"mkdir -p {q(jobdir)}/personas && printf '%s' {q(persona)} > {q(jobdir)}/personas/{rid}.md",
|
||||
timeout=30)
|
||||
net = serving.net_name(cfg)
|
||||
schema_file = "extraction.schema.json" if role == "extractor" else "grades.schema.json"
|
||||
env = base_env(cfg, rid, name, role, model, temperature)
|
||||
mounts = (
|
||||
f"-v {q(deckdir)}/docs:/docs:ro "
|
||||
f"-v {q(deckdir)}/BDEF.md:/BDEF.md:ro "
|
||||
f"-v {q(deckdir)}/schemas/{schema_file}:/schema.json:ro "
|
||||
f"-v {q(deckdir)}/personas/{rid}.md:/persona/PERSONA.md:ro "
|
||||
f"-v {q(deckdir)}/out:/out "
|
||||
)
|
||||
cname = f"bm-grader-{rid}-{container_suffix(deckdir)}"
|
||||
cmd = (
|
||||
f"docker rm -f {cname} >/dev/null 2>&1; "
|
||||
f"docker run -d --name {cname} --network {q(net)} {HARDEN} {env} {mounts} "
|
||||
f"{q(cfg['graderImage'])}"
|
||||
)
|
||||
return cname, cmd
|
||||
|
||||
|
||||
def run_wave_reviewers(cfg: dict, jobdir: str, panel: list[dict], rubric: str, log,
|
||||
wait_timeout: int = 1800) -> list[dict]:
|
||||
"""Launch every reviewer in `panel` (already filtered to this wave's models),
|
||||
wait for them to finish, and report status. Reports land in <jobdir>/out."""
|
||||
def _wait_for(cfg: dict, cname: str, out_file: str, log, wait_timeout: int) -> tuple[str, bool]:
|
||||
"""docker-wait a launched container, check its output file, remove it."""
|
||||
head = sc.head(cfg)
|
||||
q = shlex.quote
|
||||
# Rubric is shared; write it once into the job dir, mounted into every container.
|
||||
sc.run(head, f"mkdir -p {q(jobdir)}/out && printf '%s' {q(rubric)} > {q(jobdir)}/RUBRIC.md", timeout=30)
|
||||
w = sc.run(head, f"docker wait {cname}", timeout=wait_timeout)
|
||||
code = (w.stdout or "").strip()
|
||||
chk = sc.run(head, f"test -s {q(out_file)} && echo OK || echo MISSING", timeout=30)
|
||||
wrote = "OK" in (chk.stdout or "")
|
||||
sc.run(head, f"docker rm -f {cname} 2>/dev/null; true", timeout=30)
|
||||
return code, wrote
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ runs
|
||||
def run_extractor(cfg: dict, remote_deck_dir: str, model_alias: str, log,
|
||||
wait_timeout: int = 1800) -> dict:
|
||||
"""Launch the stage-1 extractor for one deck and wait for out/extraction.json."""
|
||||
head = sc.head(cfg)
|
||||
cname, cmd = _container(cfg, remote_deck_dir, "extractor", "extractor",
|
||||
model_alias, None, role="extractor")
|
||||
r = sc.run(head, cmd, timeout=120)
|
||||
if r.returncode != 0:
|
||||
log(f"[graders] extractor launch FAILED: {r.stderr or r.stdout}")
|
||||
return {"rid": "extractor", "model": model_alias, "ok": False,
|
||||
"error": (r.stderr or r.stdout)[:300], "report": False}
|
||||
log(f"[graders] up: extractor -> {model_alias}")
|
||||
code, wrote = _wait_for(cfg, cname, f"{remote_deck_dir}/out/extraction.json",
|
||||
log, wait_timeout)
|
||||
log(f"[graders] extractor exited (code={code or '?'}), "
|
||||
f"extraction.json={'written' if wrote else 'MISSING'}")
|
||||
return {"rid": "extractor", "model": model_alias, "ok": True, "error": "",
|
||||
"exit": code, "report": wrote}
|
||||
|
||||
|
||||
def run_wave_graders(cfg: dict, remote_deck_dir: str, wave_panel: list[dict], log,
|
||||
wait_timeout: int = 1800) -> list[dict]:
|
||||
"""Launch every grader in `wave_panel` (already filtered to this wave's
|
||||
models) against one deck, wait for them, and report status. Grade JSONs land
|
||||
in <remote_deck_dir>/out/<rid>.json."""
|
||||
head = sc.head(cfg)
|
||||
launched = []
|
||||
for r in panel:
|
||||
_write_persona(cfg, jobdir, r["rid"], r["persona"])
|
||||
cmd = _container(cfg, jobdir, r["rid"], r["name"], r["model"], r["persona"],
|
||||
r.get("temperature"), role="reviewer",
|
||||
extra_mounts=f"-v {q(jobdir)}/RUBRIC.md:/RUBRIC.md:ro ")
|
||||
for r in wave_panel:
|
||||
cname, cmd = _container(cfg, remote_deck_dir, r["rid"], r["name"], r["model"],
|
||||
r.get("temperature"), role="grader")
|
||||
res = sc.run(head, cmd, timeout=120)
|
||||
if res.returncode != 0:
|
||||
log(f"[reviewers] launch {r['rid']} FAILED: {res.stderr or res.stdout}")
|
||||
launched.append({**r, "ok": False, "error": (res.stderr or res.stdout)[:300]})
|
||||
log(f"[graders] launch {r['rid']} FAILED: {res.stderr or res.stdout}")
|
||||
launched.append({**r, "ok": False, "error": (res.stderr or res.stdout)[:300],
|
||||
"cname": cname})
|
||||
continue
|
||||
log(f"[reviewers] up: {r['rid']} -> {r['model']}")
|
||||
launched.append({**r, "ok": True, "error": ""})
|
||||
log(f"[graders] up: {r['rid']} -> {r['model']}")
|
||||
launched.append({**r, "ok": True, "error": "", "cname": cname})
|
||||
|
||||
# Wait for each launched container to exit (they run in parallel; waiting
|
||||
# sequentially still finishes when the slowest does).
|
||||
results = []
|
||||
for r in launched:
|
||||
if not r["ok"]:
|
||||
results.append(r)
|
||||
results.append({k: v for k, v in r.items() if k != "cname"})
|
||||
continue
|
||||
cname = f"bm-grader-{r['rid']}"
|
||||
w = sc.run(head, f"docker wait {cname}", timeout=wait_timeout)
|
||||
code = (w.stdout or "").strip()
|
||||
out_check = sc.run(head, f"test -s {q(jobdir)}/out/{q(r['rid'])}.md && echo OK || echo MISSING", timeout=30)
|
||||
wrote = "OK" in (out_check.stdout or "")
|
||||
log(f"[reviewers] {r['rid']} exited (code={code or '?'}), report={'written' if wrote else 'MISSING'}")
|
||||
sc.run(head, f"docker rm -f {cname} 2>/dev/null; true", timeout=30)
|
||||
results.append({**r, "exit": code, "report": wrote})
|
||||
code, wrote = _wait_for(cfg, r["cname"], f"{remote_deck_dir}/out/{r['rid']}.json",
|
||||
log, wait_timeout)
|
||||
log(f"[graders] {r['rid']} exited (code={code or '?'}), "
|
||||
f"grades={'written' if wrote else 'MISSING'}")
|
||||
results.append({k: v for k, v in r.items() if k != "cname"}
|
||||
| {"exit": code, "report": wrote})
|
||||
return results
|
||||
|
||||
Reference in New Issue
Block a user