Jonathan KirkwoodandClaude Fable 5
b1d7aed9f4
- Deterministic scoring.py (quant 60 / qual 40 / flags -15, profitability heaviest) - Per-company JSON ledger with forecast-target chaining deck N-1 -> N - Single-shot sandbox agent with guided-JSON fallback ladder (no tool loop) - Portfolio dashboard with sparklines, KPI hit rates, BDEF category bars - 48 unit tests green; endpoints smoke-tested; npm check+build green Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
74 lines
3.1 KiB
Python
74 lines
3.1 KiB
Python
"""Panel adjudication — a local model weighs the graders' evidence. No scores.
|
|
|
|
After the panel finishes grading one deck (and the outputs have been validated),
|
|
one more hardened one-shot container reads the structured extraction
|
|
(/extraction.json, ro) plus every panel grade report (/grades, ro) and writes a
|
|
MARKDOWN adjudication to /out/ADJUDICATION.md: consensus per BDEF category,
|
|
material disagreements and whose evidence is stronger, red flags confirmed or
|
|
dismissed, and three questions for next quarter. It never computes numbers —
|
|
scoring is deterministic Python (scoring.py).
|
|
|
|
It reuses the grader image, switched to BM_ROLE=adjudicator. Mount contract
|
|
(remote paths under the per-deck dir):
|
|
out/ -> /grades:ro (panel *.json; the agent skips
|
|
extraction.json and *.invalid)
|
|
out/extraction.json -> /extraction.json:ro
|
|
personas/adjudicator.md -> /persona/PERSONA.md:ro
|
|
adjudicator-out/ -> /out:rw
|
|
No /docs — the adjudicator judges the panel's evidence, not the deck first-hand.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import shlex
|
|
|
|
import graders as gr_mod
|
|
import serving
|
|
import spark_client as sc
|
|
|
|
|
|
def pick_model(cfg: dict) -> str:
|
|
alias = (cfg.get("adjudicatorModel") or "").strip()
|
|
if alias:
|
|
return alias
|
|
models = cfg.get("models") or []
|
|
return models[0]["alias"] if models else ""
|
|
|
|
|
|
def run_adjudication(cfg: dict, remote_deck_dir: str, log, wait_timeout: int = 1800) -> dict:
|
|
"""Launch the adjudicator container for one deck and wait for ADJUDICATION.md."""
|
|
head = sc.head(cfg)
|
|
q = shlex.quote
|
|
model = pick_model(cfg)
|
|
if not model:
|
|
raise RuntimeError("no model available for adjudication (configure a model catalog)")
|
|
|
|
rid = "adjudicator"
|
|
net = serving.net_name(cfg)
|
|
env = gr_mod.base_env(cfg, rid, "adjudicator", "adjudicator", model, None)
|
|
mounts = (
|
|
f"-v {q(remote_deck_dir)}/out:/grades:ro "
|
|
f"-v {q(remote_deck_dir)}/out/extraction.json:/extraction.json:ro "
|
|
f"-v {q(remote_deck_dir)}/personas/{rid}.md:/persona/PERSONA.md:ro "
|
|
f"-v {q(remote_deck_dir)}/adjudicator-out:/out "
|
|
)
|
|
cname = f"bm-grader-{rid}-{gr_mod.container_suffix(remote_deck_dir)}"
|
|
cmd = (
|
|
f"docker rm -f {cname} >/dev/null 2>&1; "
|
|
f"docker run -d --name {cname} --network {q(net)} {gr_mod.HARDEN} {env} {mounts} "
|
|
f"{q(cfg['graderImage'])}"
|
|
)
|
|
log(f"[adjudicator] up -> {model}")
|
|
r = sc.run(head, cmd, timeout=120)
|
|
if r.returncode != 0:
|
|
raise RuntimeError(f"adjudicator launch failed: {r.stderr or r.stdout}")
|
|
|
|
w = sc.run(head, f"docker wait {cname}", timeout=wait_timeout)
|
|
code = (w.stdout or "").strip()
|
|
chk = sc.run(head, f"test -s {q(remote_deck_dir)}/adjudicator-out/ADJUDICATION.md "
|
|
"&& echo OK || echo MISSING", timeout=30)
|
|
wrote = "OK" in (chk.stdout or "")
|
|
sc.run(head, f"docker rm -f {cname} 2>/dev/null; true", timeout=30)
|
|
log(f"[adjudicator] exited (code={code or '?'}), "
|
|
f"adjudication={'written' if wrote else 'MISSING'}")
|
|
return {"model": model, "exit": code, "report": wrote}
|