Implement BDEF v1.1 grading: scoring core, per-deck pipeline, ledger, dashboard, StartOS layer
- Deterministic scoring.py (quant 60 / qual 40 / flags -15, profitability heaviest) - Per-company JSON ledger with forecast-target chaining deck N-1 -> N - Single-shot sandbox agent with guided-JSON fallback ladder (no tool loop) - Portfolio dashboard with sparklines, KPI hit rates, BDEF category bars - 48 unit tests green; endpoints smoke-tested; npm check+build green Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Jonathan Kirkwood
co-authored by
Claude Fable 5
Claude Fable 5
parent
1dde915540
commit
b1d7aed9f4
+37
-52
@@ -1,88 +1,73 @@
|
||||
"""Local lead-reviewer synthesis — no frontier model.
|
||||
"""Panel adjudication — a local model weighs the graders' evidence. No scores.
|
||||
|
||||
After the panel finishes, one more hardened container (the "lead reviewer") reads
|
||||
all the individual reports (mounted read-only at /reports) plus the documents
|
||||
(/docs), runs a configured local model, and writes a single consolidated report
|
||||
to /out/CONSOLIDATED_REPORT.md: shared themes, where reviewers disagree, the
|
||||
consensus, and an overall recommendation.
|
||||
After the panel finishes grading one deck (and the outputs have been validated),
|
||||
one more hardened one-shot container reads the structured extraction
|
||||
(/extraction.json, ro) plus every panel grade report (/grades, ro) and writes a
|
||||
MARKDOWN adjudication to /out/ADJUDICATION.md: consensus per BDEF category,
|
||||
material disagreements and whose evidence is stronger, red flags confirmed or
|
||||
dismissed, and three questions for next quarter. It never computes numbers —
|
||||
scoring is deterministic Python (scoring.py).
|
||||
|
||||
It reuses the same one-shot reviewer image, switched to BM_ROLE=synthesizer.
|
||||
It reuses the grader image, switched to BM_ROLE=adjudicator. Mount contract
|
||||
(remote paths under the per-deck dir):
|
||||
out/ -> /grades:ro (panel *.json; the agent skips
|
||||
extraction.json and *.invalid)
|
||||
out/extraction.json -> /extraction.json:ro
|
||||
personas/adjudicator.md -> /persona/PERSONA.md:ro
|
||||
adjudicator-out/ -> /out:rw
|
||||
No /docs — the adjudicator judges the panel's evidence, not the deck first-hand.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import shlex
|
||||
|
||||
import spark_client as sc
|
||||
import graders as gr_mod
|
||||
import serving
|
||||
|
||||
DEFAULT_LEAD_PERSONA = (
|
||||
"You are the lead reviewer chairing the panel. You did not read the documents "
|
||||
"first-hand for a fresh opinion — your job is to CONSOLIDATE the panel's "
|
||||
"individual reports into one authoritative report. Identify the findings the "
|
||||
"reviewers agree on, surface and adjudicate where they conflict, note anything "
|
||||
"only one reviewer caught, and end with a prioritized recommendation. Attribute "
|
||||
"points to the reviewer(s) who raised them. Do not invent findings."
|
||||
)
|
||||
import spark_client as sc
|
||||
|
||||
|
||||
def pick_model(cfg: dict) -> str:
|
||||
alias = (cfg.get("synthesisModel") or "").strip()
|
||||
alias = (cfg.get("adjudicatorModel") or "").strip()
|
||||
if alias:
|
||||
return alias
|
||||
models = cfg.get("models") or []
|
||||
return models[0]["alias"] if models else ""
|
||||
|
||||
|
||||
def run_synthesis(cfg: dict, jobdir: str, rubric: str, log, wait_timeout: int = 1800) -> dict:
|
||||
"""Launch the lead-reviewer container and wait for the consolidated report."""
|
||||
def run_adjudication(cfg: dict, remote_deck_dir: str, log, wait_timeout: int = 1800) -> dict:
|
||||
"""Launch the adjudicator container for one deck and wait for ADJUDICATION.md."""
|
||||
head = sc.head(cfg)
|
||||
q = shlex.quote
|
||||
model = pick_model(cfg)
|
||||
if not model:
|
||||
raise RuntimeError("no model available for synthesis (configure a model catalog)")
|
||||
raise RuntimeError("no model available for adjudication (configure a model catalog)")
|
||||
|
||||
persona = (cfg.get("synthesisPersona") or "").strip() or DEFAULT_LEAD_PERSONA
|
||||
rid = "adjudicator"
|
||||
net = serving.net_name(cfg)
|
||||
base = serving.reviewer_proxy_base(cfg)
|
||||
rid = "lead-reviewer"
|
||||
|
||||
sc.run(head,
|
||||
f"mkdir -p {q(jobdir)}/personas {q(jobdir)}/synth-out && "
|
||||
f"printf '%s' {q(persona)} > {q(jobdir)}/personas/{rid}.md && "
|
||||
f"printf '%s' {q(rubric)} > {q(jobdir)}/RUBRIC.md",
|
||||
timeout=30)
|
||||
|
||||
env = (
|
||||
f"-e BM_REVIEWER_ID={q(rid)} -e BM_REVIEWER_NAME=lead-reviewer -e BM_ROLE=synthesizer "
|
||||
f"-e BM_MODEL={q(model)} -e BM_LLM_BASE={q(base)} -e BM_LLM_KEY=sk-local "
|
||||
f"-e BM_MAX_MODEL_LEN={q(str(int(cfg.get('maxModelLen', 32768))))} -e HOME=/home/rev "
|
||||
)
|
||||
harden = (
|
||||
"--user 1000:1000 --security-opt no-new-privileges --cap-drop ALL "
|
||||
"--read-only --tmpfs /tmp:size=256m --tmpfs /home/rev:size=128m "
|
||||
"--pids-limit 256 --memory 6g --cpus 4"
|
||||
)
|
||||
env = gr_mod.base_env(cfg, rid, "adjudicator", "adjudicator", model, None)
|
||||
mounts = (
|
||||
f"-v {q(jobdir)}/docs:/docs:ro "
|
||||
f"-v {q(jobdir)}/out:/reports:ro "
|
||||
f"-v {q(jobdir)}/synth-out:/out "
|
||||
f"-v {q(jobdir)}/personas/{rid}.md:/persona/PERSONA.md:ro "
|
||||
f"-v {q(jobdir)}/RUBRIC.md:/RUBRIC.md:ro "
|
||||
f"-v {q(remote_deck_dir)}/out:/grades:ro "
|
||||
f"-v {q(remote_deck_dir)}/out/extraction.json:/extraction.json:ro "
|
||||
f"-v {q(remote_deck_dir)}/personas/{rid}.md:/persona/PERSONA.md:ro "
|
||||
f"-v {q(remote_deck_dir)}/adjudicator-out:/out "
|
||||
)
|
||||
cname = f"bm-grader-{rid}"
|
||||
cname = f"bm-grader-{rid}-{gr_mod.container_suffix(remote_deck_dir)}"
|
||||
cmd = (
|
||||
f"docker rm -f {cname} >/dev/null 2>&1; "
|
||||
f"docker run -d --name {cname} --network {q(net)} {harden} {env} {mounts} {q(cfg['graderImage'])}"
|
||||
f"docker run -d --name {cname} --network {q(net)} {gr_mod.HARDEN} {env} {mounts} "
|
||||
f"{q(cfg['graderImage'])}"
|
||||
)
|
||||
log(f"[synthesis] lead reviewer up -> {model}")
|
||||
log(f"[adjudicator] up -> {model}")
|
||||
r = sc.run(head, cmd, timeout=120)
|
||||
if r.returncode != 0:
|
||||
raise RuntimeError(f"synthesis launch failed: {r.stderr or r.stdout}")
|
||||
raise RuntimeError(f"adjudicator launch failed: {r.stderr or r.stdout}")
|
||||
|
||||
w = sc.run(head, f"docker wait {cname}", timeout=wait_timeout)
|
||||
code = (w.stdout or "").strip()
|
||||
chk = sc.run(head, f"test -s {q(jobdir)}/synth-out/CONSOLIDATED_REPORT.md && echo OK || echo MISSING", timeout=30)
|
||||
chk = sc.run(head, f"test -s {q(remote_deck_dir)}/adjudicator-out/ADJUDICATION.md "
|
||||
"&& echo OK || echo MISSING", timeout=30)
|
||||
wrote = "OK" in (chk.stdout or "")
|
||||
sc.run(head, f"docker rm -f {cname} 2>/dev/null; true", timeout=30)
|
||||
log(f"[synthesis] lead reviewer exited (code={code or '?'}), report={'written' if wrote else 'MISSING'}")
|
||||
log(f"[adjudicator] exited (code={code or '?'}), "
|
||||
f"adjudication={'written' if wrote else 'MISSING'}")
|
||||
return {"model": model, "exit": code, "report": wrote}
|
||||
|
||||
Reference in New Issue
Block a user