jonathan/boardroom-map
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement BDEF v1.1 grading: scoring core, per-deck pipeline, ledger, dashboard, StartOS layer

Browse Source 
- Deterministic scoring.py (quant 60 / qual 40 / flags -15, profitability heaviest)
- Per-company JSON ledger with forecast-target chaining deck N-1 -> N
- Single-shot sandbox agent with guided-JSON fallback ladder (no tool loop)
- Portfolio dashboard with sparklines, KPI hit rates, BDEF category bars
- 48 unit tests green; endpoints smoke-tested; npm check+build green

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Jonathan Kirkwood
2026-07-06 14:15:12 -05:00
co-authored by Claude Fable 5
parent 1dde915540
commit b1d7aed9f4
48 changed files with 4907 additions and 971 deletions
Download Patch File Download Diff File Expand all files Collapse all files
orchestrator/adjudicator.py
+37 -52
View File
@@ -1,88 +1,73 @@
"""Local lead-reviewer synthesis — no frontier model.
"""Panel adjudication — a local model weighs the graders' evidence. No scores.
After the panel finishes, one more hardened container (the "lead reviewer") reads
all the individual reports (mounted read-only at /reports) plus the documents
(/docs), runs a configured local model, and writes a single consolidated report
to /out/CONSOLIDATED_REPORT.md: shared themes, where reviewers disagree, the
consensus, and an overall recommendation.
After the panel finishes grading one deck (and the outputs have been validated),
one more hardened one-shot container reads the structured extraction
(/extraction.json, ro) plus every panel grade report (/grades, ro) and writes a
MARKDOWN adjudication to /out/ADJUDICATION.md: consensus per BDEF category,
material disagreements and whose evidence is stronger, red flags confirmed or
dismissed, and three questions for next quarter. It never computes numbers —
scoring is deterministic Python (scoring.py).
It reuses the same one-shot reviewer image, switched to BM_ROLE=synthesizer.
It reuses the grader image, switched to BM_ROLE=adjudicator. Mount contract
(remote paths under the per-deck dir):
out/ -> /grades:ro (panel *.json; the agent skips
extraction.json and *.invalid)
out/extraction.json -> /extraction.json:ro
personas/adjudicator.md -> /persona/PERSONA.md:ro
adjudicator-out/ -> /out:rw
No /docs — the adjudicator judges the panel's evidence, not the deck first-hand.
"""
from __future__ import annotations
import shlex
import spark_client as sc
import graders as gr_mod
import serving
DEFAULT_LEAD_PERSONA = (
"You are the lead reviewer chairing the panel. You did not read the documents "
"first-hand for a fresh opinion — your job is to CONSOLIDATE the panel's "
"individual reports into one authoritative report. Identify the findings the "
"reviewers agree on, surface and adjudicate where they conflict, note anything "
"only one reviewer caught, and end with a prioritized recommendation. Attribute "
"points to the reviewer(s) who raised them. Do not invent findings."
)
import spark_client as sc
def pick_model(cfg: dict) -> str:
alias = (cfg.get("synthesisModel") or "").strip()
alias = (cfg.get("adjudicatorModel") or "").strip()
if alias:
return alias
models = cfg.get("models") or []
return models[0]["alias"] if models else ""
def run_synthesis(cfg: dict, jobdir: str, rubric: str, log, wait_timeout: int = 1800) -> dict:
"""Launch the lead-reviewer container and wait for the consolidated report."""
def run_adjudication(cfg: dict, remote_deck_dir: str, log, wait_timeout: int = 1800) -> dict:
"""Launch the adjudicator container for one deck and wait for ADJUDICATION.md."""
head = sc.head(cfg)
q = shlex.quote
model = pick_model(cfg)
if not model:
raise RuntimeError("no model available for synthesis (configure a model catalog)")
raise RuntimeError("no model available for adjudication (configure a model catalog)")
persona = (cfg.get("synthesisPersona") or "").strip() or DEFAULT_LEAD_PERSONA
rid = "adjudicator"
net = serving.net_name(cfg)
base = serving.reviewer_proxy_base(cfg)
rid = "lead-reviewer"
sc.run(head,
f"mkdir -p {q(jobdir)}/personas {q(jobdir)}/synth-out && "
f"printf '%s' {q(persona)} > {q(jobdir)}/personas/{rid}.md && "
f"printf '%s' {q(rubric)} > {q(jobdir)}/RUBRIC.md",
timeout=30)
env = (
f"-e BM_REVIEWER_ID={q(rid)} -e BM_REVIEWER_NAME=lead-reviewer -e BM_ROLE=synthesizer "
f"-e BM_MODEL={q(model)} -e BM_LLM_BASE={q(base)} -e BM_LLM_KEY=sk-local "
f"-e BM_MAX_MODEL_LEN={q(str(int(cfg.get('maxModelLen', 32768))))} -e HOME=/home/rev "
)
harden = (
"--user 1000:1000 --security-opt no-new-privileges --cap-drop ALL "
"--read-only --tmpfs /tmp:size=256m --tmpfs /home/rev:size=128m "
"--pids-limit 256 --memory 6g --cpus 4"
)
env = gr_mod.base_env(cfg, rid, "adjudicator", "adjudicator", model, None)
mounts = (
f"-v {q(jobdir)}/docs:/docs:ro "
f"-v {q(jobdir)}/out:/reports:ro "
f"-v {q(jobdir)}/synth-out:/out "
f"-v {q(jobdir)}/personas/{rid}.md:/persona/PERSONA.md:ro "
f"-v {q(jobdir)}/RUBRIC.md:/RUBRIC.md:ro "
f"-v {q(remote_deck_dir)}/out:/grades:ro "
f"-v {q(remote_deck_dir)}/out/extraction.json:/extraction.json:ro "
f"-v {q(remote_deck_dir)}/personas/{rid}.md:/persona/PERSONA.md:ro "
f"-v {q(remote_deck_dir)}/adjudicator-out:/out "
)
cname = f"bm-grader-{rid}"
cname = f"bm-grader-{rid}-{gr_mod.container_suffix(remote_deck_dir)}"
cmd = (
f"docker rm -f {cname} >/dev/null 2>&1; "
f"docker run -d --name {cname} --network {q(net)} {harden} {env} {mounts} {q(cfg['graderImage'])}"
f"docker run -d --name {cname} --network {q(net)} {gr_mod.HARDEN} {env} {mounts} "
f"{q(cfg['graderImage'])}"
)
log(f"[synthesis] lead reviewer up -> {model}")
log(f"[adjudicator] up -> {model}")
r = sc.run(head, cmd, timeout=120)
if r.returncode != 0:
raise RuntimeError(f"synthesis launch failed: {r.stderr or r.stdout}")
raise RuntimeError(f"adjudicator launch failed: {r.stderr or r.stdout}")
w = sc.run(head, f"docker wait {cname}", timeout=wait_timeout)
code = (w.stdout or "").strip()
chk = sc.run(head, f"test -s {q(jobdir)}/synth-out/CONSOLIDATED_REPORT.md && echo OK || echo MISSING", timeout=30)
chk = sc.run(head, f"test -s {q(remote_deck_dir)}/adjudicator-out/ADJUDICATION.md "
"&& echo OK || echo MISSING", timeout=30)
wrote = "OK" in (chk.stdout or "")
sc.run(head, f"docker rm -f {cname} 2>/dev/null; true", timeout=30)
log(f"[synthesis] lead reviewer exited (code={code or '?'}), report={'written' if wrote else 'MISSING'}")
log(f"[adjudicator] exited (code={code or '?'}), "
f"adjudication={'written' if wrote else 'MISSING'}")
return {"model": model, "exit": code, "report": wrote}
orchestrator/app.py
+267 -52
View File
@@ -1,28 +1,34 @@
"""Boardroom Map orchestrator web app.
Serves the control-panel UI and a small JSON API, and starts the background job
runner (see jobs.py) that actually convenes the panel. Most configuration happens
through the StartOS *actions* (Configure Sparks / Models / Reviewers / Review);
this UI is for dropping documents, triggering a review, watching it run, and
reading reports.
Serves the portfolio dashboard and a small JSON API, and starts the background
job runner (see jobs.py) that grades the dropped board decks. Most configuration
happens through the StartOS *actions* (Configure Sparks / Models / Graders /
Grading); this UI is for dropping decks per company, triggering a grading run,
watching it, and reading the per-company scorecard ledger.
"""
from __future__ import annotations
import os
import threading
from fastapi import FastAPI, HTTPException, UploadFile, File
from fastapi.responses import HTMLResponse, PlainTextResponse
from fastapi import FastAPI, File, Form, HTTPException, UploadFile
from fastapi.responses import HTMLResponse, JSONResponse, PlainTextResponse
from fastapi.templating import Jinja2Templates
from starlette.requests import Request
import bm_config
import extraction
import reviewers as rev_mod
import decks
import graders as grader_mod
import jobs
import ledger as ledger_mod
import serving
from jobs import runner, INBOX, REPORTS_DIR
DATA_DIR = os.environ.get("BM_DATA_DIR", "/data")
LEDGER_DIR = os.path.join(DATA_DIR, "ledger")
runner = jobs.runner
INBOX = getattr(jobs, "INBOX", os.path.join(DATA_DIR, "inbox"))
REPORTS_DIR = getattr(jobs, "REPORTS_DIR", os.path.join(DATA_DIR, "reports"))
app = FastAPI(title="Boardroom Map Orchestrator")
templates = Jinja2Templates(directory=os.path.join(os.path.dirname(__file__), "templates"))
@@ -53,19 +59,19 @@ def status():
"configured": {
"sparks": bool(cfg.get("primarySparkHost")),
"models": len(cfg.get("models") or []),
"reviewers": len(cfg.get("reviewers") or []),
"graders": len(cfg.get("graders") or []),
},
"networkMode": cfg.get("networkMode"),
"synthesis": bool(cfg.get("synthesisEnabled")),
"adjudicator": bool(cfg.get("adjudicatorEnabled")),
"wipeRemoteDocs": bool(cfg.get("wipeRemoteDocs")),
"autoRunOnDrop": bool(cfg.get("autoRunOnDrop")),
"models": [{"alias": m["alias"], "hfModel": m["hfModel"], "spark": m.get("spark", "primary")}
for m in (cfg.get("models") or [])],
"panel": [{"name": r.get("name"), "model": r.get("model"),
"persona": bool((r.get("persona") or "").strip()),
"known": (r.get("model") in catalog)}
for r in (cfg.get("reviewers") or [])],
"inbox": _inbox_list(),
"panel": [{"name": g.get("name"), "model": g.get("model"),
"persona": bool((g.get("persona") or "").strip()),
"known": (g.get("model") in catalog)}
for g in (cfg.get("graders") or [])],
"inbox": _inbox_grouped(),
"runtime": runner.snapshot(),
}
@@ -75,46 +81,89 @@ def events():
return {"events": runner.events()}
def _inbox_list() -> list[dict]:
if not os.path.isdir(INBOX):
return []
out = []
for fn in sorted(os.listdir(INBOX)):
p = os.path.join(INBOX, fn)
if os.path.isfile(p):
ext = os.path.splitext(fn)[1].lower()
out.append({"name": fn, "bytes": os.path.getsize(p),
"supported": ext in extraction.SUPPORTED})
return out
# ----------------------------------------------------------------------- inbox
def _inbox_grouped() -> dict:
"""Company-grouped inbox view: {companies: {slug: [file dicts]}, skipped: [...]}."""
try:
d = decks.discover(INBOX)
except Exception:
return {"companies": {}, "skipped": []}
companies: dict[str, list] = {}
for u in d.get("units") or []:
lst = companies.setdefault(u["company_slug"], [])
for f in u.get("files") or []:
try:
size = os.path.getsize(f)
except OSError:
size = 0
lst.append({"name": os.path.basename(f), "bytes": size,
"period": u.get("period"), "supported": True})
for fn in u.get("ignored") or []:
lst.append({"name": fn, "bytes": 0, "period": None, "supported": False})
# discover() drops units with no supported files, so sweep the company dirs
# for anything it didn't list (unsupported strays) and flag them.
try:
for entry in sorted(os.listdir(INBOX)):
cdir = os.path.join(INBOX, entry)
if entry.startswith(".") or not os.path.isdir(cdir):
continue
slug = decks.slugify(entry)
seen = {f["name"] for f in companies.get(slug, [])}
for fn in sorted(os.listdir(cdir)):
if fn.startswith(".") or fn in seen or not os.path.isfile(os.path.join(cdir, fn)):
continue
try:
size = os.path.getsize(os.path.join(cdir, fn))
except OSError:
size = 0
companies.setdefault(slug, []).append(
{"name": fn, "bytes": size, "period": decks.parse_period_from_name(fn),
"supported": os.path.splitext(fn)[1].lower() in decks.SUPPORTED_EXTS})
except OSError:
pass
return {"companies": companies, "skipped": d.get("skipped") or []}
@app.get("/api/inbox")
def inbox():
return {"inbox": _inbox_list()}
return _inbox_grouped()
# ----------------------------------------------------------------------- documents
@app.post("/api/upload")
async def upload(files: list[UploadFile] = File(...)):
os.makedirs(INBOX, exist_ok=True)
async def upload(request: Request,
files: list[UploadFile] = File(...),
company: str | None = Form(None)):
name = (company or request.query_params.get("company") or "").strip()
if not name:
raise HTTPException(400, "company is required — root-level files are not graded")
slug = decks.slugify(name)
dest_dir = os.path.join(INBOX, slug)
os.makedirs(dest_dir, exist_ok=True)
saved = []
for f in files:
name = os.path.basename(f.filename or "document")
dest = os.path.join(INBOX, name)
fn = os.path.basename(f.filename or "deck")
dest = os.path.join(dest_dir, fn)
with open(dest, "wb") as out:
while chunk := await f.read(1 << 20):
out.write(chunk)
saved.append(name)
return {"ok": True, "saved": saved}
saved.append(fn)
return {"ok": True, "company": slug, "saved": saved}
@app.post("/api/inbox/clear")
def inbox_clear():
import shutil
if os.path.isdir(INBOX):
for fn in os.listdir(INBOX):
p = os.path.join(INBOX, fn)
if os.path.isfile(p):
os.remove(p)
try:
if os.path.isfile(p):
os.remove(p)
elif os.path.isdir(p):
shutil.rmtree(p)
except OSError:
pass
return {"ok": True}
@@ -124,12 +173,16 @@ def run_now():
cfg = bm_config.load()
if not cfg.get("primarySparkHost"):
raise HTTPException(400, "No Spark configured (Configure Sparks).")
if not (cfg.get("models") and cfg.get("reviewers")):
raise HTTPException(400, "Configure at least one model and one reviewer first.")
if not _inbox_list():
raise HTTPException(400, "Inbox is empty — upload documents first.")
if not (cfg.get("models") and cfg.get("graders")):
raise HTTPException(400, "Configure at least one model and one grader first.")
try:
units = decks.discover(INBOX).get("units") or []
except Exception:
units = []
if not units:
raise HTTPException(400, "Inbox is empty — upload decks into a company folder first.")
runner.request_run()
return {"ok": True, "message": "Review requested — watch the activity log."}
return {"ok": True, "message": "Grading requested — watch the activity log."}
@app.get("/api/serving")
@@ -140,20 +193,21 @@ def serving_status():
return {"serving": serving.health(cfg)}
@app.post("/api/reviewer/build-image")
def build_reviewer_image():
@app.post("/api/grader/build-image")
def build_grader_image():
cfg = bm_config.load()
if not cfg.get("primarySparkHost"):
raise HTTPException(400, "No Spark configured.")
threading.Thread(target=lambda: _safe_build(cfg), daemon=True).start()
return {"ok": True, "message": "Building reviewer image on the head Spark — watch the activity log."}
return {"ok": True, "message": "Building grader image on the head Spark — watch the activity log."}
def _safe_build(cfg: dict):
try:
rev_mod.ensure_reviewer_image(cfg, runner.log)
fn = getattr(grader_mod, "ensure_grader_image", None) or grader_mod.ensure_reviewer_image
fn(cfg, runner.log)
except Exception as e:
runner.log(f"[reviewers] image build failed: {e}")
runner.log(f"[graders] image build failed: {e}")
@app.post("/api/stop")
@@ -167,21 +221,182 @@ def stop():
return {"ok": True}
# ----------------------------------------------------------------------- reports
# ----------------------------------------------------------------------- companies / ledger
def _config_companies(cfg: dict) -> list[dict]:
"""Companies registered in the StartOS config (may have zero graded decks)."""
out = []
for c in (cfg.get("companies") or []):
if isinstance(c, dict):
name = (c.get("name") or c.get("company") or c.get("slug") or "").strip()
else:
name = str(c).strip()
if name:
out.append({"slug": decks.slugify(name), "name": name})
return out
def _ledger_companies() -> list[dict]:
try:
led = ledger_mod.Ledger(LEDGER_DIR)
return led.all_companies() or []
except Exception:
return []
@app.get("/api/companies")
def companies():
cfg = bm_config.load()
rows, seen = [], set()
for c in _ledger_companies():
try:
hist = [{"period": h.get("period"), "composite": h.get("composite")}
for h in (c.get("history") or [])]
latest = None
if hist:
delta = None
cur, prev = hist[-1]["composite"], (hist[-2]["composite"] if len(hist) >= 2 else None)
if isinstance(cur, (int, float)) and isinstance(prev, (int, float)):
delta = round(cur - prev, 2)
latest = {"period": hist[-1]["period"], "composite": cur, "delta": delta}
slug = c.get("slug") or decks.slugify(c.get("name") or "")
seen.add(slug)
rows.append({"slug": slug, "name": c.get("name") or slug,
"auto_created": bool(c.get("auto_created")),
"deck_count": len(hist), "latest": latest, "history": hist})
except Exception:
continue
for c in _config_companies(cfg):
if c["slug"] not in seen:
seen.add(c["slug"])
rows.append({"slug": c["slug"], "name": c["name"], "auto_created": False,
"deck_count": 0, "latest": None, "history": []})
rows.sort(key=lambda r: (r["name"] or "").lower())
return {"companies": rows}
def _kpi_hit_rate(records: list[dict]) -> dict:
"""Per canonical KPI across records (oldest→newest):
attempts, hits (credit>=0.999), streak of hits ending at the latest attempt,
last_credit and profitability tag."""
stats: dict[str, dict] = {}
for rec in records:
for k in (rec.get("kpi_results") or []):
cn = k.get("canonical_name") or k.get("name")
if not cn:
continue
s = stats.setdefault(cn, {"name": k.get("name") or cn, "attempts": 0, "hits": 0,
"streak": 0, "last_credit": None, "profitability": False})
if k.get("name"):
s["name"] = k["name"]
s["profitability"] = bool(k.get("profitability", s["profitability"]))
credit = k.get("credit")
if not isinstance(credit, (int, float)):
continue # no target matched — not an attempt
s["attempts"] += 1
s["last_credit"] = credit
if credit >= 0.999:
s["hits"] += 1
s["streak"] += 1
else:
s["streak"] = 0
return stats
def _categories_latest(records: list[dict]) -> dict:
if not records:
return {}
def cats(rec):
return ((rec.get("qual") or {}).get("categories")) or {}
latest = cats(records[-1])
prev = cats(records[-2]) if len(records) >= 2 else {}
out = {}
for cid in "ABCDEFGH":
cur = latest.get(cid)
if cur is None and prev.get(cid) is None:
continue
out[cid] = {"latest_adjusted": (cur or {}).get("adjusted"),
"previous_adjusted": (prev.get(cid) or {}).get("adjusted")}
return out
@app.get("/api/companies/{slug}")
def company_detail(slug: str):
slug = os.path.basename(slug)
company, records = None, []
try:
led = ledger_mod.Ledger(LEDGER_DIR)
company = led.get_company(slug)
if company is not None:
records = led.deck_records(slug) or []
except Exception:
company, records = None, []
if company is None:
cfg = bm_config.load()
match = next((c for c in _config_companies(cfg) if c["slug"] == slug), None)
if not match:
raise HTTPException(404, "no such company")
company = {"slug": slug, "name": match["name"], "auto_created": False,
"kpi_aliases": {}, "pinned_targets": [], "extracted_targets": {},
"history": []}
open_flags = (((records[-1].get("penalties") or {}).get("flags")) or []) if records else []
return {
"company": company,
"records": records,
"kpi_hit_rate": _kpi_hit_rate(records),
"categories_latest": _categories_latest(records),
"open_flags": open_flags,
}
@app.get("/api/companies/{slug}/scorecard", response_class=PlainTextResponse)
def company_scorecard(slug: str):
slug = os.path.basename(slug)
path = os.path.join(LEDGER_DIR, slug, "SCORECARD.md")
if not os.path.exists(path):
raise HTTPException(404, "no scorecard yet for this company")
return open(path, errors="replace").read()
@app.get("/api/companies/{slug}/decks/{deck_id}")
def deck_record(slug: str, deck_id: str):
slug, deck_id = os.path.basename(slug), os.path.basename(deck_id)
rec = None
try:
led = ledger_mod.Ledger(LEDGER_DIR)
rec = led.deck_record(slug, deck_id)
except Exception:
rec = None
if rec is None:
raise HTTPException(404, "no such deck record")
return JSONResponse(rec)
@app.get("/api/companies/{slug}/decks/{deck_id}/report", response_class=PlainTextResponse)
def deck_report(slug: str, deck_id: str):
slug, deck_id = os.path.basename(slug), os.path.basename(deck_id)
if deck_id.endswith(".md"):
deck_id = deck_id[:-3]
path = os.path.join(LEDGER_DIR, slug, "decks", f"{deck_id}.md")
if not os.path.exists(path):
raise HTTPException(404, "no report for this deck")
return open(path, errors="replace").read()
# ----------------------------------------------------------------------- reports (legacy job reports)
@app.get("/api/reports")
def list_reports():
if not os.path.isdir(REPORTS_DIR):
return {"reports": []}
jobs = sorted((d for d in os.listdir(REPORTS_DIR)
if os.path.isdir(os.path.join(REPORTS_DIR, d))), reverse=True)
return {"reports": jobs}
jobs_ = sorted((d for d in os.listdir(REPORTS_DIR)
if os.path.isdir(os.path.join(REPORTS_DIR, d))), reverse=True)
return {"reports": jobs_}
@app.get("/api/report", response_class=PlainTextResponse)
def latest_report():
path = os.path.join(REPORTS_DIR, "latest.md")
if not os.path.exists(path):
return "(no report yet — drop documents in the inbox and run a review)"
return "(no report yet — drop decks in a company folder and run a grading job)"
return open(path, errors="replace").read().strip() or "(empty report)"
orchestrator/bdef.md
+103
View File
@@ -0,0 +1,103 @@
# Board Deck Evaluation Framework (BDEF v1.1)
Inch Wide, Mile Deep — Girdley traits integrated with Munger & Buffett principles.
You are grading a portfolio-company board deck. The deck should let an owner's
representative answer, with high confidence: Are incentives aligned with long-term
owners? Has management inverted the problem and built in margin of safety? Are they
inside (and rationally expanding) their circle of competence? Is capital allocated
with owner-like patience, or is activity masquerading as progress? Would this
company survive a Lollapalooza of bad incentives, biases, and external shocks?
Score each category 1–5. A score above or below 3 REQUIRES verbatim evidence
quotes from the deck. Judge what the deck actually shows — absence of evidence on
a category is itself information (score 2–3 with the absence noted, not a guess).
## A. Incentive Alignment & Skin in the Game
Probes: Does compensation/promotion demonstrably reward rational long-term capital
allocation and owner-like behavior? Visible misalignments (short-term bonus
weighting, option reloads, metrics that invite channel stuffing or earnings
management)? Does management have skin in the game that survives a multi-year
downturn? Munger test: if I changed the incentives, would behavior change predictably?
- 1: Incentives invisible or visibly perverse. 3: Headcount/engagement shown but no
comp structure or ownership data. 5: Comp, promotion criteria, and ownership shown
and clearly aligned with long-term owners.
## B. Inversion Discipline & Margin of Safety
Probes: Are plausible failure modes explicitly modeled for major initiatives and
forecasts? Visible conservatism in assumptions, capital buffers, competitive-response
planning? Does the deck show what the company would NOT do even if it looked attractive?
- 1: Only upside shown; hockey-stick forecasts with no falsifiers. 3: Generic risk
slide, no quantified margin of safety. 5: Explicit inversion — what breaks the
thesis, how much buffer exists, and pre-committed "we won't do X" boundaries.
## C. Circle of Competence & Rational Learning
Probes: Does management accurately describe the boundaries of what they know well?
Disciplined expansion of the circle rather than overreach into new areas? Is learning
from mistakes visible and systematic?
- 1: Confident claims in adjacencies with no demonstrated competence. 3: Competent in
core but boundaries unstated. 5: Explicit "we know / we don't know", postmortems,
and disciplined expansion criteria.
## D. Capital Allocation Quality
Probes: Is every significant capital decision framed as opportunity cost vs long-term
owner return (including returning capital)? Patience ("sit on your ass") vs activity
bias? Are buybacks, dividends, M&A, and reinvestment held to the same owner rigor?
- 1: Growth for its own sake; projects listed without expected returns. 3: Budgets
shown but no alternatives comparison. 5: Every major incremental dollar shown with
expected return vs alternatives, including the do-nothing/return-it option.
## E. Moat Durability & Competitive Reality
Probes: Is the moat described in specific, testable terms (cost, switching costs,
network effects, brand) rather than generic "great team" language? What is management
actively doing to widen/defend it, and which threats are acknowledged? Buffett test:
would an intelligent owner buy this business at a fair price today based on the
durability shown?
- 1: "Great team / huge TAM" hand-waving. 3: Moat named but not evidenced or
threatened realistically. 5: Specific, testable moat with widening actions and
honestly acknowledged threats.
## F. Psychological & Cultural Health
Probes: Does the deck's framing reward early surfacing of problems, or filter
information upward? Evidence of Lollapalooza effects (multiple biases/misaligned
incentives compounding)? Does "no drama" reflect genuine psychological safety or
suppressed dissent? Do problem employees move on quickly; do values drive hiring/firing?
- 1: Only good news; problems appear late and pre-spun. 3: Engagement scores without
bad-news examples. 5: Bad news travels fast and visibly; the deck itself surfaces
problems early with owners' candor.
## G. Simplicity, Clarity & Decision Velocity
Probes: Does the deck avoid unnecessary complexity ("simple stays simple")? Are
repeatable processes and decision frameworks visible, or is the company reliant on
heroic individual effort? Is the board asked to judge the few things that matter
enormously rather than many that matter little?
- 1: Impressively complex deck obscuring weak economics. 3: Clear but unfocused.
5: A model of clarity an intelligent owner could absorb in one sitting, focused on
the 2–3 decisions that matter.
## H. Board Value-Add & Governance Quality
Probes: Does the deck position the board to pull (high-leverage questions on
incentives, inversion, capital allocation, moat) rather than rubber-stamp? Evidence
the board functions as owners' representatives rather than management's advisors?
Clear asks with recommendations and the inversion of those decisions?
- 1: No asks, or trivia; board presides rather than governs. 3: Asks listed without
recommendation or inversion. 5: The few decisions that matter, each with a clear
recommendation and what would make it wrong.
## Red-flag taxonomy
Use these codes (severity 1–5; suggest severity per guidance):
- `adjusted_metrics` (2–4): heavy reliance on adjusted/non-GAAP numbers without bridges.
- `metric_redefinition` (3–5): a KPI's definition changed between periods.
- `kpi_dropped` (2–3): a previously reported KPI silently disappeared.
- `hockey_stick_forecast` (2–4): forecast with no inversion or margin of safety.
- `channel_stuffing_risk` (3–5): incentives/metrics that invite pull-forward behavior.
- `short_term_comp` (2–4): compensation heavily weighted to short-term outcomes.
- `related_party` (3–5): related-party transactions or conflicts.
- `governance_gap` (2–4): big questions (succession, major bets, incentive redesign) get superficial treatment while minutiae fill the deck.
- `cash_runway_silence` (3–5): cash/runway/burn not clearly disclosed.
- `no_profitability_visibility` (3): no profit/margin/cash KPI reported at all.
- `overreach_adjacency` (2–4): confident expansion outside demonstrated competence.
- `activity_bias` (2–3): busy project lists without linkage to moat or owner returns.
- `complexity_smokescreen` (2–4): complexity that appears designed to obscure economics.
- `suppressed_dissent` (3–5): signs bad news is filtered before reaching the board.
Do NOT compute totals or a composite score. Numbers are computed elsewhere.
orchestrator/bm_config.py
+45 -15
View File
@@ -12,13 +12,23 @@ import spark_client as sc
DATA_DIR = os.environ.get("BM_DATA_DIR", "/data")
HF_TOKEN_PATH = os.path.join(DATA_DIR, "secrets", "hf_token")
BDEF_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "bdef.md")
DEFAULT_RUBRIC = (
"Review the attached document(s). Produce a structured report: a 3-5 sentence "
"summary, the key findings and insights, risks or red flags, open questions, "
"and concrete recommendations. Cite the document and section for each point. "
"Be honest about uncertainty; never invent facts not present in the documents."
)
# Scoring weights (composite 0-100 = quant 60 + qual 40 - penalties).
# Every knob the deterministic scorer uses lives here so the user can retune
# without a rebuild. Keep flat: StartOS action inputs are flat number fields.
WEIGHTS_DEFAULTS = {
"profitabilityKpi": 30, # profitability KPI attainment bucket
"otherKpi": 20, # non-profitability measurable KPI bucket
"forecastIntegrity": 10, # deck N actuals vs deck N-1 stated targets
"qualCategoryMax": 5, # each BDEF category A-H maxes at this (8x5=40)
"redFlagCap": 15, # max total penalty
"kpiCreditFloor": 0.5, # actual/target ratio below which credit = 0
"droppedKpiPenalty": 2, # severity of a KPI that silently disappeared
"droppedKpiMax": 3, # count at most this many dropped-KPI flags
"evidenceFullCredit": 400, # quote chars for full qualitative weight
"singleSourceFlagFactor": 0.5, # damping for flags raised by one source only
}
CONFIG_DEFAULTS = {
# Spark connection
@@ -39,22 +49,29 @@ CONFIG_DEFAULTS = {
"proxyPort": 4000,
"maxConcurrentModels": 1,
"models": [
{"alias": "reviewer-a", "hfModel": "Qwen/Qwen3-32B-FP8", "spark": "primary", "port": 8001},
{"alias": "grader-a", "hfModel": "Qwen/Qwen3-32B-FP8", "spark": "primary", "port": 8001},
],
# Review panel
"reviewers": [
{"name": "reviewer-1", "model": "reviewer-a", "persona": "", "temperature": None},
# Grading panel
"graders": [
{"name": "munger-lens", "model": "grader-a", "persona": "", "temperature": None},
],
# Review job settings
"reviewInstructions": DEFAULT_RUBRIC,
# Which catalog model runs the stage-1 structured extractor ("" = first model)
"extractorModel": "",
# Grading job settings
"bdefOverride": "", # non-empty replaces the baked-in bdef.md rubric
"weights": dict(WEIGHTS_DEFAULTS),
"networkMode": "airgapped",
"searxngUrl": "",
"synthesisEnabled": True,
"synthesisModel": "",
"synthesisPersona": "",
"adjudicatorEnabled": True,
"adjudicatorModel": "",
"adjudicatorPersona": "",
"wipeRemoteDocs": True,
"autoRunOnDrop": False,
"networkName": "boardroom-net",
# Portfolio companies (authoritative source of pinned targets / aliases).
# pinnedTargets: [{kpi, target, unit, direction: gte|lte, profitability}]
# kpiAliases: newline-separated "canonical=alias1;alias2" lines.
"companies": [],
# Flags
"hfTokenSet": False,
}
@@ -68,9 +85,22 @@ def load() -> dict:
except FileNotFoundError:
return merged
merged.update({k: v for k, v in saved.items() if v is not None})
# weights merge key-by-key so a partially-saved weights object keeps defaults
w = dict(WEIGHTS_DEFAULTS)
w.update({k: v for k, v in (merged.get("weights") or {}).items() if v is not None})
merged["weights"] = w
return merged
def bdef_text(cfg: dict) -> str:
"""The grading rubric: config override if set, else the baked-in spec."""
override = (cfg.get("bdefOverride") or "").strip()
if override:
return override
with open(BDEF_PATH, encoding="utf-8") as f:
return f.read()
def hf_token() -> str | None:
if os.path.exists(HF_TOKEN_PATH):
t = open(HF_TOKEN_PATH).read().strip()
orchestrator/decks.py
+122
View File
@@ -0,0 +1,122 @@
"""Deck discovery — map the inbox onto (company, period) grading units.
The inbox is organized by company: /data/inbox/<company>/<deck files>. The
subfolder name is the company slug; the reporting period is parsed from each
filename (2026-Q2, Q2 2026, 2026-H1, 2026-05, FY2026 ...). Files whose period
cannot be parsed form a period-less unit that the extractor's own deck.period
can later fill in. Files at the inbox root are skipped (we would not know the
company) and reported so the UI can nag the operator.
"""
from __future__ import annotations
import os
import re
SUPPORTED_EXTS = {".pdf", ".pptx", ".docx", ".txt", ".md", ".text"}
# Canonical period forms: "2026-Q2", "2026-H1", "2026-05", "FY2026".
_PERIOD_PATTERNS = [
# 2026-Q2 / 2026Q2 / 2026_Q2 / 2026 Q2
(re.compile(r"(?<!\d)((?:19|20)\d{2})[-_ ]?[Qq]([1-4])(?!\d)"),
lambda m: f"{m.group(1)}-Q{m.group(2)}"),
# Q2-2026 / Q2_2026 / Q2 2026
(re.compile(r"(?<![A-Za-z0-9])[Qq]([1-4])[-_ ]((?:19|20)\d{2})(?!\d)"),
lambda m: f"{m.group(2)}-Q{m.group(1)}"),
# 2026-H1 / 2026H2
(re.compile(r"(?<!\d)((?:19|20)\d{2})[-_ ]?[Hh]([12])(?!\d)"),
lambda m: f"{m.group(1)}-H{m.group(2)}"),
# FY2026 / FY-2026 / FY 2026
(re.compile(r"(?<![A-Za-z0-9])[Ff][Yy][-_ ]?((?:19|20)\d{2})(?!\d)"),
lambda m: f"FY{m.group(1)}"),
# 2026-05 (month 01-12; checked last so Q/H/FY forms win)
(re.compile(r"(?<!\d)((?:19|20)\d{2})[-_](0[1-9]|1[0-2])(?!\d)"),
lambda m: f"{m.group(1)}-{m.group(2)}"),
]
# Granularity ranks break ties between periods starting the same month
# (coarser first: FY2026 < 2026-H1 < 2026-Q1 < 2026-01).
_GRAN_FY, _GRAN_H, _GRAN_Q, _GRAN_M = 0, 1, 2, 3
_UNKNOWN_KEY = (9999, 99, 9)
def slugify(name: str) -> str:
"""Lowercase [a-z0-9-] slug for a company folder name."""
s = re.sub(r"[^a-z0-9]+", "-", (name or "").strip().lower()).strip("-")
return s or "company"
def parse_period_from_name(filename: str) -> str | None:
"""Canonical period parsed from a filename, or None."""
base = os.path.basename(filename)
for pat, canon in _PERIOD_PATTERNS:
m = pat.search(base)
if m:
return canon(m)
return None
def period_sort_key(period: str | None) -> tuple:
"""(year, start_month, granularity_rank); unknown/None sorts last."""
if not period:
return _UNKNOWN_KEY
m = re.fullmatch(r"((?:19|20)\d{2})-Q([1-4])", period)
if m:
return (int(m.group(1)), (int(m.group(2)) - 1) * 3 + 1, _GRAN_Q)
m = re.fullmatch(r"((?:19|20)\d{2})-H([12])", period)
if m:
return (int(m.group(1)), (int(m.group(2)) - 1) * 6 + 1, _GRAN_H)
m = re.fullmatch(r"((?:19|20)\d{2})-(0[1-9]|1[0-2])", period)
if m:
return (int(m.group(1)), int(m.group(2)), _GRAN_M)
m = re.fullmatch(r"FY((?:19|20)\d{2})", period)
if m:
return (int(m.group(1)), 1, _GRAN_FY)
return _UNKNOWN_KEY
def discover(inbox_dir: str) -> dict:
"""Scan the inbox into grading units.
Returns {"units": [{company_slug, period, period_source, files, ignored}],
"skipped": [root-level file names]}. Units are grouped by (company, parsed
period), sorted by company then oldest period first (period-less last)."""
units: dict[tuple, dict] = {}
skipped: list[str] = []
if not os.path.isdir(inbox_dir):
return {"units": [], "skipped": []}
for entry in sorted(os.listdir(inbox_dir)):
if entry.startswith("."):
continue
path = os.path.join(inbox_dir, entry)
if os.path.isfile(path):
skipped.append(entry)
continue
if not os.path.isdir(path):
continue
company = slugify(entry)
for fn in sorted(os.listdir(path)):
if fn.startswith("."):
continue
fpath = os.path.join(path, fn)
if not os.path.isfile(fpath):
continue
period = parse_period_from_name(fn)
key = (company, period_sort_key(period), period)
unit = units.setdefault(key, {
"company_slug": company,
"period": period,
"period_source": "filename" if period else "unknown",
"files": [],
"ignored": [],
})
ext = os.path.splitext(fn)[1].lower()
if ext in SUPPORTED_EXTS:
unit["files"].append(os.path.abspath(fpath))
else:
unit["ignored"].append(fn)
out = [u for _, u in sorted(units.items(), key=lambda kv: (kv[0][0], kv[0][1]))
if u["files"]]
for u in out:
u["files"].sort()
u["ignored"].sort()
return {"units": out, "skipped": skipped}
orchestrator/extraction.py
+88 -9
View File
@@ -5,6 +5,7 @@ to the Sparks, we extract plain text here so that only normalized text (never th
original binaries) crosses to the review containers. Supported formats:
.pdf -> pypdf
.pptx -> python-pptx (text frames, tables, chart data, notes)
.docx -> python-docx
.txt .md .text -> read as UTF-8
@@ -17,7 +18,7 @@ from __future__ import annotations
import os
TEXT_EXTS = {".txt", ".md", ".text", ".markdown"}
SUPPORTED = TEXT_EXTS | {".pdf", ".docx"}
SUPPORTED = TEXT_EXTS | {".pdf", ".docx", ".pptx"}
def _extract_pdf(path: str) -> str:
@@ -48,6 +49,73 @@ def _extract_docx(path: str) -> str:
return "\n".join(lines).strip()
def _pptx_chart_text(shape) -> list[str]:
"""Best-effort chart title + series names/values (chart XML varies wildly)."""
lines: list[str] = []
try:
chart = shape.chart
try:
if chart.has_title and chart.chart_title.has_text_frame:
lines.append(f"[chart] {chart.chart_title.text_frame.text}")
except Exception:
pass
for plot in chart.plots:
try:
cats = [str(c) for c in (plot.categories or [])]
except Exception:
cats = []
for series in plot.series:
try:
name = str(series.name)
except Exception:
name = "(series)"
try:
vals = ["" if v is None else f"{v:g}" for v in series.values]
except Exception:
vals = []
if cats and len(cats) == len(vals):
pairs = ", ".join(f"{c}={v}" for c, v in zip(cats, vals))
else:
pairs = ", ".join(vals)
lines.append(f"[chart series] {name}: {pairs}")
except Exception:
pass
return lines
def _extract_pptx(path: str) -> str:
from pptx import Presentation
prs = Presentation(path)
parts: list[str] = []
for n, slide in enumerate(prs.slides, 1):
body: list[str] = []
for shape in slide.shapes:
if getattr(shape, "has_text_frame", False):
txt = shape.text_frame.text.strip()
if txt:
body.append(txt)
if getattr(shape, "has_table", False):
for row in shape.table.rows:
cells = [c.text.strip() for c in row.cells]
if any(cells):
body.append(" | ".join(cells))
if getattr(shape, "has_chart", False):
body.extend(_pptx_chart_text(shape))
notes = ""
try:
if slide.has_notes_slide:
notes = (slide.notes_slide.notes_text_frame.text or "").strip()
except Exception:
pass
if notes:
body.append(f"--- notes ---\n{notes}")
if len("".join(body)) < 20:
body.append(f"[low_text_slide: slide {n}]")
parts.append(f"\n\n===== slide {n} =====\n" + "\n".join(body))
return "".join(parts).strip()
def _extract_text(path: str) -> str:
with open(path, errors="replace") as f:
return f.read().strip()
@@ -57,6 +125,8 @@ def extract_file(path: str) -> str:
ext = os.path.splitext(path)[1].lower()
if ext == ".pdf":
return _extract_pdf(path)
if ext == ".pptx":
return _extract_pptx(path)
if ext == ".docx":
return _extract_docx(path)
if ext in TEXT_EXTS:
@@ -70,20 +140,16 @@ def _safe_name(name: str) -> str:
return (keep or "document").replace(" ", "_")
def extract_inbox(inbox: str, out_dir: str, log=print) -> list[dict]:
"""Extract every supported file in `inbox` to a .txt in `out_dir`.
def extract_files(files: list[str], out_dir: str, log=print) -> list[dict]:
"""Extract an explicit list of files (a deck unit) to .txt files in `out_dir`.
Returns a manifest: [{source, out, chars, ok, error}]. Skips unsupported
files (recorded with ok=False) rather than failing the whole job."""
os.makedirs(out_dir, exist_ok=True)
manifest: list[dict] = []
if not os.path.isdir(inbox):
return manifest
seen: dict[str, int] = {}
for fn in sorted(os.listdir(inbox)):
src = os.path.join(inbox, fn)
if not os.path.isfile(src):
continue
for src in files:
fn = os.path.basename(src)
ext = os.path.splitext(fn)[1].lower()
rec = {"source": fn, "out": None, "chars": 0, "ok": False, "error": ""}
if ext not in SUPPORTED:
@@ -113,3 +179,16 @@ def extract_inbox(inbox: str, out_dir: str, log=print) -> list[dict]:
log(f"[extract] {fn} -> {out_name} ({len(text)} chars)")
manifest.append(rec)
return manifest
def extract_inbox(inbox: str, out_dir: str, log=print) -> list[dict]:
"""Extract every supported file in `inbox` to a .txt in `out_dir`.
Returns a manifest: [{source, out, chars, ok, error}]. Skips unsupported
files (recorded with ok=False) rather than failing the whole job."""
if not os.path.isdir(inbox):
os.makedirs(out_dir, exist_ok=True)
return []
files = [os.path.join(inbox, fn) for fn in sorted(os.listdir(inbox))
if os.path.isfile(os.path.join(inbox, fn))]
return extract_files(files, out_dir, log)
orchestrator/graders.py
+158 -82
View File
@@ -1,67 +1,87 @@
"""Launch the reviewer panel on the head Spark over SSH.
"""Launch the grading panel (and the stage-1 extractor) on the head Spark.
Each reviewer is a ONE-SHOT, hardened, read-only container: it reads the document
text mounted at /docs, runs its model (through the on-Spark proxy) under its
persona + the shared rubric, writes a single report to /out/<id>.md, and exits.
There is no shared writable workspace and no git — reviewers cannot alter the
documents or each other's reports.
Each role agent is a ONE-SHOT, hardened, read-only container running
sandbox/grader_agent.py: it reads the deck text mounted at /docs, runs its model
(through the on-Spark proxy) under its persona + the BDEF rubric, writes exactly
one output file to /out, and exits. There is no tool loop and no shared writable
workspace — agents cannot alter the deck text or each other's reports.
Per-deck mount contract (remote paths under {remoteWorkDir}/jobs/<job>/<company>/<deck>):
docs/ -> /docs:ro
BDEF.md -> /BDEF.md:ro
schemas/<role>.schema.json -> /schema.json:ro (extractor|grades)
personas/<rid>.md -> /persona/PERSONA.md:ro
out/ -> /out:rw
Sandbox (per the operator's confidentiality requirement):
* non-root, --cap-drop ALL, --security-opt no-new-privileges
* read-only rootfs + small writable tmpfs; only /docs (ro), /persona (ro), and
/out (rw, this reviewer's report dir) are mounted
* read-only rootfs + small writable tmpfs (/tmp, /home/rev)
* NO docker socket, cpu/mem/pid caps
* attached to the per-job network — in airgapped mode that network is
--internal, so the reviewer can reach ONLY the model proxy, never the internet
--internal, so the agent can reach ONLY the model proxy, never the internet
Reviewers hold no credentials beyond a dummy proxy key.
Agents hold no credentials beyond a dummy proxy key.
"""
from __future__ import annotations
import os
import re
import shlex
import shutil
import spark_client as sc
import bm_config
import prompts
import serving
import spark_client as sc
SANDBOX_SRC = os.path.join(os.path.dirname(os.path.abspath(__file__)), "sandbox")
ORCH_DIR = os.path.dirname(os.path.abspath(__file__))
SANDBOX_SRC = os.path.join(ORCH_DIR, "sandbox")
SCHEMAS_SRC = os.path.join(ORCH_DIR, "schemas")
SCHEMA_FILES = ("extraction.schema.json", "grades.schema.json")
HARDEN = (
"--user 1000:1000 --security-opt no-new-privileges --cap-drop ALL "
"--read-only --tmpfs /tmp:size=256m --tmpfs /home/rev:size=128m "
"--pids-limit 256 --memory 6g --cpus 4"
)
def ensure_reviewer_image(cfg: dict, log) -> None:
"""Build the reviewer image on the head Spark if missing (aarch64, native)."""
# ------------------------------------------------------------------ image
def ensure_grader_image(cfg: dict, log) -> None:
"""Build the grader image on the head Spark if missing (aarch64, native)."""
head = sc.head(cfg)
image = cfg["graderImage"]
q = shlex.quote
r = sc.run(head, f"docker image inspect {q(image)} >/dev/null 2>&1 && echo PRESENT || echo MISSING",
timeout=30)
if "PRESENT" in (r.stdout or ""):
log(f"[reviewers] image {image} already present on {head.host}")
log(f"[graders] image {image} already present on {head.host}")
return
if not os.path.isdir(SANDBOX_SRC):
raise RuntimeError(f"reviewer build context missing at {SANDBOX_SRC} (image not baked in?)")
raise RuntimeError(f"grader build context missing at {SANDBOX_SRC} (image not baked in?)")
remote_dir = f"{cfg['remoteWorkDir']}/sandbox-build"
log(f"[reviewers] building reviewer image {image} on {head.host} (first run; a few minutes)…")
log(f"[graders] building grader image {image} on {head.host} (first run; a few minutes)…")
push = sc.push_dir(head, SANDBOX_SRC, remote_dir)
if push.returncode != 0:
raise RuntimeError(f"rsync reviewer build context to {head.host} failed: {push.stderr}")
raise RuntimeError(f"rsync grader build context to {head.host} failed: {push.stderr}")
b = sc.run(head, f"cd {q(remote_dir)} && IMAGE={q(image)} bash build.sh", timeout=1800)
if b.returncode != 0:
raise RuntimeError(f"reviewer image build failed on {head.host}: {b.stderr or b.stdout}")
log(f"[reviewers] reviewer image built: {image}")
raise RuntimeError(f"grader image build failed on {head.host}: {b.stderr or b.stdout}")
log(f"[graders] grader image built: {image}")
# ------------------------------------------------------------------ roster
def slug(name: str) -> str:
s = re.sub(r"[^A-Za-z0-9]+", "-", (name or "").strip().lower()).strip("-")
return s or "reviewer"
return s or "grader"
def roster(cfg: dict) -> list[dict]:
"""Reviewer roster from config. Each: {rid, name, model alias, persona, temperature}."""
"""Grader roster from config. Each: {rid, name, model alias, persona, temperature}."""
out: list[dict] = []
seen: dict[str, int] = {}
for w in (cfg.get("reviewers") or []):
name = (w.get("name") or "reviewer").strip()
for w in (cfg.get("graders") or []):
name = (w.get("name") or "grader").strip()
rid = slug(name)
if rid in seen:
seen[rid] += 1
@@ -76,87 +96,143 @@ def roster(cfg: dict) -> list[dict]:
return out
def _container(cfg: dict, jobdir: str, rid: str, name: str, model: str, persona: str,
temperature, role: str, extra_mounts: str = "") -> str:
"""docker run command for one reviewer/synthesizer container (detached, one-shot)."""
q = shlex.quote
net = serving.net_name(cfg)
base = serving.reviewer_proxy_base(cfg)
searxng = (cfg.get("searxngUrl") or "").strip()
persona_path = f"{jobdir}/personas/{rid}.md"
# ------------------------------------------------------------------ staging
def stage_deck_files(cfg: dict, local_deck_dir: str, panel: list[dict]) -> None:
"""Write everything the role containers need into the LOCAL per-deck staging
dir (jobs.py rsyncs the whole dir to the Spark afterwards): the BDEF rubric,
both schemas, and one persona file per role agent (extractor + each grader +
adjudicator). Also pre-creates out/ and adjudicator-out/ so the rsync creates
them remotely with the SSH user's ownership (uid 1000 = the container user)."""
for sub in ("personas", "schemas", "out", "adjudicator-out"):
os.makedirs(os.path.join(local_deck_dir, sub), exist_ok=True)
with open(os.path.join(local_deck_dir, "BDEF.md"), "w") as f:
f.write(bm_config.bdef_text(cfg))
for fn in SCHEMA_FILES:
shutil.copyfile(os.path.join(SCHEMAS_SRC, fn),
os.path.join(local_deck_dir, "schemas", fn))
def persona(rid: str, text: str) -> None:
with open(os.path.join(local_deck_dir, "personas", f"{rid}.md"), "w") as f:
f.write((text or "").strip() + "\n")
persona("extractor", prompts.extractor_persona())
for r in panel:
persona(r["rid"], r["persona"] or prompts.default_grader_persona(r["name"]))
persona("adjudicator",
(cfg.get("adjudicatorPersona") or "").strip() or prompts.adjudicator_persona())
# ------------------------------------------------------------------ containers
def container_suffix(deckdir: str) -> str:
"""Short, docker-safe name suffix from the deck dir (…/<company>/<deck_id>)."""
parts = deckdir.rstrip("/").split("/")
tail = "-".join(parts[-2:]) if len(parts) >= 2 else parts[-1]
return slug(tail)[:48] or "deck"
def base_env(cfg: dict, rid: str, name: str, role: str, model: str, temperature) -> str:
"""The env-var block shared by every role container (also used by adjudicator.py)."""
q = shlex.quote
base = serving.reviewer_proxy_base(cfg)
env = (
f"-e BM_REVIEWER_ID={q(rid)} -e BM_REVIEWER_NAME={q(name)} -e BM_ROLE={q(role)} "
f"-e BM_ROLE={q(role)} -e BM_GRADER_ID={q(rid)} -e BM_GRADER_NAME={q(name)} "
f"-e BM_MODEL={q(model)} -e BM_LLM_BASE={q(base)} -e BM_LLM_KEY=sk-local "
f"-e BM_MAX_MODEL_LEN={q(str(int(cfg.get('maxModelLen', 32768))))} "
f"-e HOME=/home/rev "
)
if temperature is not None:
if role == "extractor":
env += "-e BM_TEMPERATURE=0.0 "
elif temperature is not None:
env += f"-e BM_TEMPERATURE={q(str(temperature))} "
# web_search is offered ONLY in local_services mode with a SearXNG URL.
if cfg.get("networkMode") == "local_services" and searxng:
env += f"-e BM_SEARXNG_URL={q(searxng)} "
harden = (
"--user 1000:1000 --security-opt no-new-privileges --cap-drop ALL "
"--read-only --tmpfs /tmp:size=256m --tmpfs /home/rev:size=128m "
"--pids-limit 256 --memory 6g --cpus 4"
)
mounts = (
f"-v {q(jobdir)}/docs:/docs:ro "
f"-v {q(jobdir)}/out:/out "
f"-v {q(persona_path)}:/persona/PERSONA.md:ro "
+ extra_mounts
)
cname = f"bm-grader-{rid}"
return (
f"docker rm -f {cname} >/dev/null 2>&1; "
f"docker run -d --name {cname} --network {q(net)} {harden} {env} {mounts} {q(cfg['graderImage'])}"
)
return env
def _write_persona(cfg: dict, jobdir: str, rid: str, persona: str) -> None:
def _container(cfg: dict, deckdir: str, rid: str, name: str, model: str,
temperature, role: str) -> tuple[str, str]:
"""(container name, docker run command) for one extractor/grader container."""
q = shlex.quote
sc.run(sc.head(cfg),
f"mkdir -p {q(jobdir)}/personas && printf '%s' {q(persona)} > {q(jobdir)}/personas/{rid}.md",
timeout=30)
net = serving.net_name(cfg)
schema_file = "extraction.schema.json" if role == "extractor" else "grades.schema.json"
env = base_env(cfg, rid, name, role, model, temperature)
mounts = (
f"-v {q(deckdir)}/docs:/docs:ro "
f"-v {q(deckdir)}/BDEF.md:/BDEF.md:ro "
f"-v {q(deckdir)}/schemas/{schema_file}:/schema.json:ro "
f"-v {q(deckdir)}/personas/{rid}.md:/persona/PERSONA.md:ro "
f"-v {q(deckdir)}/out:/out "
)
cname = f"bm-grader-{rid}-{container_suffix(deckdir)}"
cmd = (
f"docker rm -f {cname} >/dev/null 2>&1; "
f"docker run -d --name {cname} --network {q(net)} {HARDEN} {env} {mounts} "
f"{q(cfg['graderImage'])}"
)
return cname, cmd
def run_wave_reviewers(cfg: dict, jobdir: str, panel: list[dict], rubric: str, log,
wait_timeout: int = 1800) -> list[dict]:
"""Launch every reviewer in `panel` (already filtered to this wave's models),
wait for them to finish, and report status. Reports land in <jobdir>/out."""
def _wait_for(cfg: dict, cname: str, out_file: str, log, wait_timeout: int) -> tuple[str, bool]:
"""docker-wait a launched container, check its output file, remove it."""
head = sc.head(cfg)
q = shlex.quote
# Rubric is shared; write it once into the job dir, mounted into every container.
sc.run(head, f"mkdir -p {q(jobdir)}/out && printf '%s' {q(rubric)} > {q(jobdir)}/RUBRIC.md", timeout=30)
w = sc.run(head, f"docker wait {cname}", timeout=wait_timeout)
code = (w.stdout or "").strip()
chk = sc.run(head, f"test -s {q(out_file)} && echo OK || echo MISSING", timeout=30)
wrote = "OK" in (chk.stdout or "")
sc.run(head, f"docker rm -f {cname} 2>/dev/null; true", timeout=30)
return code, wrote
# ------------------------------------------------------------------ runs
def run_extractor(cfg: dict, remote_deck_dir: str, model_alias: str, log,
wait_timeout: int = 1800) -> dict:
"""Launch the stage-1 extractor for one deck and wait for out/extraction.json."""
head = sc.head(cfg)
cname, cmd = _container(cfg, remote_deck_dir, "extractor", "extractor",
model_alias, None, role="extractor")
r = sc.run(head, cmd, timeout=120)
if r.returncode != 0:
log(f"[graders] extractor launch FAILED: {r.stderr or r.stdout}")
return {"rid": "extractor", "model": model_alias, "ok": False,
"error": (r.stderr or r.stdout)[:300], "report": False}
log(f"[graders] up: extractor -> {model_alias}")
code, wrote = _wait_for(cfg, cname, f"{remote_deck_dir}/out/extraction.json",
log, wait_timeout)
log(f"[graders] extractor exited (code={code or '?'}), "
f"extraction.json={'written' if wrote else 'MISSING'}")
return {"rid": "extractor", "model": model_alias, "ok": True, "error": "",
"exit": code, "report": wrote}
def run_wave_graders(cfg: dict, remote_deck_dir: str, wave_panel: list[dict], log,
wait_timeout: int = 1800) -> list[dict]:
"""Launch every grader in `wave_panel` (already filtered to this wave's
models) against one deck, wait for them, and report status. Grade JSONs land
in <remote_deck_dir>/out/<rid>.json."""
head = sc.head(cfg)
launched = []
for r in panel:
_write_persona(cfg, jobdir, r["rid"], r["persona"])
cmd = _container(cfg, jobdir, r["rid"], r["name"], r["model"], r["persona"],
r.get("temperature"), role="reviewer",
extra_mounts=f"-v {q(jobdir)}/RUBRIC.md:/RUBRIC.md:ro ")
for r in wave_panel:
cname, cmd = _container(cfg, remote_deck_dir, r["rid"], r["name"], r["model"],
r.get("temperature"), role="grader")
res = sc.run(head, cmd, timeout=120)
if res.returncode != 0:
log(f"[reviewers] launch {r['rid']} FAILED: {res.stderr or res.stdout}")
launched.append({**r, "ok": False, "error": (res.stderr or res.stdout)[:300]})
log(f"[graders] launch {r['rid']} FAILED: {res.stderr or res.stdout}")
launched.append({**r, "ok": False, "error": (res.stderr or res.stdout)[:300],
"cname": cname})
continue
log(f"[reviewers] up: {r['rid']} -> {r['model']}")
launched.append({**r, "ok": True, "error": ""})
log(f"[graders] up: {r['rid']} -> {r['model']}")
launched.append({**r, "ok": True, "error": "", "cname": cname})
# Wait for each launched container to exit (they run in parallel; waiting
# sequentially still finishes when the slowest does).
results = []
for r in launched:
if not r["ok"]:
results.append(r)
results.append({k: v for k, v in r.items() if k != "cname"})
continue
cname = f"bm-grader-{r['rid']}"
w = sc.run(head, f"docker wait {cname}", timeout=wait_timeout)
code = (w.stdout or "").strip()
out_check = sc.run(head, f"test -s {q(jobdir)}/out/{q(r['rid'])}.md && echo OK || echo MISSING", timeout=30)
wrote = "OK" in (out_check.stdout or "")
log(f"[reviewers] {r['rid']} exited (code={code or '?'}), report={'written' if wrote else 'MISSING'}")
sc.run(head, f"docker rm -f {cname} 2>/dev/null; true", timeout=30)
results.append({**r, "exit": code, "report": wrote})
code, wrote = _wait_for(cfg, r["cname"], f"{remote_deck_dir}/out/{r['rid']}.json",
log, wait_timeout)
log(f"[graders] {r['rid']} exited (code={code or '?'}), "
f"grades={'written' if wrote else 'MISSING'}")
results.append({k: v for k, v in r.items() if k != "cname"}
| {"exit": code, "report": wrote})
return results
orchestrator/jobs.py
+425 -153
View File
@@ -1,40 +1,61 @@
"""The Boardroom Map job runner — convenes the review panel over dropped documents.
"""The Boardroom Map job runner — grades dropped board decks against the BDEF.
Runs as a background thread inside the FastAPI app. It does NOT run on a clock
like Nightshift; it reacts to triggers:
Runs as a background thread inside the FastAPI app. It does NOT run on a clock;
it reacts to triggers:
* an explicit "Run Review" (drops /data/state/run_request), or
* autoRunOnDrop: files landing in /data/inbox, once the inbox is stable.
* an explicit "Grade Decks" run request (drops /data/state/run_request), or
* autoRunOnDrop: files landing under /data/inbox/<company-slug>/, once the
inbox is stable across two ticks.
One job at a time. A job:
1. extract text from the inbox locally (CPU) — only text crosses to the Sparks
2. rsync the text to a per-job dir on the head Spark
3. serve the needed models in WAVES; run the reviewers for each wave
4. optionally run the local lead-reviewer synthesis
5. pull the reports back to /data/reports/<job>, assemble latest.md
6. wipe the documents from the Sparks (unless disabled) and tear serving down
One job at a time. A job iterates the discovered deck units OLDEST FIRST per
company, and for each deck:
All state (phase, current job, per-reviewer status, last report) is mirrored to
1. extract text locally (CPU) — only text crosses to the Sparks
2. rsync the per-deck bundle (docs/, BDEF.md, personas/, schemas/, out/,
adjudicator-out/) to {remoteWorkDir}/jobs/<job>/<company>/<deck>/
3. serve the needed models in WAVES (graders' models ∪ the extractor model);
the extractor runs when its model's wave is up, graders in their waves
4. pull out/, validate: extraction.json invalid => the DECK fails (the job
continues); grader JSONs are validated individually, invalid ones dropped;
fewer than 2 valid grade reports => the deck fails
5. adjudicate (optional, non-fatal): a local model weighs the panel's evidence
6. score deterministically (scoring.score_deck) against the company's pinned
targets + the prior deck's forward targets, and record it in the ledger
7. render DECK_REPORT.md + refresh the company SCORECARD.md and the
/data/reports copies
Then it wipes the remote job dir (unless disabled), tears serving down, and
moves the graded originals to /data/processed/<job>/<slug>/ (the company folder
stays in the inbox for reuse). One deck's failure never kills the job: the job
ends "done" if at least one deck was graded.
All state (phase, per-deck status, panel status, last report) is mirrored to
/data/state/runtime.json so the Web UI can render it.
"""
from __future__ import annotations
import json
import os
import re
import shutil
import threading
import time
import traceback
from collections import deque
from datetime import datetime
from datetime import datetime, timezone
import adjudicator as adj_mod
import bm_config
import decks
import extraction
import graders as gr_mod
import ledger as ledger_mod
import preflight
import reviewers as rev_mod
import scorecard
import scoring
import serving
import spark_client as sc
import synthesis as synth_mod
import validate
DATA_DIR = os.environ.get("BM_DATA_DIR", "/data")
INBOX = os.path.join(DATA_DIR, "inbox")
@@ -42,40 +63,77 @@ PROCESSED = os.path.join(DATA_DIR, "processed")
STATE_DIR = os.path.join(DATA_DIR, "state")
JOBS_DIR = os.path.join(STATE_DIR, "jobs")
REPORTS_DIR = os.path.join(DATA_DIR, "reports")
LEDGER_DIR = os.path.join(DATA_DIR, "ledger")
RUNTIME_PATH = os.path.join(STATE_DIR, "runtime.json")
REQUEST_PATH = os.path.join(STATE_DIR, "run_request")
TICK_SECONDS = 10
RUNNING_PHASES = ("extracting", "grading", "adjudicating", "scoring", "collecting")
# Canonical-ish reporting periods: 2026-Q2, 2026-H1, FY2026, 2026-05, 2026.
_PERIOD_RE = re.compile(
r"^(?:FY\s?-?\d{4}|\d{4}(?:[-/ ]?(?:Q[1-4]|H[12]|0[1-9]|1[0-2]))?)$", re.IGNORECASE)
def _inbox_signature() -> tuple[int, str]:
"""(count, signature) of supported files in the inbox, for stability checks."""
"""(count, signature) of supported files anywhere in the inbox tree, for
autoRunOnDrop stability checks (decks live in per-company subfolders)."""
if not os.path.isdir(INBOX):
return (0, "")
items = []
for fn in sorted(os.listdir(INBOX)):
p = os.path.join(INBOX, fn)
if os.path.isfile(p) and os.path.splitext(fn)[1].lower() in extraction.SUPPORTED:
items.append(f"{fn}:{os.path.getsize(p)}:{int(os.path.getmtime(p))}")
for root, _dirs, files in os.walk(INBOX):
for fn in files:
p = os.path.join(root, fn)
if os.path.splitext(fn)[1].lower() in extraction.SUPPORTED:
try:
items.append(f"{os.path.relpath(p, INBOX)}:{os.path.getsize(p)}:"
f"{int(os.path.getmtime(p))}")
except OSError:
pass
items.sort()
return (len(items), "|".join(items))
def _token(s: str) -> str:
t = re.sub(r"[^a-z0-9]+", "-", (s or "").lower()).strip("-")
return t or "deck"
def _composite(record) -> float | None:
"""Best-effort composite lookup on the scoring record (shape owned by scoring.py)."""
if not isinstance(record, dict):
return None
for k in ("composite", "composite_score"):
v = record.get(k)
if isinstance(v, (int, float)):
return v
for parent in ("scores", "totals", "score"):
d = record.get(parent)
if isinstance(d, dict) and isinstance(d.get("composite"), (int, float)):
return d["composite"]
return None
class JobRunner:
def __init__(self):
self._events = deque(maxlen=500)
self._lock = threading.Lock()
self.phase = "idle" # idle | extracting | reviewing | synthesizing | collecting | done | error
self.phase = "idle" # idle | extracting | grading | adjudicating | scoring | collecting | done | error
self.job_id = None
self.message = ""
self.panel: list[dict] = []
self.waves_total = 0
self.wave_index = 0
self.decks_total = 0
self.deck_index = 0
self.company = None
self.period = None
self.decks: list[dict] = []
self.last_report_path = None
self._thread = None
self._last_sig = None
self._stable_sig = None
self._last_done_sig = None
for d in (STATE_DIR, JOBS_DIR, REPORTS_DIR, INBOX, PROCESSED):
for d in (STATE_DIR, JOBS_DIR, REPORTS_DIR, LEDGER_DIR, INBOX, PROCESSED):
os.makedirs(d, exist_ok=True)
self._restore()
@@ -108,11 +166,12 @@ class JobRunner:
self.job_id = d.get("job_id")
self.message = d.get("message", "")
self.panel = d.get("panel", [])
self.decks = d.get("decks", [])
self.last_report_path = d.get("last_report_path")
for e in d.get("events", []):
self._events.append(e)
# A job can't survive a restart; reset a stuck running phase.
if self.phase in ("extracting", "reviewing", "synthesizing", "collecting"):
if self.phase in RUNNING_PHASES:
self.phase = "idle"
except Exception:
pass
@@ -125,6 +184,11 @@ class JobRunner:
"panel": self.panel,
"waves_total": self.waves_total,
"wave_index": self.wave_index,
"decks_total": self.decks_total,
"deck_index": self.deck_index,
"company": self.company,
"period": self.period,
"decks": self.decks,
"last_report_path": self.last_report_path,
}
@@ -136,7 +200,7 @@ class JobRunner:
self._thread.start()
def request_run(self):
"""Public hook (used by the API) to request a review immediately."""
"""Public hook (used by the API) to request a grading run immediately."""
try:
with open(REQUEST_PATH, "w") as f:
f.write(str(time.time()))
@@ -160,23 +224,17 @@ class JobRunner:
if os.path.exists(REQUEST_PATH):
os.remove(REQUEST_PATH)
triggered = True
self.log("[runner] review requested")
self.log("[runner] grading run requested")
elif cfg.get("autoRunOnDrop"):
count, sig = _inbox_signature()
if count and sig == self._last_sig and sig != self._last_done_sig:
# stable across two ticks and not the batch we last processed
triggered = True
self.log("[runner] inbox stable — auto-running review")
self.log("[runner] inbox stable — auto-running grading")
self._last_sig = sig
if not triggered:
return
count, _ = _inbox_signature()
if not count:
self.log("[runner] nothing to review (inbox empty of supported files)")
self.phase = "idle"
self._persist()
return
self._run_job(cfg)
# ------------------------------------------------------------- the job
@@ -186,114 +244,124 @@ class JobRunner:
self.message = ""
self.waves_total = 0
self.wave_index = 0
self.decks_total = 0
self.deck_index = 0
self.company = None
self.period = None
self.decks = []
self.panel = []
local_job = os.path.join(JOBS_DIR, job_id)
local_docs = os.path.join(local_job, "docs")
remote_job = f"{cfg['remoteWorkDir'].rstrip('/')}/jobs/{job_id}"
rubric = cfg.get("reviewInstructions") or bm_config.DEFAULT_RUBRIC
self.log(f"=== Review job {job_id} begins ===")
remote_root = f"{cfg['remoteWorkDir'].rstrip('/')}/jobs/{job_id}"
self.log(f"=== Grading job {job_id} begins ===")
try:
# 1. Extract locally (only text crosses to the Sparks).
self.phase = "extracting"; self._persist()
manifest = extraction.extract_inbox(INBOX, local_docs, self.log)
ok_docs = [m for m in manifest if m["ok"]]
if not ok_docs:
raise RuntimeError("no documents could be extracted (unsupported or empty inbox)")
self.log(f"[runner] extracted {len(ok_docs)} document(s)")
# 1. Discover deck units (per-company folders; oldest first).
disc = decks.discover(INBOX)
units = disc.get("units") or []
for fn in disc.get("skipped") or []:
self.log(f"[runner] WARNING: skipping root-level inbox file '{fn}' — "
"decks belong in /data/inbox/<company-slug>/")
if not units:
raise RuntimeError("nothing to grade — drop decks into /data/inbox/<company-slug>/")
# 2. Resolve the panel against the model catalog.
catalog = {m["alias"] for m in (cfg.get("models") or [])}
panel = rev_mod.roster(cfg)
# 2. Ledger, merged with the configured portfolio companies.
led = ledger_mod.Ledger(LEDGER_DIR)
led.merge_config_companies(cfg.get("companies") or [])
# Resolve the grader panel + extractor model against the catalog.
models = cfg.get("models") or []
catalog = {m["alias"] for m in models}
panel = gr_mod.roster(cfg)
valid = [r for r in panel if r["model"] in catalog]
invalid = [r for r in panel if r["model"] not in catalog]
for r in invalid:
self.log(f"[runner] WARNING: reviewer '{r['name']}' uses unknown model '{r['model']}' — skipped")
if not valid:
raise RuntimeError("no reviewers reference a configured model (see Configure Models/Reviewers)")
self.panel = [{"name": r["name"], "model": r["model"], "status": "pending"} for r in valid]
for r in panel:
if r["model"] not in catalog:
self.log(f"[runner] WARNING: grader '{r['name']}' uses unknown model "
f"'{r['model']}' — skipped")
if len(valid) < 2:
raise RuntimeError(
"need at least 2 graders referencing configured models — every deck "
"requires >= 2 valid grade reports (see Configure Models/Graders)")
extractor_model = (cfg.get("extractorModel") or "").strip() or \
(models[0]["alias"] if models else "")
if extractor_model not in catalog:
raise RuntimeError(f"extractor model '{extractor_model}' is not in the model catalog")
needed = {r["model"] for r in valid}
if cfg.get("synthesisEnabled"):
sm = synth_mod.pick_model(cfg)
if sm:
needed.add(sm)
needed = {r["model"] for r in valid} | {extractor_model}
adjudicate = bool(cfg.get("adjudicatorEnabled"))
adj_model = adj_mod.pick_model(cfg) if adjudicate else ""
needed_all = needed | ({adj_model} if adjudicate and adj_model else set())
# Air-gapped mode can't route to second-Spark models (internal net).
if cfg.get("networkMode") == "airgapped":
cat = {m["alias"]: m for m in (cfg.get("models") or [])}
offenders = [a for a in needed if cat.get(a, {}).get("spark") == "secondary"]
cat = {m["alias"]: m for m in models}
offenders = [a for a in needed_all if cat.get(a, {}).get("spark") == "secondary"]
if offenders:
raise RuntimeError(
"air-gapped mode requires all models on the head Spark, but these are "
f"on the secondary: {', '.join(sorted(offenders))}. Move them to the "
"primary Spark or switch to local-services mode.")
# 3. Ship text to the Spark + ensure infra.
self.phase = "reviewing"; self._persist()
push = sc.push_dir(sc.head(cfg), local_docs, f"{remote_job}/docs")
if push.returncode != 0:
raise RuntimeError(f"shipping documents to the Spark failed: {push.stderr}")
rev_mod.ensure_reviewer_image(cfg, self.log)
# 3. Infra once per job.
gr_mod.ensure_grader_image(cfg, self.log)
serving.ensure_network(cfg, self.log)
# In local_services mode, warn early if the optional web_search backend
# (SearXNG, self-signed HTTPS) is unreachable — non-fatal.
preflight.check_searxng(cfg, self.log)
# 4. Run the panel in waves (synthesis is handled separately, below).
review_aliases = {r["model"] for r in valid}
waves = serving.plan_waves(cfg, review_aliases)
self.waves_total = len(waves)
hf = bm_config.hf_token()
collected = []
for i, wave in enumerate(waves, 1):
self.wave_index = i
wave_aliases = {m["alias"] for m in wave}
wpanel = [r for r in valid if r["model"] in wave_aliases]
self.log(f"[runner] wave {i}/{len(waves)}: models={sorted(wave_aliases)} "
f"reviewers={[r['name'] for r in wpanel]}")
serving.bring_up_wave(cfg, wave, hf, self.log)
self._await_serving(cfg, wave)
preflight.check_wave(cfg, wave, self.log)
res = rev_mod.run_wave_reviewers(cfg, remote_job, wpanel, rubric, self.log)
collected.extend(res)
self._mark_panel(res)
serving.tear_down_wave(cfg, wave, self.log)
# 5. Synthesis (its own single-model wave).
synth_ok = False
if cfg.get("synthesisEnabled"):
self.phase = "synthesizing"; self._persist()
sm = synth_mod.pick_model(cfg)
swave = serving.plan_waves(cfg, {sm})
for wave in swave:
serving.bring_up_wave(cfg, wave, hf, self.log)
self._await_serving(cfg, wave)
preflight.check_wave(cfg, wave, self.log)
sres = synth_mod.run_synthesis(cfg, remote_job, rubric, self.log)
synth_ok = bool(sres.get("report"))
serving.tear_down_wave(cfg, wave, self.log)
# 4. Grade each deck unit, oldest first. One deck's failure never
# kills the job.
self.decks_total = len(units)
succeeded = 0
for idx, unit in enumerate(units, 1):
self.deck_index = idx
self.company = unit["company_slug"]
self.period = unit.get("period")
entry = {"company": unit["company_slug"], "period": unit.get("period"),
"status": "running"}
self.decks.append(entry)
self.panel = [{"name": r["name"], "model": r["model"], "status": "pending"}
for r in valid]
self._persist()
try:
result = self._grade_deck(cfg, led, job_id, remote_root, unit, idx,
valid, extractor_model, adjudicate, hf)
entry.update({"status": "done", "period": result["period"],
"composite": result["composite"]})
succeeded += 1
comp = result["composite"]
self.log(f"[runner] deck done: {unit['company_slug']} {result['period']}"
f" composite={comp if comp is not None else '?'}")
except Exception as e:
entry.update({"status": "failed", "error": str(e)[:300]})
self.log(f"[runner] DECK FAILED ({unit['company_slug']} "
f"{unit.get('period') or '?'}): {e}")
self.log(traceback.format_exc().splitlines()[-1])
self._persist()
# 6. Collect reports + assemble.
# 5. Job-level report.
self.phase = "collecting"; self._persist()
self._collect(cfg, job_id, remote_job, local_job, valid, manifest, synth_ok)
self._write_job_report(job_id)
# 7. Confidentiality: wipe the documents from the Spark.
# 6. Confidentiality: wipe the deck text from the Spark + teardown.
if cfg.get("wipeRemoteDocs", True):
sc.run(sc.head(cfg), f"rm -rf {remote_job}", timeout=60)
self.log("[runner] wiped document text from the Spark")
sc.run(sc.head(cfg), f"rm -rf {remote_root}", timeout=120)
self.log("[runner] wiped deck text from the Spark")
serving.tear_down_all(cfg, self.log)
# 8. Clear the inbox (move originals aside so they aren't re-reviewed).
self._drain_inbox(job_id)
# 7. Move the graded originals aside (the company folders stay).
self._drain_inbox(job_id, units)
self._last_done_sig = _inbox_signature()[1]
self.phase = "done"
self.message = f"Reviewed {len(ok_docs)} document(s) with {len(valid)} reviewer(s)."
self.log(f"=== Review job {job_id} complete ===")
if succeeded:
self.phase = "done"
self.message = (f"Graded {succeeded}/{len(units)} deck(s) with "
f"{len(valid)} grader(s).")
else:
self.phase = "error"
self.message = f"All {len(units)} deck(s) failed — see the activity log."
self.log(f"=== Grading job {job_id} complete ({succeeded}/{len(units)} decks) ===")
self._persist()
except Exception as e:
self.phase = "error"
self.message = f"Review failed: {e}"
self.message = f"Grading failed: {e}"
self.log(f"[runner] JOB FAILED — {e}")
self.log(traceback.format_exc().splitlines()[-1])
try:
@@ -302,6 +370,205 @@ class JobRunner:
pass
self._persist()
# ------------------------------------------------------------- one deck
def _grade_deck(self, cfg: dict, led, job_id: str, remote_root: str, unit: dict,
idx: int, panel: list[dict], extractor_model: str,
adjudicate: bool, hf) -> dict:
"""Grade one deck unit end to end. Raises on deck failure (caller continues)."""
slug_c = unit["company_slug"]
# The staging/remote dir token; the final deck_id is the resolved period.
token = _token(unit["period"]) if unit.get("period") else f"deck-{idx:02d}"
local_deck = os.path.join(JOBS_DIR, job_id, slug_c, token)
remote_deck = f"{remote_root}/{slug_c}/{token}"
head = sc.head(cfg)
# --- extract locally + stage + push -----------------------------------
self.phase = "extracting"; self._persist()
manifest = extraction.extract_files(unit["files"], os.path.join(local_deck, "docs"),
self.log)
ok_docs = [m for m in manifest if m["ok"]]
for m in manifest:
if not m["ok"]:
self.log(f"[runner] WARNING: {m['source']}: {m['error']}")
if not ok_docs:
raise RuntimeError("no document text could be extracted from this deck")
gr_mod.stage_deck_files(cfg, local_deck, panel)
push = sc.push_dir(head, local_deck, remote_deck)
if push.returncode != 0:
raise RuntimeError(f"shipping deck text to the Spark failed: {push.stderr}")
# --- serve in waves; extractor + graders run in their model's wave ----
self.phase = "grading"; self._persist()
waves = serving.plan_waves(cfg, {r["model"] for r in panel} | {extractor_model})
self.waves_total = len(waves)
for i, wave in enumerate(waves, 1):
self.wave_index = i
aliases = {m["alias"] for m in wave}
wpanel = [r for r in panel if r["model"] in aliases]
self.log(f"[runner] wave {i}/{len(waves)}: models={sorted(aliases)} "
f"graders={[r['name'] for r in wpanel]}"
f"{' +extractor' if extractor_model in aliases else ''}")
serving.bring_up_wave(cfg, wave, hf, self.log)
try:
self._await_serving(cfg, wave)
preflight.check_wave(cfg, wave, self.log)
if extractor_model in aliases:
er = gr_mod.run_extractor(cfg, remote_deck, extractor_model, self.log)
if not er.get("report"):
raise RuntimeError("extractor produced no extraction.json")
if wpanel:
res = gr_mod.run_wave_graders(cfg, remote_deck, wpanel, self.log)
self._mark_panel(res)
finally:
serving.tear_down_wave(cfg, wave, self.log)
# --- pull the panel outputs + validate --------------------------------
local_out = os.path.join(local_deck, "out")
pull = sc.pull_dir(head, f"{remote_deck}/out", local_out)
if pull.returncode != 0:
raise RuntimeError(f"pulling panel outputs from the Spark failed: {pull.stderr}")
ext_path = os.path.join(local_out, "extraction.json")
ext_obj, ext_err = validate.validate_file(ext_path, "extraction")
if ext_obj is None or ext_err:
raise RuntimeError(f"extraction.json invalid: {ext_err or 'missing'}")
period, deck_id = self._resolve_period(unit, ext_obj, ext_path, token)
self.period = period; self._persist()
grades, panel_meta = [], []
for r in panel:
gpath = os.path.join(local_out, f"{r['rid']}.json")
gobj, gerr = (None, "no output file")
if os.path.exists(gpath) and not os.path.exists(gpath + ".invalid"):
gobj, gerr = validate.validate_file(gpath, "grades")
ok = gobj is not None and not gerr
if ok:
grades.append(gobj)
else:
self.log(f"[runner] grader {r['rid']} report dropped: {gerr}")
panel_meta.append({"rid": r["rid"], "model": r["model"], "valid": ok})
if len(grades) < 2:
raise RuntimeError(f"only {len(grades)} valid grade report(s) (need >= 2)")
# --- adjudication (non-fatal) ------------------------------------------
adjudication_md = None
if adjudicate:
self.phase = "adjudicating"; self._persist()
try:
adj_model = adj_mod.pick_model(cfg)
for wave in serving.plan_waves(cfg, {adj_model}):
serving.bring_up_wave(cfg, wave, hf, self.log)
try:
self._await_serving(cfg, wave)
preflight.check_wave(cfg, wave, self.log)
adj_mod.run_adjudication(cfg, remote_deck, self.log)
finally:
serving.tear_down_wave(cfg, wave, self.log)
local_adj = os.path.join(local_deck, "adjudicator-out")
sc.pull_dir(head, f"{remote_deck}/adjudicator-out", local_adj)
apath = os.path.join(local_adj, "ADJUDICATION.md")
if os.path.exists(apath):
adjudication_md = open(apath, errors="replace").read().strip() or None
if not adjudication_md:
self.log("[runner] WARNING: no adjudication produced (continuing without)")
except Exception as e:
self.log(f"[runner] WARNING: adjudication failed (non-fatal): {e}")
# --- deterministic scoring + ledger ------------------------------------
self.phase = "scoring"; self._persist()
company = led.ensure_company(slug_c)
pinned = company.get("pinned_targets") or []
aliases_map = company.get("kpi_aliases") or {}
prior = led.prior_targets(slug_c, period)
report_deck_dir = os.path.join(REPORTS_DIR, job_id, slug_c, deck_id)
meta = {
"company": slug_c,
"period": period,
"deck_id": deck_id,
"job_id": job_id,
"graded_at": datetime.now(timezone.utc).isoformat(),
"panel": panel_meta,
"artifacts": {
"report_dir": report_deck_dir,
"extraction": os.path.join(report_deck_dir, "extraction.json"),
"grades": [os.path.join(report_deck_dir, f"{p['rid']}.json")
for p in panel_meta if p["valid"]],
"adjudication": (os.path.join(report_deck_dir, "ADJUDICATION.md")
if adjudication_md else None),
},
}
record = scoring.score_deck(ext_obj, grades, pinned, prior, aliases_map,
cfg["weights"], meta)
rec_path = led.record_deck(slug_c, record, ext_obj.get("forward_targets") or [])
self.log(f"[runner] ledger updated: {rec_path}")
# --- reports ------------------------------------------------------------
self.phase = "collecting"; self._persist()
deck_md = scorecard.render_deck_report(record, ext_obj, adjudication_md)
if rec_path and str(rec_path).endswith(".json"):
ledger_md = os.path.splitext(str(rec_path))[0] + ".md"
else:
ledger_md = os.path.join(LEDGER_DIR, slug_c, "decks", f"{deck_id}.md")
os.makedirs(os.path.dirname(ledger_md), exist_ok=True)
with open(ledger_md, "w") as f:
f.write(deck_md)
os.makedirs(report_deck_dir, exist_ok=True)
with open(os.path.join(report_deck_dir, "DECK_REPORT.md"), "w") as f:
f.write(deck_md)
for fn in sorted(os.listdir(local_out)): # extraction + raw grader jsons (+ .invalid)
src = os.path.join(local_out, fn)
if os.path.isfile(src):
shutil.copyfile(src, os.path.join(report_deck_dir, fn))
if adjudication_md:
with open(os.path.join(report_deck_dir, "ADJUDICATION.md"), "w") as f:
f.write(adjudication_md + "\n")
# Refresh the company scorecard + the /data/reports latest copy.
sc_md = scorecard.render_scorecard(led.get_company(slug_c), led.deck_records(slug_c))
sc_path = os.path.join(LEDGER_DIR, slug_c, "SCORECARD.md")
os.makedirs(os.path.dirname(sc_path), exist_ok=True)
with open(sc_path, "w") as f:
f.write(sc_md)
with open(os.path.join(REPORTS_DIR, "latest-scorecard.md"), "w") as f:
f.write(sc_md)
self.log(f"[runner] reports saved to {report_deck_dir}")
return {"period": period, "deck_id": deck_id, "composite": _composite(record),
"report_dir": report_deck_dir}
def _resolve_period(self, unit: dict, ext_obj: dict, ext_path: str,
token: str) -> tuple[str, str]:
"""(period, deck_id) for this deck. Filename-derived period wins; else the
extractor's deck.period if canonical-ish; else the file's mtime month
(flagged as period_inferred in the extraction's red-flag candidates)."""
if unit.get("period"):
return unit["period"], _token(unit["period"])
p = ((ext_obj.get("deck") or {}).get("period") or "").strip()
if p and _PERIOD_RE.match(p):
self.log(f"[runner] period '{p}' taken from the deck text")
return p, _token(p)
try:
mtime = os.path.getmtime(unit["files"][0])
except OSError:
mtime = time.time()
period = time.strftime("%Y-%m", time.localtime(mtime))
ext_obj.setdefault("red_flag_candidates", []).append({
"code": "period_inferred",
"description": ("Reporting period was not stated in the filename or the deck "
f"text; inferred from the file's modification time as {period}."),
"severity": 2,
"evidence": "",
})
try:
with open(ext_path, "w") as f:
json.dump(ext_obj, f, indent=2)
except Exception:
pass
self.log(f"[runner] WARNING: period inferred from file mtime: {period}")
return period, (_token(period) or token)
# ------------------------------------------------------------- helpers
def _await_serving(self, cfg, wave, timeout=900):
self.log("[runner] waiting for wave serving to come online…")
@@ -329,50 +596,55 @@ class JobRunner:
p["status"] = "no-report"
self._persist()
def _collect(self, cfg, job_id, remote_job, local_job, valid, manifest, synth_ok):
out_local = os.path.join(REPORTS_DIR, job_id)
os.makedirs(out_local, exist_ok=True)
sc.pull_dir(sc.head(cfg), f"{remote_job}/out", os.path.join(out_local, "reviewers"))
if synth_ok:
sc.pull_dir(sc.head(cfg), f"{remote_job}/synth-out", os.path.join(out_local, "synthesis"))
# Assemble a single latest.md: the consolidated report if present, else a
# concatenation of the individual reports.
parts = [f"# Boardroom Map review — {job_id}\n",
"Documents reviewed: " + ", ".join(m["source"] for m in manifest if m["ok"]) + "\n",
"Panel: " + ", ".join(f"{r['name']} ({r['model']})" for r in valid) + "\n"]
consolidated = os.path.join(out_local, "synthesis", "CONSOLIDATED_REPORT.md")
if synth_ok and os.path.exists(consolidated):
parts.append("\n---\n\n## Consolidated report (lead reviewer)\n\n")
parts.append(open(consolidated, errors="replace").read())
parts.append("\n\n---\n")
parts.append("\n## Individual reviewer reports\n")
rev_local = os.path.join(out_local, "reviewers")
if os.path.isdir(rev_local):
for fn in sorted(os.listdir(rev_local)):
if fn.endswith(".md"):
parts.append(f"\n### {fn[:-3]}\n\n")
parts.append(open(os.path.join(rev_local, fn), errors="replace").read())
parts.append("\n")
assembled = "".join(parts)
with open(os.path.join(out_local, "report.md"), "w") as f:
def _write_job_report(self, job_id: str):
"""latest.md — one job summary across every deck graded (or failed)."""
lines = [f"# Boardroom Map grading job — {job_id}\n"]
done = [d for d in self.decks if d.get("status") == "done"]
failed = [d for d in self.decks if d.get("status") == "failed"]
lines.append(f"Decks graded: {len(done)}/{len(self.decks)}\n")
lines.append("\n## Results\n")
for d in self.decks:
if d.get("status") == "done":
comp = d.get("composite")
comp_s = f"{comp:.1f}" if isinstance(comp, (int, float)) else "?"
lines.append(f"- **{d['company']}** — {d.get('period') or '?'}: "
f"composite **{comp_s}** / 100\n")
else:
lines.append(f"- **{d['company']}** — {d.get('period') or '?'}: "
f"FAILED — {d.get('error', 'unknown error')}\n")
if done:
lines.append("\nPer-deck reports (DECK_REPORT.md, extraction, raw grades, "
f"adjudication): `/data/reports/{job_id}/<company>/<deck>/`.\n")
lines.append("Company scorecards: `/data/ledger/<company>/SCORECARD.md` "
"(latest copy at `/data/reports/latest-scorecard.md`).\n")
if failed:
lines.append("\nFailed decks were still moved to "
f"`/data/processed/{job_id}/` — re-drop them to regrade.\n")
assembled = "".join(lines)
out_dir = os.path.join(REPORTS_DIR, job_id)
os.makedirs(out_dir, exist_ok=True)
with open(os.path.join(out_dir, "report.md"), "w") as f:
f.write(assembled)
with open(os.path.join(REPORTS_DIR, "latest.md"), "w") as f:
f.write(assembled)
self.last_report_path = os.path.join(out_local, "report.md")
self.log(f"[runner] reports saved to {out_local}")
self.last_report_path = os.path.join(out_dir, "report.md")
def _drain_inbox(self, job_id):
dest = os.path.join(PROCESSED, job_id)
os.makedirs(dest, exist_ok=True)
for fn in os.listdir(INBOX):
src = os.path.join(INBOX, fn)
if os.path.isfile(src):
try:
shutil.move(src, os.path.join(dest, fn))
except Exception:
pass
self.log(f"[runner] inbox cleared (originals moved to processed/{job_id})")
def _drain_inbox(self, job_id: str, units: list[dict]):
"""Move each unit's ORIGINAL files to /data/processed/<job>/<slug>/. The
per-company inbox folders are kept — the user reuses them next quarter."""
moved = 0
for unit in units:
dest = os.path.join(PROCESSED, job_id, unit["company_slug"])
os.makedirs(dest, exist_ok=True)
for src in unit["files"]:
if os.path.isfile(src):
try:
shutil.move(src, os.path.join(dest, os.path.basename(src)))
moved += 1
except Exception:
pass
self.log(f"[runner] inbox cleared ({moved} file(s) moved to processed/{job_id}; "
"company folders kept)")
# Module-level singleton used by app.py
orchestrator/ledger.py
+180
View File
@@ -0,0 +1,180 @@
"""The per-company running score ledger under /data/ledger.
Layout:
/data/ledger/<slug>/company.json identity, aliases, pinned targets,
extracted forward targets, history
/data/ledger/<slug>/decks/<id>.json full scoring record per graded deck
Config (bm_config `companies`) is the source of truth for name/aliases/pinned
targets; extracted_targets and history are owned by the grading pipeline.
Re-graded decks supersede (rename, never delete) the previous record.
"""
from __future__ import annotations
import json
import os
from datetime import datetime, timezone
import decks as decks_mod
def atomic_write_json(path: str, obj) -> None:
"""Write JSON via temp file + os.replace so readers never see a torn file."""
tmp = f"{path}.tmp"
with open(tmp, "w", encoding="utf-8") as f:
json.dump(obj, f, indent=2, sort_keys=False)
f.write("\n")
os.replace(tmp, path)
def parse_alias_lines(text: str) -> dict:
"""Parse newline-separated "canonical=alias1;alias2" lines into a dict."""
out: dict[str, list[str]] = {}
for line in (text or "").splitlines():
line = line.strip()
if not line or "=" not in line:
continue
canonical, _, rest = line.partition("=")
canonical = canonical.strip().lower()
aliases = [a.strip() for a in rest.split(";") if a.strip()]
if canonical and aliases:
out[canonical] = aliases
return out
class Ledger:
def __init__(self, base_dir: str):
self.base = base_dir
os.makedirs(self.base, exist_ok=True)
# ------------------------------------------------------------- paths
def _company_dir(self, slug: str) -> str:
return os.path.join(self.base, slug)
def _company_path(self, slug: str) -> str:
return os.path.join(self._company_dir(slug), "company.json")
def _decks_dir(self, slug: str) -> str:
return os.path.join(self._company_dir(slug), "decks")
# ------------------------------------------------------------- companies
def ensure_company(self, slug: str, name: str | None = None) -> dict:
"""Load the company, creating a skeleton entry on first sight."""
existing = self.get_company(slug)
if existing is not None:
return existing
company = {
"schema_version": 1,
"slug": slug,
"name": name or slug,
"auto_created": name is None,
"kpi_aliases": {},
"pinned_targets": [],
"extracted_targets": {},
"history": [],
}
os.makedirs(self._company_dir(slug), exist_ok=True)
atomic_write_json(self._company_path(slug), company)
return company
def get_company(self, slug: str) -> dict | None:
try:
with open(self._company_path(slug), encoding="utf-8") as f:
return json.load(f)
except (FileNotFoundError, json.JSONDecodeError):
return None
def all_slugs(self) -> list[str]:
if not os.path.isdir(self.base):
return []
return sorted(d for d in os.listdir(self.base)
if os.path.isfile(self._company_path(d)))
def all_companies(self) -> list[dict]:
return [c for c in (self.get_company(s) for s in self.all_slugs()) if c]
def merge_config_companies(self, companies_cfg: list) -> None:
"""Config wins for name/aliases/pinned targets; ledger keeps the rest."""
for cc in companies_cfg or []:
name = (cc.get("name") or "").strip()
slug = (cc.get("slug") or "").strip() or decks_mod.slugify(name)
company = self.ensure_company(slug, name or slug)
company["name"] = name or company["name"]
company["auto_created"] = False
company["kpi_aliases"] = parse_alias_lines(cc.get("kpiAliases") or "")
company["pinned_targets"] = list(cc.get("pinnedTargets") or [])
atomic_write_json(self._company_path(slug), company)
# ------------------------------------------------------------- targets
def prior_targets(self, slug: str, period: str) -> list[dict]:
"""Forward targets an earlier deck set for `period` (this deck's exam)."""
company = self.get_company(slug)
if not company or not period:
return []
return (company.get("extracted_targets", {}).get(period) or {}).get("targets", [])
# ------------------------------------------------------------- decks
def record_deck(self, slug: str, record: dict, forward_targets: list[dict]) -> str:
"""Persist a scoring record + its forward targets. Returns the deck path.
A re-graded deck_id supersedes (renames) the old record; the history
entry for the same period is replaced; a target period's forward
targets are replaced wholesale when set by a newer (or same) deck."""
company = self.ensure_company(slug)
deck_id = record["deck_id"]
ddir = self._decks_dir(slug)
os.makedirs(ddir, exist_ok=True)
deck_path = os.path.join(ddir, f"{deck_id}.json")
if os.path.exists(deck_path):
stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
os.replace(deck_path, os.path.join(ddir, f"{deck_id}.superseded-{stamp}.json"))
atomic_write_json(deck_path, record)
period = record.get("period")
entry = {"period": period, "composite": record.get("composite"),
"graded_at": record.get("graded_at"),
"deck_file": f"decks/{deck_id}.json"}
history = [h for h in company.get("history", []) if h.get("period") != period]
history.append(entry)
history.sort(key=lambda h: decks_mod.period_sort_key(h.get("period")))
company["history"] = history
extracted = company.setdefault("extracted_targets", {})
from_key = decks_mod.period_sort_key(period)
by_period: dict[str, list[dict]] = {}
for ft in forward_targets or []:
tp = ft.get("target_period")
if tp:
by_period.setdefault(tp, []).append(ft)
for tp, targets in by_period.items():
cur = extracted.get(tp)
if cur is None or from_key >= decks_mod.period_sort_key(cur.get("from_deck")):
extracted[tp] = {"from_deck": period, "targets": targets}
atomic_write_json(self._company_path(slug), company)
return deck_path
def deck_record(self, slug: str, deck_id: str) -> dict | None:
try:
with open(os.path.join(self._decks_dir(slug), f"{deck_id}.json"),
encoding="utf-8") as f:
return json.load(f)
except (FileNotFoundError, json.JSONDecodeError):
return None
def deck_records(self, slug: str) -> list[dict]:
"""All live (non-superseded) deck records, oldest period first."""
ddir = self._decks_dir(slug)
if not os.path.isdir(ddir):
return []
out: list[dict] = []
for fn in sorted(os.listdir(ddir)):
if not fn.endswith(".json") or ".superseded-" in fn:
continue
try:
with open(os.path.join(ddir, fn), encoding="utf-8") as f:
out.append(json.load(f))
except (json.JSONDecodeError, OSError):
continue
out.sort(key=lambda r: decks_mod.period_sort_key(r.get("period")))
return out
orchestrator/prompts.py
+80
View File
@@ -0,0 +1,80 @@
"""Persona texts for the extractor, graders, and adjudicator.
These are the PERSONA.md contents written into each per-job dir and mounted
into the one-shot sandbox containers. The mechanical role instructions (read
/docs, emit JSON matching the schema at /schema.json, write to /out) live in
the sandbox agent itself — these texts only shape judgment and voice.
"""
from __future__ import annotations
def extractor_persona() -> str:
"""Stage-1 structured extractor: a forensic analyst, never a calculator."""
return (
"You are a forensic financial analyst extracting structured data from a "
"portfolio-company board deck. You are exhaustive and literal.\n\n"
"Extract:\n"
"- EVERY quantitative KPI actual reported for the deck's period: revenue, "
"ARR, margins, burn, cash, churn, NRR, headcount, pipeline — anything with "
"a number attached to a metric.\n"
"- Every stated forward target or guidance, with the exact period it "
"applies to (target_period).\n"
"- Red-flag candidates, using ONLY the taxonomy codes from the BDEF rubric "
"(adjusted_metrics, metric_redefinition, kpi_dropped, hockey_stick_forecast, "
"channel_stuffing_risk, short_term_comp, related_party, governance_gap, "
"cash_runway_silence, no_profitability_visibility, overreach_adjacency, "
"activity_bias, complexity_smokescreen, suppressed_dissent).\n"
"- Deck metadata: company hint, reporting period as printed, meeting date, "
"title.\n\n"
"Rules:\n"
"- canonical_name is lower_snake_case, GENERIC, and stable across quarters: "
"arr, ebitda_margin, churn_rate — not q2_arr_2026 or acme_revenue.\n"
"- profitability=true ONLY for profit/margin/cash metrics (EBITDA, net "
"margin, FCF, burn, runway) — never growth or activity metrics.\n"
"- NEVER compute, derive, or infer a number that is not printed in the "
"deck. If a margin is not printed, do not divide two numbers to get it.\n"
"- Copy the source location for every item (e.g. 'slide 6, financial "
"summary').\n"
"- direction: gte when higher is better, lte when lower is better "
"(churn, burn, CAC).\n"
"- target_in_deck is only a target printed NEXT TO the actual for the SAME "
"period; guidance for future periods goes in forward_targets."
)
def default_grader_persona(name: str) -> str:
"""Neutral BDEF lens for graders the operator has not customized."""
return (
f"You are '{name}', an owner's representative on the board grading this "
"deck strictly against the BDEF rubric provided.\n\n"
"- Score each category A-H from 1 to 5. A score above or below 3 REQUIRES "
"verbatim evidence quotes from the deck, with locations.\n"
"- Judge what the deck actually shows. Absence of evidence on a category "
"is itself information: score 2-3 and note the absence — never guess in "
"management's favor.\n"
"- Quote exactly; do not paraphrase inside quotes.\n"
"- Raise red flags only with the rubric's taxonomy codes, each with the "
"evidence that triggered it.\n"
"- Be specific and terse in rationales; write for a board member with "
"five minutes.\n"
"- Do NOT compute totals or a composite score; numbers are computed "
"elsewhere."
)
def adjudicator_persona() -> str:
"""Panel chair: consolidates the graders' verdicts, adds no new scores."""
return (
"You are the panel chair. You did not grade the deck yourself — you read "
"the graders' completed evaluations and adjudicate.\n\n"
"Produce a short markdown memo covering:\n"
"1. Consensus: what the panel agrees on, in one tight paragraph.\n"
"2. Disagreements: where graders diverge, which grader's evidence is "
"stronger and why (judge the quotes, not the adjectives).\n"
"3. Red flags: confirm or dismiss each raised flag based on the cited "
"evidence; say which deserve board attention.\n"
"4. Exactly 3 questions the board should ask management next quarter — "
"high-leverage, inversion-minded, answerable with data.\n\n"
"Attribute points to the grader(s) who raised them. Do not invent "
"findings, do not re-grade, and do NOT produce scores or totals."
)
orchestrator/requirements.txt
+2
View File
@@ -5,3 +5,5 @@ python-multipart==0.0.20
pyyaml==6.0.2
pypdf==5.1.0
python-docx==1.1.2
python-pptx==1.0.2
jsonschema==4.23.0
orchestrator/schemas/extraction.schema.json
+83
View File
@@ -0,0 +1,83 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "boardroom_extraction",
"type": "object",
"additionalProperties": false,
"required": ["schema_version", "deck", "kpis", "forward_targets", "red_flag_candidates", "narrative"],
"properties": {
"schema_version": {"type": "integer"},
"deck": {
"type": "object",
"additionalProperties": false,
"required": ["period"],
"properties": {
"company_hint": {"type": ["string", "null"]},
"period": {"type": ["string", "null"], "description": "Reporting period as printed on the deck, e.g. 2026-Q2, 2026-H1, FY2026, 2026-05"},
"meeting_date": {"type": ["string", "null"]},
"title": {"type": ["string", "null"]},
"truncated": {"type": "boolean", "default": false}
}
},
"kpis": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"required": ["name", "canonical_name", "actual", "direction", "profitability", "source"],
"properties": {
"name": {"type": "string", "description": "KPI label exactly as printed in the deck"},
"canonical_name": {"type": "string", "pattern": "^[a-z0-9_]+$", "description": "lower_snake_case, generic, stable across quarters (arr, ebitda_margin, churn_rate...)"},
"actual": {"type": "number"},
"unit": {"type": "string", "default": ""},
"period": {"type": ["string", "null"]},
"direction": {"type": "string", "enum": ["gte", "lte"], "description": "gte = higher is better, lte = lower is better"},
"profitability": {"type": "boolean", "description": "true only for profit/margin/cash metrics (EBITDA, net margin, FCF, burn...)"},
"target_in_deck": {"type": ["number", "null"], "description": "Target/plan value printed NEXT TO the actual for the SAME period, if any"},
"source": {"type": "string", "description": "Where in the deck, e.g. 'slide 6, financial summary'"},
"notes": {"type": "string", "default": ""}
}
}
},
"forward_targets": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"required": ["name", "canonical_name", "target", "target_period", "direction", "profitability", "source"],
"properties": {
"name": {"type": "string"},
"canonical_name": {"type": "string", "pattern": "^[a-z0-9_]+$"},
"target": {"type": "number"},
"unit": {"type": "string", "default": ""},
"target_period": {"type": "string", "description": "Period this guidance applies to, e.g. 2026-Q3"},
"direction": {"type": "string", "enum": ["gte", "lte"]},
"profitability": {"type": "boolean"},
"source": {"type": "string"}
}
}
},
"red_flag_candidates": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"required": ["code", "description", "severity"],
"properties": {
"code": {"type": "string", "pattern": "^[a-z0-9_]+$"},
"description": {"type": "string"},
"severity": {"type": "integer", "minimum": 1, "maximum": 5},
"evidence": {"type": "string", "default": ""}
}
}
},
"narrative": {
"type": "object",
"additionalProperties": false,
"required": ["summary"],
"properties": {
"summary": {"type": "string"},
"asks": {"type": "array", "items": {"type": "string"}, "default": []}
}
}
}
}
orchestrator/schemas/grades.schema.json
+53
View File
@@ -0,0 +1,53 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "boardroom_grades",
"type": "object",
"additionalProperties": false,
"required": ["schema_version", "grader", "categories", "red_flags", "overall_comment"],
"properties": {
"schema_version": {"type": "integer"},
"grader": {"type": "string"},
"categories": {
"type": "array",
"minItems": 8,
"maxItems": 8,
"items": {
"type": "object",
"additionalProperties": false,
"required": ["id", "score", "evidence", "rationale"],
"properties": {
"id": {"type": "string", "enum": ["A", "B", "C", "D", "E", "F", "G", "H"]},
"score": {"type": "integer", "minimum": 1, "maximum": 5},
"evidence": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"required": ["quote", "location"],
"properties": {
"quote": {"type": "string", "description": "Verbatim text from the deck"},
"location": {"type": "string", "description": "e.g. 'slide 3'"}
}
}
},
"rationale": {"type": "string"}
}
}
},
"red_flags": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"required": ["code", "description", "severity"],
"properties": {
"code": {"type": "string", "pattern": "^[a-z0-9_]+$"},
"description": {"type": "string"},
"severity": {"type": "integer", "minimum": 1, "maximum": 5},
"evidence": {"type": "string", "default": ""}
}
}
},
"overall_comment": {"type": "string"}
}
}
orchestrator/scorecard.py
+265
View File
@@ -0,0 +1,265 @@
"""Markdown renderers: per-deck DECK_REPORT and per-company SCORECARD.
Pure string builders over the scoring record shape (scoring.score_deck) and
ledger deck records — no I/O here; jobs.py decides where the files land.
"""
from __future__ import annotations
_CATEGORIES = "ABCDEFGH"
_CATEGORY_TITLES = {
"A": "Incentive Alignment & Skin in the Game",
"B": "Inversion Discipline & Margin of Safety",
"C": "Circle of Competence & Rational Learning",
"D": "Capital Allocation Quality",
"E": "Moat Durability & Competitive Reality",
"F": "Psychological & Cultural Health",
"G": "Simplicity, Clarity & Decision Velocity",
"H": "Board Value-Add & Governance Quality",
}
def _fmt(x, digits: int = 1) -> str:
if x is None:
return "—"
if isinstance(x, bool):
return "yes" if x else "no"
if isinstance(x, float):
return f"{x:.{digits}f}"
return str(x)
def _num(x, unit: str = "") -> str:
if x is None:
return "—"
s = f"{x:g}" if isinstance(x, (int, float)) else str(x)
return f"{s}{unit}" if unit and unit in ("%",) else (f"{s} {unit}".strip() if unit else s)
def _bucket_row(name: str, b: dict) -> str:
if b.get("na"):
return f"| {name} | — | — | {b.get('kpi_count', 0)} | NA — weight redistributed |"
return (f"| {name} | {_fmt(b.get('weight'))} | {_fmt(b.get('score'))} "
f"| {b.get('kpi_count', 0)} | |")
def render_deck_report(record: dict, extraction: dict, adjudication_md: str | None = None) -> str:
"""One deck's full markdown report."""
lines: list[str] = []
company = record.get("company") or "?"
period = record.get("period") or "unknown period"
lines.append(f"# Deck report — {company} · {period}")
lines.append("")
lines.append(f"## Composite: **{_fmt(record.get('composite'))} / 100**")
lines.append("")
q = record.get("quant", {})
lines.append(f"Quant {_fmt(q.get('score'))} · Qual {_fmt(record.get('qual', {}).get('score'))}"
f" · Penalties −{_fmt(record.get('penalties', {}).get('total'))}"
f" · graded {record.get('graded_at') or '?'} (job {record.get('job_id') or '?'})")
lines.append("")
# --- quantitative buckets
lines.append("## Quantitative (max 60)")
lines.append("")
lines.append("| Bucket | Weight | Score | KPIs | Note |")
lines.append("|---|---|---|---|---|")
lines.append(_bucket_row("Profitability KPIs", q.get("profitability", {})))
lines.append(_bucket_row("Other KPIs", q.get("other", {})))
lines.append(_bucket_row("Forecast integrity", q.get("forecast_integrity", {})))
lines.append("")
kpi_results = record.get("kpi_results") or []
if kpi_results:
lines.append("### KPI results")
lines.append("")
lines.append("| KPI | Actual | Target | Source | Credit |")
lines.append("|---|---|---|---|---|")
for r in kpi_results:
name = r.get("name") or r.get("canonical_name") or "?"
if r.get("matched_via") == "fuzzy":
name += " (≈ matched via fuzzy)"
src = r.get("target_source") or "—"
credit = "—" if r.get("credit") is None else _fmt(r.get("credit"), 2)
lines.append(f"| {name} | {_num(r.get('actual'), r.get('unit') or '')} "
f"| {_num(r.get('target'), r.get('unit') or '')} | {src} | {credit} |")
lines.append("")
fresults = record.get("forecast_results") or []
if fresults:
lines.append("### Forecast integrity (prior targets vs this period's actuals)")
lines.append("")
lines.append("| KPI | Prior target | Actual | Accuracy |")
lines.append("|---|---|---|---|")
for r in fresults:
lines.append(f"| {r.get('canonical_name')} | {_num(r.get('target'))} "
f"| {_num(r.get('actual'))} | {_fmt(r.get('accuracy'), 2)} |")
lines.append("")
# --- qualitative
lines.append("## Qualitative (max 40)")
lines.append("")
lines.append("| Category | Median | Evidence quality | Adjusted | Points |")
lines.append("|---|---|---|---|---|")
cats = record.get("qual", {}).get("categories", {})
for cid in _CATEGORIES:
c = cats.get(cid, {})
lines.append(f"| {cid}. {_CATEGORY_TITLES[cid]} | {_fmt(c.get('median'))} "
f"| {_fmt(c.get('evidence_quality'), 2)} | {_fmt(c.get('adjusted'), 2)} "
f"| {_fmt(c.get('points'), 2)} |")
lines.append("")
for cid in _CATEGORIES:
c = cats.get(cid, {})
rats = c.get("rationales") or []
if not rats:
continue
best = max(rats, key=lambda r: sum(len(e.get("quote") or "") for e in r.get("evidence") or []))
lines.append(f"### {cid}. {_CATEGORY_TITLES[cid]}")
lines.append("")
lines.append(f"**{best.get('grader')}**: {best.get('rationale')}")
for ev in best.get("evidence") or []:
loc = f" — {ev.get('location')}" if ev.get("location") else ""
lines.append(f"> \"{ev.get('quote')}\"{loc}")
lines.append("")
# --- red flags
flags = record.get("penalties", {}).get("flags") or []
lines.append(f"## Red flags (penalty −{_fmt(record.get('penalties', {}).get('total'))})")
lines.append("")
if flags:
lines.append("| Code | Severity | Points | Sources | Description |")
lines.append("|---|---|---|---|---|")
for f in flags:
lines.append(f"| `{f.get('code')}` | {f.get('severity')} | {_fmt(f.get('points'))} "
f"| {', '.join(f.get('sources') or [])} | {f.get('description')} |")
else:
lines.append("None raised.")
lines.append("")
# --- narrative
narrative = record.get("narrative") or extraction.get("narrative") or {}
if narrative.get("summary"):
lines.append("## Narrative")
lines.append("")
lines.append(narrative["summary"])
lines.append("")
asks = narrative.get("asks") or []
if asks:
lines.append("### Asks")
lines.append("")
for a in asks:
lines.append(f"- {a}")
lines.append("")
if adjudication_md:
lines.append("## Panel adjudication")
lines.append("")
lines.append(adjudication_md.strip())
lines.append("")
return "\n".join(lines).rstrip() + "\n"
def _arrow(delta: float) -> str:
if delta > 0:
return "↑"
if delta < 0:
return "↓"
return "→"
def render_scorecard(company: dict, records: list[dict]) -> str:
"""Company SCORECARD.md across all live deck records (oldest first)."""
name = company.get("name") or company.get("slug") or "?"
lines: list[str] = [f"# Scorecard — {name}", ""]
if not records:
lines.append("No graded decks yet.")
return "\n".join(lines) + "\n"
latest = records[-1]
prev = records[-2] if len(records) > 1 else None
comp = latest.get("composite") or 0.0
if prev is not None:
delta = round(comp - (prev.get("composite") or 0.0), 1)
lines.append(f"## Latest composite: **{_fmt(comp)}** ({latest.get('period')}) "
f"{_arrow(delta)} {'+' if delta > 0 else ''}{_fmt(delta)} vs {prev.get('period')}")
else:
lines.append(f"## Latest composite: **{_fmt(comp)}** ({latest.get('period')}) — first graded deck")
lines.append("")
# --- composite history
lines.append("## Composite history")
lines.append("")
lines.append("| Period | Composite | Quant | Qual | Penalties |")
lines.append("|---|---|---|---|---|")
for r in records:
lines.append(f"| {r.get('period') or '?'} | {_fmt(r.get('composite'))} "
f"| {_fmt(r.get('quant', {}).get('score'))} "
f"| {_fmt(r.get('qual', {}).get('score'))} "
f"| −{_fmt(r.get('penalties', {}).get('total'))} |")
lines.append("")
# --- categories latest vs previous
lines.append("## BDEF categories (latest vs previous)")
lines.append("")
lines.append("| Category | Latest | Previous | Δ |")
lines.append("|---|---|---|---|")
lcats = latest.get("qual", {}).get("categories", {})
pcats = (prev or {}).get("qual", {}).get("categories", {})
for cid in _CATEGORIES:
lp = lcats.get(cid, {}).get("points")
pp = pcats.get(cid, {}).get("points")
if lp is not None and pp is not None:
d = round(lp - pp, 2)
dcol = f"{_arrow(d)} {'+' if d > 0 else ''}{_fmt(d, 2)}"
else:
dcol = "—"
lines.append(f"| {cid}. {_CATEGORY_TITLES[cid]} | {_fmt(lp, 2)} | {_fmt(pp, 2)} | {dcol} |")
lines.append("")
# --- KPI hit-rate across records
order: list[str] = []
per_kpi: dict[str, list] = {}
for r in records:
for k in r.get("kpi_results") or []:
cn = k.get("canonical_name") or k.get("name") or "?"
if cn not in per_kpi:
per_kpi[cn] = []
order.append(cn)
per_kpi[cn].append(k.get("credit"))
lines.append("## KPI hit-rate")
lines.append("")
lines.append("| KPI | Attempts | Hits | Streak | Last credit |")
lines.append("|---|---|---|---|---|")
for cn in order:
credits = [c for c in per_kpi[cn] if c is not None]
if not credits:
continue
hits = sum(1 for c in credits if c >= 1)
streak = 0
for c in reversed(credits):
if c >= 1:
streak += 1
else:
break
lines.append(f"| {cn} | {len(credits)} | {hits} | {streak} | {_fmt(credits[-1], 2)} |")
lines.append("")
# --- open flags on the latest deck
flags = latest.get("penalties", {}).get("flags") or []
lines.append(f"## Open flags ({latest.get('period')})")
lines.append("")
if flags:
lines.append("| Code | Severity | Points | Sources | Description |")
lines.append("|---|---|---|---|---|")
for f in flags:
lines.append(f"| `{f.get('code')}` | {f.get('severity')} | {_fmt(f.get('points'))} "
f"| {', '.join(f.get('sources') or [])} | {f.get('description')} |")
else:
lines.append("None.")
lines.append("")
# --- deck reports
lines.append("## Deck reports")
lines.append("")
for r in records:
lines.append(f"- {r.get('period') or '?'} — composite {_fmt(r.get('composite'))} — "
f"decks/{r.get('deck_id')}.json / DECK_REPORT.md")
return "\n".join(lines).rstrip() + "\n"
orchestrator/scoring.py
+327
View File
@@ -0,0 +1,327 @@
"""Deterministic BDEF scoring — pure functions, stdlib only, no I/O.
Everything numeric happens here, never in a model: the panel supplies 1-5
category scores with verbatim evidence, the extractor supplies KPI actuals and
targets, and this module turns them into the 0-100 composite:
composite = quant (60) + qualitative (40) - red-flag penalties (cap 15)
All knobs come from the `weights` dict (bm_config.WEIGHTS_DEFAULTS shape) so
the operator can retune without a rebuild.
"""
from __future__ import annotations
import difflib
import re
import statistics
FUZZY_THRESHOLD = 0.85
_CATEGORIES = "ABCDEFGH"
# ---------------------------------------------------------------- KPI matching
def _norm(name: str) -> str:
return re.sub(r"[^a-z0-9]+", " ", (name or "").lower()).strip()
def match_kpi(canonical: str, candidates: list[dict], aliases: dict) -> tuple[dict | None, str | None]:
"""Match a canonical KPI name against candidate dicts ({canonical_name, name}).
Exact canonical -> alias map (either direction) -> fuzzy ratio >= 0.85.
Returns (candidate, via) with via in exact|alias|fuzzy, or (None, None)."""
canon = (canonical or "").strip().lower()
if not canon:
return None, None
for c in candidates:
if (c.get("canonical_name") or "").strip().lower() == canon:
return c, "exact"
amap = {(k or "").strip().lower(): {(a or "").strip().lower() for a in (v or [])}
for k, v in (aliases or {}).items()}
ours = amap.get(canon, set())
for c in candidates:
cn = (c.get("canonical_name") or "").strip().lower()
nm = (c.get("name") or "").strip().lower()
if cn in ours or nm in ours or canon in amap.get(cn, set()):
return c, "alias"
best, best_r = None, 0.0
for c in candidates:
for other in (c.get("canonical_name") or "", c.get("name") or ""):
r = difflib.SequenceMatcher(None, _norm(canon), _norm(other)).ratio()
if r > best_r:
best, best_r = c, r
if best is not None and best_r >= FUZZY_THRESHOLD:
return best, "fuzzy"
return None, None
# ---------------------------------------------------------------- KPI credit
def _passes(actual: float, target: float, direction: str) -> bool:
return actual <= target if direction == "lte" else actual >= target
def _credit(actual: float, target: float, direction: str, floor: float) -> float:
"""Partial credit for a targeted KPI: 0 below floor, linear to 1 at target."""
if target == 0 or (actual < 0) != (target < 0) or (direction == "lte" and actual == 0):
return 1.0 if _passes(actual, target, direction) else 0.0
r = target / actual if direction == "lte" else actual / target
if actual < 0 and target < 0:
# Both negative (EBITDA margin target -2, actual -3): the plain ratio
# inverts the ordering, so flip it back.
r = 1.0 / r
if r >= 1:
return 1.0
if floor >= 1 or r < floor:
return 0.0
return max(0.0, min(1.0, (r - floor) / (1 - floor)))
def _resolve_target(kpi: dict, pinned_targets: list[dict], prior_targets: list[dict],
aliases: dict) -> tuple[dict | None, str | None, str | None]:
"""(target dict {target, direction}, target_source, matched_via) for one KPI.
Precedence: pinned config target > prior deck's extracted forward target
for this period > target printed in the deck itself."""
canon = kpi.get("canonical_name") or ""
pinned_cands = [{"canonical_name": p.get("kpi"), "name": p.get("kpi"), "_src": p}
for p in (pinned_targets or [])]
cand, via = match_kpi(canon, pinned_cands, aliases)
if cand is not None:
p = cand["_src"]
return {"target": p.get("target"), "direction": p.get("direction") or kpi.get("direction")}, "pinned", via
cand, via = match_kpi(canon, prior_targets or [], aliases)
if cand is not None:
return {"target": cand.get("target"),
"direction": cand.get("direction") or kpi.get("direction")}, "extracted", via
if kpi.get("target_in_deck") is not None:
return {"target": kpi["target_in_deck"], "direction": kpi.get("direction")}, "in_deck", None
return None, None, None
# ---------------------------------------------------------------- scoring
def _bucket(results: list[dict], weight: float) -> dict:
if not results:
return {"score": 0.0, "weight": 0.0, "na": True, "kpi_count": 0}
mean = sum(r["credit"] for r in results) / len(results)
return {"score": mean, "weight": weight, "na": False, "kpi_count": len(results)}
def _qual(grades: list[dict], weights: dict) -> tuple[float, dict]:
"""(qual score, per-category detail). Evidence regresses medians toward 3."""
full_credit = max(1, int(weights.get("evidenceFullCredit", 400)))
cat_max = float(weights.get("qualCategoryMax", 5))
out: dict[str, dict] = {}
total = 0.0
for cid in _CATEGORIES:
scores: list[int] = []
equalities: list[float] = []
rationales: list[dict] = []
for g in grades or []:
cat = next((c for c in (g.get("categories") or []) if c.get("id") == cid), None)
if cat is None:
continue
evidence = cat.get("evidence") or []
quote_chars = sum(min(len(ev.get("quote") or ""), 200) for ev in evidence)
scores.append(int(cat.get("score", 3)))
equalities.append(min(1.0, quote_chars / full_credit))
rationales.append({
"grader": g.get("grader") or "grader",
"rationale": cat.get("rationale") or "",
"evidence": [{"quote": ev.get("quote") or "", "location": ev.get("location") or ""}
for ev in evidence],
})
if scores:
median = float(statistics.median(scores))
e_mean = sum(equalities) / len(equalities)
else:
median, e_mean = 3.0, 0.0
adjusted = 3.0 + (median - 3.0) * e_mean
points = adjusted * cat_max / 5.0
total += points
out[cid] = {
"panel_scores": scores,
"median": round(median, 2),
"evidence_quality": round(e_mean, 4),
"adjusted": round(adjusted, 4),
"points": round(points, 4),
"rationales": rationales,
}
return total, out
def score_deck(extraction: dict, grades: list[dict], pinned_targets: list[dict],
prior_targets: list[dict], kpi_aliases: dict, weights: dict,
meta: dict) -> dict:
"""Score one graded deck into the canonical ledger record. Pure."""
kpis = extraction.get("kpis") or []
floor = float(weights.get("kpiCreditFloor", 0.5))
scoring_flags: list[dict] = []
# --- per-KPI target resolution + credit
kpi_results: list[dict] = []
for k in kpis:
tgt, source, via = _resolve_target(k, pinned_targets, prior_targets, kpi_aliases)
credit = None
target = None
if tgt is not None and tgt.get("target") is not None:
target = float(tgt["target"])
credit = _credit(float(k.get("actual", 0)), target,
tgt.get("direction") or k.get("direction") or "gte", floor)
kpi_results.append({
"canonical_name": k.get("canonical_name"), "name": k.get("name"),
"actual": k.get("actual"), "unit": k.get("unit") or "",
"direction": k.get("direction"), "profitability": bool(k.get("profitability")),
"target": target, "target_source": source, "matched_via": via,
"credit": None if credit is None else round(credit, 4),
})
prof_all = [r for r in kpi_results if r["profitability"]]
prof_hit = [r for r in prof_all if r["credit"] is not None]
other_hit = [r for r in kpi_results if not r["profitability"] and r["credit"] is not None]
# --- forecast integrity: this deck's actuals vs the prior deck's targets
forecast_results: list[dict] = []
dropped: list[str] = []
for pt in prior_targets or []:
cand, _via = match_kpi(pt.get("canonical_name") or "", kpis, kpi_aliases)
if cand is None:
dropped.append(pt.get("canonical_name") or pt.get("name") or "kpi")
continue
actual = float(cand.get("actual", 0))
target = float(pt.get("target", 0))
direction = pt.get("direction") or "gte"
if target == 0:
acc = 1.0 if _passes(actual, target, direction) else 0.0
else:
err = (actual - target) / abs(target)
if direction == "lte":
err = -err
e = abs(err) if err < 0 else abs(err) / 2.0 # overshoot penalized half
acc = 1.0 - min(1.0, e)
forecast_results.append({
"canonical_name": pt.get("canonical_name"), "target": target,
"actual": actual, "accuracy": round(acc, 4),
})
# Pinned targets that no reported actual matches count as dropped too.
for p in pinned_targets or []:
cand, _via = match_kpi(p.get("kpi") or "", kpis, kpi_aliases)
if cand is None:
dropped.append(p.get("kpi") or "kpi")
seen: set[str] = set()
dropped_unique = [d for d in dropped
if not (d.strip().lower() in seen or seen.add(d.strip().lower()))]
for name in dropped_unique[: int(weights.get("droppedKpiMax", 3))]:
scoring_flags.append({
"code": "kpi_dropped",
"description": f"previously targeted KPI '{name}' is not reported this period",
"severity": int(weights.get("droppedKpiPenalty", 2)),
})
# --- quant buckets + renormalization (NA weight redistributes pro-rata)
prof = _bucket(prof_hit, float(weights.get("profitabilityKpi", 30)))
other = _bucket(other_hit, float(weights.get("otherKpi", 20)))
fmean = (sum(f["accuracy"] for f in forecast_results) / len(forecast_results)
if forecast_results else 0.0)
if forecast_results:
forecast = {"score": fmean, "weight": float(weights.get("forecastIntegrity", 10)),
"na": False, "kpi_count": len(forecast_results)}
else:
forecast = {"score": 0.0, "weight": 0.0, "na": True, "kpi_count": 0}
if not prof_all:
prof["na"] = True
prof["weight"] = 0.0
scoring_flags.append({
"code": "no_profitability_visibility",
"description": "no profit/margin/cash KPI reported at all",
"severity": 3,
})
total_quant_w = (float(weights.get("profitabilityKpi", 30))
+ float(weights.get("otherKpi", 20))
+ float(weights.get("forecastIntegrity", 10)))
present = [b for b in (prof, other, forecast) if not b["na"]]
if present:
scale = total_quant_w / sum(b["weight"] for b in present)
for b in present:
b["weight"] = round(b["weight"] * scale, 4)
b["score"] = round(b["score"] * b["weight"], 4)
quant_score = sum(b["score"] for b in present)
all_quant_na = False
else:
quant_score = 0.0
all_quant_na = True
scoring_flags.append({
"code": "no_quantitative_kpis",
"description": "no quantitative bucket could be scored (no targeted KPIs, "
"no prior targets)",
"severity": 4,
})
# --- qualitative
qual_score, categories = _qual(grades, weights)
qual_max = 8.0 * float(weights.get("qualCategoryMax", 5))
# --- red flags: extractor + graders + scoring; dedup, damp single-source
damp = float(weights.get("singleSourceFlagFactor", 0.5))
cap = float(weights.get("redFlagCap", 15))
flag_map: dict[str, dict] = {}
def add_flag(f: dict, source: str, scoring_flag: bool = False):
code = (f.get("code") or "flag").strip().lower()
key = f"{code}:{f.get('description', '')}" if scoring_flag and code == "kpi_dropped" else code
sev = int(f.get("severity", 1))
cur = flag_map.get(key)
if cur is None:
flag_map[key] = {"code": code, "description": f.get("description") or "",
"severity": sev, "sources": {source}, "scoring": scoring_flag}
else:
if sev > cur["severity"]:
cur["severity"] = sev
cur["description"] = f.get("description") or cur["description"]
cur["sources"].add(source)
cur["scoring"] = cur["scoring"] or scoring_flag
for f in extraction.get("red_flag_candidates") or []:
add_flag(f, "extractor")
for g in grades or []:
for f in g.get("red_flags") or []:
add_flag(f, g.get("grader") or "grader")
for f in scoring_flags:
add_flag(f, "scoring", scoring_flag=True)
flags: list[dict] = []
for f in flag_map.values():
full = f["scoring"] or len(f["sources"]) >= 2
points = float(f["severity"]) if full else float(f["severity"]) * damp
flags.append({"code": f["code"], "description": f["description"],
"severity": f["severity"], "points": round(points, 4),
"sources": sorted(f["sources"])})
flags.sort(key=lambda f: (-f["points"], f["code"]))
penalty_total = round(min(cap, sum(f["points"] for f in flags)), 4)
# --- composite
if all_quant_na:
base = (qual_score / qual_max * 100.0) if qual_max else 0.0
else:
base = quant_score + qual_score
composite = round(max(0.0, min(100.0, base - penalty_total)), 1)
return {
"schema_version": 1,
"company": meta.get("company"),
"period": meta.get("period"),
"deck_id": meta.get("deck_id"),
"job_id": meta.get("job_id"),
"graded_at": meta.get("graded_at"),
"composite": composite,
"quant": {"score": round(quant_score, 4), "profitability": prof,
"other": other, "forecast_integrity": forecast},
"qual": {"score": round(qual_score, 4), "categories": categories},
"penalties": {"total": penalty_total, "flags": flags},
"kpi_results": kpi_results,
"forecast_results": forecast_results,
"panel": meta.get("panel") or [],
"artifacts": meta.get("artifacts", {}),
"narrative": extraction.get("narrative", {}),
}
orchestrator/templates/index.html
+399 -39
View File
@@ -6,7 +6,7 @@
<title>Boardroom Map</title>
<style>
:root { --bg:#10131a; --panel:#181d27; --edge:#2a313f; --ink:#e9edf5; --dim:#97a1b5;
--accent:#c9a24b; --ok:#5fd08a; --warn:#ffcf66; --bad:#ff7a7a; }
--accent:#c9a24b; --ok:#5fd08a; --yg:#b9d05f; --warn:#ffcf66; --bad:#ff7a7a; }
* { box-sizing:border-box; }
body { margin:0; font:14px/1.5 system-ui,-apple-system,Segoe UI,Roboto,sans-serif;
background:var(--bg); color:var(--ink); }
@@ -20,7 +20,7 @@
.card h2 { margin:0 0 10px; font-size:13px; text-transform:uppercase; letter-spacing:1px; color:var(--dim); }
.pill { display:inline-block; padding:2px 10px; border-radius:999px; font-size:12px; font-weight:600; }
.pill.idle,.pill.done{background:#26303f;color:var(--dim)}
.pill.reviewing,.pill.synthesizing,.pill.collecting,.pill.extracting{background:#3b3414;color:var(--warn)}
.pill.busy{background:#3b3414;color:var(--warn)}
.pill.error{background:#3b1414;color:var(--bad)}
.kv { display:flex; justify-content:space-between; padding:4px 0; border-bottom:1px dashed var(--edge); }
.kv:last-child{border:0} .kv .k{color:var(--dim)}
@@ -28,15 +28,57 @@
.dot.on{background:var(--ok)} .dot.off{background:#55608f}
pre { background:#0c0f16; border:1px solid var(--edge); border-radius:10px; padding:12px; margin:0;
max-height:340px; overflow:auto; font:12px/1.5 ui-monospace,Menlo,monospace; color:#cdd6ff; white-space:pre-wrap; }
.row{display:flex;gap:8px;flex-wrap:wrap;margin-top:10px}
pre.tall{max-height:520px}
.row{display:flex;gap:8px;flex-wrap:wrap;margin-top:10px;align-items:center}
button{background:#222a3a;color:var(--ink);border:1px solid var(--edge);border-radius:9px;
padding:8px 12px;font-size:13px;cursor:pointer} button:hover{background:#2c3650}
button.primary{background:#3a2f12;border-color:#6b551f;color:#ffdf9a}
.drop{border:1.5px dashed var(--edge);border-radius:12px;padding:18px;text-align:center;color:var(--dim);cursor:pointer}
button.mini{padding:2px 8px;font-size:11px;border-radius:7px}
select,input[type=text]{background:#0c0f16;color:var(--ink);border:1px solid var(--edge);
border-radius:9px;padding:7px 10px;font-size:13px}
.drop{border:1.5px dashed var(--edge);border-radius:12px;padding:18px;text-align:center;color:var(--dim);cursor:pointer;margin-top:10px}
.drop.hot{border-color:var(--accent);color:var(--ink)}
.badge{font-size:11px;color:var(--dim)}
.muted{color:var(--dim);font-size:12px}
a{color:#9ab8ff}
/* portfolio */
table.port{width:100%;border-collapse:collapse}
table.port th{font-size:11px;text-transform:uppercase;letter-spacing:1px;color:var(--dim);
text-align:left;padding:4px 8px;border-bottom:1px solid var(--edge)}
table.port td{padding:7px 8px;border-bottom:1px dashed var(--edge);vertical-align:middle}
table.port tr.co{cursor:pointer} table.port tr.co:hover td{background:#1e2532}
.score{display:inline-block;min-width:46px;text-align:center;padding:2px 8px;border-radius:8px;
font-weight:700;font-size:13px}
.score.green{background:#15351f;color:var(--ok)} .score.yg{background:#2c3315;color:var(--yg)}
.score.amber{background:#3b3414;color:var(--warn)} .score.red{background:#3b1414;color:var(--bad)}
.score.none{background:#20263200;color:var(--dim);font-weight:400}
.delta{font-size:12px;font-weight:600;margin-left:4px}
.delta.up{color:var(--ok)} .delta.dn{color:var(--bad)} .delta.flat{color:var(--dim)}
.tag{display:inline-block;padding:1px 7px;border-radius:999px;font-size:10px;font-weight:600;
letter-spacing:.5px;text-transform:uppercase}
.tag.unreg{background:#3b3414;color:var(--warn)}
.tag.profit{background:#15351f;color:var(--ok)}
.chip{display:inline-block;padding:2px 9px;border-radius:999px;font-size:11px;margin:2px 4px 2px 0;
background:#222a3a;border:1px solid var(--edge)}
.chip.done{background:#15351f;color:var(--ok);border-color:#1f4a2c}
.chip.error{background:#3b1414;color:var(--bad);border-color:#552}
.chip.busy{background:#3b3414;color:var(--warn);border-color:#554416}
.chip.warn{background:#3b1414;color:var(--bad);border-color:#552222}
.chip.period{background:#1c2430;color:#9ab8ff;border-color:#2a3a55}
/* detail */
.detail-grid{display:grid;grid-template-columns:1fr 1fr;gap:18px;margin-top:12px}
@media (max-width:900px){ .detail-grid{grid-template-columns:1fr} .wrap{grid-template-columns:1fr} }
h3.sub{margin:14px 0 6px;font-size:12px;text-transform:uppercase;letter-spacing:1px;color:var(--dim)}
table.mini{width:100%;border-collapse:collapse;font-size:12px}
table.mini th{font-size:10px;text-transform:uppercase;letter-spacing:1px;color:var(--dim);
text-align:left;padding:3px 6px;border-bottom:1px solid var(--edge)}
table.mini td{padding:4px 6px;border-bottom:1px dashed var(--edge)}
.flag{border-left:3px solid var(--bad);background:#221318;border-radius:0 8px 8px 0;
padding:6px 10px;margin:6px 0;font-size:12px}
.flag .code{font:11px ui-monospace,Menlo,monospace;color:var(--bad)}
.inbox-co{margin-top:8px}
.inbox-co .co-name{font-weight:600;font-size:12px;color:var(--accent);letter-spacing:.5px}
details.jsonv summary{cursor:pointer;color:var(--dim);font-size:12px;margin:8px 0 4px}
</style>
</head>
<body>
@@ -48,16 +90,56 @@
</header>
<div class="wrap">
<div class="card full">
<h2>Portfolio</h2>
<div id="portfolio"><div class="muted">Loading…</div></div>
</div>
<div class="card full" id="companyCard" style="display:none">
<h2 style="display:flex;align-items:center;gap:10px">Company —
<span id="coName" style="color:var(--ink);letter-spacing:0;text-transform:none;font-size:15px"></span>
<span id="coBadges"></span>
<button class="mini" style="margin-left:auto" onclick="closeCompany()">close</button>
</h2>
<div id="coTrend"></div>
<div class="detail-grid">
<div>
<h3 class="sub">BDEF categories (latest, tick = previous)</h3>
<div id="coCats"><div class="muted">—</div></div>
<h3 class="sub">Open red flags (latest deck)</h3>
<div id="coFlags"><div class="muted">—</div></div>
</div>
<div>
<h3 class="sub">KPI hit rate</h3>
<div id="coKpis"><div class="muted">—</div></div>
</div>
</div>
<h3 class="sub">Deck history</h3>
<div id="coDecks"><div class="muted">—</div></div>
<div class="row">
<button onclick="toggleScorecard()" id="scBtn">View SCORECARD.md</button>
</div>
<div id="coViewer" style="margin-top:10px"></div>
</div>
<div class="card">
<h2>Documents</h2>
<div id="drop" class="drop">Drop confidential documents here, or click to choose<br>
<span class="badge">PDF · DOCX · TXT · MD</span></div>
<h2>Drop decks</h2>
<div class="row" style="margin-top:0">
<label class="muted" for="coSelect">Company</label>
<select id="coSelect" onchange="onCoSelect()"></select>
<span id="newCoWrap" style="display:none">
<input id="newCoName" type="text" placeholder="new company name" oninput="slugPreview()"/>
<span class="badge" id="slugPrev"></span>
</span>
</div>
<div id="drop" class="drop">Drop board decks here, or click to choose<br>
<span class="badge">PDF · PPTX · DOCX · TXT · MD — period parsed from filename (2026-Q2, FY2026…)</span></div>
<input id="file" type="file" multiple style="display:none"/>
<div id="inbox" style="margin-top:10px"></div>
<div class="row">
<button class="primary" onclick="runReview()">Run Review</button>
<button class="primary" onclick="act('/api/run','POST')">Grade decks</button>
<button onclick="act('/api/inbox/clear','POST')">Clear Inbox</button>
<button onclick="refresh()">Refresh</button>
<button onclick="refresh(); loadCompanies()">Refresh</button>
</div>
</div>
@@ -67,15 +149,15 @@
</div>
<div class="card">
<h2>Panel</h2>
<div id="panel"><div class="muted">No reviewers configured.</div></div>
<h2>Grading panel</h2>
<div id="panel"><div class="muted">No graders configured.</div></div>
</div>
<div class="card">
<h2>Serving / Job</h2>
<div id="job"><div class="muted">—</div></div>
<div class="row">
<button onclick="act('/api/reviewer/build-image','POST')">Build reviewer image</button>
<button onclick="act('/api/grader/build-image','POST')">Build grader image</button>
<button onclick="act('/api/stop','POST')">Stop serving</button>
</div>
</div>
@@ -84,72 +166,350 @@
<h2>Activity log</h2>
<pre id="log">loading…</pre>
</div>
<div class="card full">
<h2>Latest report</h2>
<pre id="report">—</pre>
</div>
</div>
<script>
// ------------------------------------------------------------------ helpers
function esc(s){ return String(s==null?'':s).replace(/[&<>"']/g,
c=>({'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;',"'":'&#39;'})[c]); }
async function getJSON(u){ const r=await fetch(u); if(!r.ok) throw new Error(await r.text()); return r.json(); }
async function act(u,m){ try{ const r=await fetch(u,{method:m}); const t=await r.text();
alert(r.ok? 'OK: '+t.slice(0,400) : 'Error: '+t.slice(0,400)); refresh(); }catch(e){ alert(e); } }
function dot(on,label){ return `<div><span class="dot ${on?'on':'off'}"></span>${label}</div>`; }
async function runReview(){ act('/api/run','POST'); }
function bandCls(c){ return c>=75?'green':c>=60?'yg':c>=45?'amber':'red'; }
function scoreBadge(c){ if(c==null||isNaN(+c)) return '<span class="score none">—</span>';
return `<span class="score ${bandCls(+c)}">${(+c).toFixed(1)}</span>`; }
function deltaArrow(d){ if(d==null||isNaN(+d)) return '';
if(Math.abs(+d)<0.05) return '<span class="delta flat">→ 0.0</span>';
return +d>0? `<span class="delta up">▲ ${(+d).toFixed(1)}</span>`
: `<span class="delta dn">▼ ${Math.abs(+d).toFixed(1)}</span>`; }
// Inline SVG sparkline: min-max normalized polyline (~120x28).
function sparkline(hist,w,h){
w=w||120; h=h||28;
const vals=(hist||[]).map(p=>+p.composite).filter(v=>!isNaN(v));
if(!vals.length) return '<span class="muted">—</span>';
if(vals.length===1) return `<svg width="${w}" height="${h}" viewBox="0 0 ${w} ${h}">`+
`<circle cx="${w/2}" cy="${h/2}" r="2.5" fill="var(--accent)"/></svg>`;
const mn=Math.min(...vals), mx=Math.max(...vals), span=(mx-mn)||1, pad=3;
const pts=vals.map((v,i)=>
`${(pad+i*(w-2*pad)/(vals.length-1)).toFixed(1)},${(h-pad-(v-mn)*(h-2*pad)/span).toFixed(1)}`).join(' ');
return `<svg width="${w}" height="${h}" viewBox="0 0 ${w} ${h}">`+
`<polyline points="${pts}" fill="none" stroke="var(--accent)" stroke-width="1.5"/></svg>`;
}
// Large composite trend with period labels + value dots.
function bigTrend(hist){
const pts=(hist||[]).filter(p=>p.composite!=null && !isNaN(+p.composite));
if(!pts.length) return '<div class="muted">No graded decks yet.</div>';
const w=680,h=150,padL=14,padR=14,padT=18,padB=26;
const vals=pts.map(p=>+p.composite);
const mn=Math.min(...vals), mx=Math.max(...vals), span=(mx-mn)||1;
const x=i=> pts.length===1? w/2 : padL+i*(w-padL-padR)/(pts.length-1);
const y=v=> h-padB-(v-mn)*(h-padT-padB)/span;
let svg=`<svg width="100%" viewBox="0 0 ${w} ${h}" style="max-width:${w}px">`;
if(pts.length>1)
svg+=`<polyline points="${pts.map((p,i)=>`${x(i).toFixed(1)},${y(+p.composite).toFixed(1)}`).join(' ')}" `+
`fill="none" stroke="var(--accent)" stroke-width="2"/>`;
const step=Math.max(1,Math.ceil(pts.length/10));
pts.forEach((p,i)=>{
const px=x(i), py=y(+p.composite);
svg+=`<circle cx="${px.toFixed(1)}" cy="${py.toFixed(1)}" r="3.5" fill="var(--accent)"/>`;
svg+=`<text x="${px.toFixed(1)}" y="${(py-8).toFixed(1)}" text-anchor="middle" `+
`font-size="10" fill="var(--ink)">${(+p.composite).toFixed(1)}</text>`;
if(i%step===0 || i===pts.length-1)
svg+=`<text x="${px.toFixed(1)}" y="${h-8}" text-anchor="middle" font-size="10" `+
`fill="var(--dim)">${esc(p.period||'?')}</text>`;
});
return svg+'</svg>';
}
// Horizontal BDEF A–H bars (0–5), previous value as a tick marker.
const BDEF={A:'Incentive alignment',B:'Inversion discipline',C:'Circle of competence',
D:'Capital allocation',E:'Moat durability',F:'Psych / cultural health',
G:'Simplicity & velocity',H:'Board value-add'};
function bdefBars(cats){
const ids=Object.keys(BDEF).filter(id=>cats[id]);
if(!ids.length) return '<div class="muted">—</div>';
const rowH=26,w=460,lab=170,barW=w-lab-46;
let svg=`<svg width="100%" viewBox="0 0 ${w} ${ids.length*rowH}" style="max-width:${w}px">`;
ids.forEach((id,r)=>{
const cy=r*rowH+rowH/2, c=cats[id]||{};
const cur=c.latest_adjusted, prev=c.previous_adjusted;
svg+=`<text x="0" y="${cy+4}" font-size="11" fill="var(--dim)">${id} · ${esc(BDEF[id])}</text>`;
svg+=`<rect x="${lab}" y="${cy-6}" width="${barW}" height="12" rx="6" fill="#0c0f16" stroke="var(--edge)"/>`;
if(cur!=null && !isNaN(+cur)){
const bw=Math.max(2,Math.min(1,+cur/5)*barW);
svg+=`<rect x="${lab}" y="${cy-6}" width="${bw.toFixed(1)}" height="12" rx="6" fill="var(--accent)"/>`;
svg+=`<text x="${lab+barW+8}" y="${cy+4}" font-size="11" fill="var(--ink)">${(+cur).toFixed(1)}</text>`;
}
if(prev!=null && !isNaN(+prev)){
const tx=lab+Math.min(1,+prev/5)*barW;
svg+=`<line x1="${tx.toFixed(1)}" y1="${cy-9}" x2="${tx.toFixed(1)}" y2="${cy+9}" `+
`stroke="var(--ink)" stroke-width="1.5" opacity="0.7"/>`;
}
});
return svg+'</svg>';
}
// Client-side mirror of decks.py period parsing (for inbox display).
const PERIOD_RES=[
[/(?:^|\D)((?:19|20)\d\d)[-_ ]?[Qq]([1-4])(?!\d)/, m=>m[1]+'-Q'+m[2]],
[/(?:^|[^A-Za-z0-9])[Qq]([1-4])[-_ ]((?:19|20)\d\d)(?!\d)/, m=>m[2]+'-Q'+m[1]],
[/(?:^|\D)((?:19|20)\d\d)[-_ ]?[Hh]([12])(?!\d)/, m=>m[1]+'-H'+m[2]],
[/(?:^|[^A-Za-z0-9])[Ff][Yy][-_ ]?((?:19|20)\d\d)(?!\d)/, m=>'FY'+m[1]],
[/(?:^|\D)((?:19|20)\d\d)[-_](0[1-9]|1[0-2])(?!\d)/, m=>m[1]+'-'+m[2]],
];
function parsePeriod(name){
for(const [re,f] of PERIOD_RES){ const m=String(name||'').match(re); if(m) return f(m); }
return null;
}
function slugify(s){ return String(s||'').toLowerCase().replace(/[^a-z0-9]+/g,'-')
.replace(/^-+|-+$/g,'') || 'company'; }
// ------------------------------------------------------------------ portfolio
let companiesCache=[], currentSlug=null;
async function loadCompanies(){
try{
const d=await getJSON('/api/companies');
companiesCache=d.companies||[];
renderPortfolio(); renderCoSelect();
if(currentSlug) openCompany(currentSlug,true);
}catch(e){
document.getElementById('portfolio').innerHTML='<div class="muted">companies error: '+esc(e.message||e)+'</div>';
}
}
function renderPortfolio(){
const el=document.getElementById('portfolio');
if(!companiesCache.length){
el.innerHTML='<div class="muted">No companies yet — drop a deck below to create one.</div>'; return;
}
el.innerHTML='<table class="port"><thead><tr>'+
'<th>Company</th><th>Latest</th><th>Δ</th><th>Trend</th><th>Decks</th></tr></thead><tbody>'+
companiesCache.map(c=>{
const l=c.latest||{};
return `<tr class="co" onclick="openCompany('${esc(c.slug)}')">`+
`<td>${esc(c.name)} ${c.auto_created?'<span class="tag unreg">unregistered</span>':''}</td>`+
`<td>${scoreBadge(l.composite)} <span class="badge">${esc(l.period||'')}</span></td>`+
`<td>${deltaArrow(l.delta)}</td>`+
`<td>${sparkline(c.history)}</td>`+
`<td class="muted">${c.deck_count||0}</td></tr>`;
}).join('')+'</tbody></table>';
}
// ------------------------------------------------------------------ company detail
async function openCompany(slug,silent){
try{
const d=await getJSON('/api/companies/'+encodeURIComponent(slug));
currentSlug=slug;
renderDetail(d);
document.getElementById('companyCard').style.display='';
if(!silent) document.getElementById('companyCard').scrollIntoView({behavior:'smooth'});
}catch(e){ if(!silent) alert('load company failed: '+(e.message||e)); }
}
function closeCompany(){ currentSlug=null; document.getElementById('companyCard').style.display='none'; }
function renderDetail(d){
const co=d.company||{}, recs=d.records||[];
document.getElementById('coName').textContent=co.name||co.slug||'?';
document.getElementById('coBadges').innerHTML=co.auto_created?'<span class="tag unreg">unregistered</span>':'';
document.getElementById('coTrend').innerHTML=
bigTrend(recs.map(r=>({period:r.period,composite:r.composite})));
document.getElementById('coCats').innerHTML=bdefBars(d.categories_latest||{});
// KPI hit-rate table — profitability KPIs pinned to top.
const kpis=Object.entries(d.kpi_hit_rate||{})
.map(([cn,s])=>Object.assign({canonical:cn},s))
.sort((a,b)=>(b.profitability?1:0)-(a.profitability?1:0)||a.canonical.localeCompare(b.canonical));
document.getElementById('coKpis').innerHTML=!kpis.length? '<div class="muted">—</div>' :
'<table class="mini"><thead><tr><th>KPI</th><th></th><th>Last credit</th><th>Hits</th><th>Streak</th></tr></thead><tbody>'+
kpis.map(k=>`<tr><td title="${esc(k.canonical)}">${esc(k.name||k.canonical)}</td>`+
`<td>${k.profitability?'<span class="tag profit">profit</span>':''}</td>`+
`<td>${k.last_credit==null?'—':(k.last_credit*100).toFixed(0)+'%'}</td>`+
`<td class="muted">${k.hits||0}/${k.attempts||0}</td>`+
`<td class="muted">${k.streak||0}</td></tr>`).join('')+'</tbody></table>';
// Open red flags (latest deck).
const flags=d.open_flags||[];
document.getElementById('coFlags').innerHTML=!flags.length? '<div class="muted">none</div>' :
flags.map(f=>`<div class="flag"><span class="code">${esc(f.code)}</span>`+
` <span class="badge">severity ${esc(f.severity)}${f.points!=null?' · -'+esc(f.points)+' pts':''}</span>`+
`<br>${esc(f.description)}</div>`).join('');
// Deck history (newest first).
document.getElementById('coDecks').innerHTML=!recs.length? '<div class="muted">no graded decks yet</div>' :
'<table class="mini"><thead><tr><th>Period</th><th>Composite</th><th>Graded</th><th></th></tr></thead><tbody>'+
recs.slice().reverse().map(r=>
`<tr><td>${esc(r.period||'?')}</td><td>${scoreBadge(r.composite)}</td>`+
`<td class="muted">${esc(String(r.graded_at||'').slice(0,16).replace('T',' '))}</td>`+
`<td><button class="mini" onclick="viewDeckReport('${esc(r.deck_id)}')">report</button> `+
`<button class="mini" onclick="viewDeckJson('${esc(r.deck_id)}')">json</button></td></tr>`).join('')+
'</tbody></table>';
document.getElementById('coViewer').innerHTML='';
document.getElementById('scBtn').textContent='View SCORECARD.md';
}
async function viewDeckReport(deckId){
if(!currentSlug) return;
const base='/api/companies/'+encodeURIComponent(currentSlug)+'/decks/'+encodeURIComponent(deckId);
try{
const r=await fetch(base+'/report');
const t=await r.text();
setViewer('Deck report — '+deckId, `<pre class="tall">${esc(r.ok?t:('error: '+t))}</pre>`);
}catch(e){ alert(e); }
}
async function viewDeckJson(deckId){
if(!currentSlug) return;
try{
const d=await getJSON('/api/companies/'+encodeURIComponent(currentSlug)+
'/decks/'+encodeURIComponent(deckId));
setViewer('Deck record — '+deckId,
`<details class="jsonv" open><summary>collapse / expand raw JSON</summary>`+
`<pre class="tall">${esc(JSON.stringify(d,null,2))}</pre></details>`);
}catch(e){ alert('load record failed: '+(e.message||e)); }
}
let scorecardOpen=false;
async function toggleScorecard(){
if(!currentSlug) return;
const v=document.getElementById('coViewer'), btn=document.getElementById('scBtn');
if(scorecardOpen){ v.innerHTML=''; scorecardOpen=false; btn.textContent='View SCORECARD.md'; return; }
try{
const r=await fetch('/api/companies/'+encodeURIComponent(currentSlug)+'/scorecard');
const t=await r.text();
setViewer('SCORECARD.md', `<pre class="tall">${esc(r.ok?t:('error: '+t))}</pre>`);
scorecardOpen=true; btn.textContent='Hide SCORECARD.md';
}catch(e){ alert(e); }
}
function setViewer(title,html){
scorecardOpen=false;
document.getElementById('scBtn').textContent='View SCORECARD.md';
document.getElementById('coViewer').innerHTML=
`<h3 class="sub" style="display:flex;align-items:center">${esc(title)}`+
`<button class="mini" style="margin-left:auto" onclick="document.getElementById('coViewer').innerHTML=''">close</button></h3>`+html;
}
// ------------------------------------------------------------------ drop card
function renderCoSelect(){
const sel=document.getElementById('coSelect');
const cur=sel.value;
sel.innerHTML=companiesCache.map(c=>
`<option value="${esc(c.slug)}">${esc(c.name)}</option>`).join('')+
'<option value="__new__">new company…</option>';
if(cur && [...sel.options].some(o=>o.value===cur)) sel.value=cur;
else if(!companiesCache.length) sel.value='__new__';
onCoSelect();
}
function onCoSelect(){
const isNew=document.getElementById('coSelect').value==='__new__';
document.getElementById('newCoWrap').style.display=isNew?'':'none';
slugPreview();
}
function slugPreview(){
const n=document.getElementById('newCoName').value.trim();
document.getElementById('slugPrev').textContent=n?('→ /data/inbox/'+slugify(n)+'/'):'';
}
function uploadCompany(){
const sel=document.getElementById('coSelect');
if(sel.value!=='__new__') return sel.value;
const n=document.getElementById('newCoName').value.trim();
return n? slugify(n) : null;
}
const drop=document.getElementById('drop'), file=document.getElementById('file');
drop.onclick=()=>file.click();
file.onchange=()=>upload(file.files);
file.onchange=()=>{ upload(file.files); file.value=''; };
['dragover','dragenter'].forEach(e=>drop.addEventListener(e,ev=>{ev.preventDefault();drop.classList.add('hot');}));
['dragleave','drop'].forEach(e=>drop.addEventListener(e,ev=>{ev.preventDefault();drop.classList.remove('hot');}));
drop.addEventListener('drop',ev=>{ if(ev.dataTransfer.files.length) upload(ev.dataTransfer.files); });
async function upload(files){
const co=uploadCompany();
if(!co){ alert('Pick a company (or type a new company name) before uploading — root-level files are not graded.'); return; }
const names=[...files].map(f=>f.name+(parsePeriod(f.name)?' ['+parsePeriod(f.name)+']':' [no period in name]'));
const fd=new FormData(); for(const f of files) fd.append('files',f);
try{ const r=await fetch('/api/upload',{method:'POST',body:fd});
if(!r.ok) alert('Upload failed: '+(await r.text()).slice(0,300)); refresh(); }catch(e){ alert(e); }
try{
const r=await fetch('/api/upload?company='+encodeURIComponent(co),{method:'POST',body:fd});
if(!r.ok) alert('Upload failed: '+(await r.text()).slice(0,300));
else console.log('uploaded to '+co+': '+names.join(', '));
refresh(); loadCompanies();
}catch(e){ alert(e); }
}
function renderInbox(ib){
const el=document.getElementById('inbox');
const cos=(ib&&ib.companies)||{}, skipped=(ib&&ib.skipped)||[];
const slugs=Object.keys(cos).sort();
let html='';
for(const slug of slugs){
html+=`<div class="inbox-co"><span class="co-name">${esc(slug)}</span>`+
cos[slug].map(f=>{
const per=f.period||parsePeriod(f.name);
return `<div class="kv"><span>${esc(f.name)} `+
(per?`<span class="chip period">${esc(per)}</span>`:'<span class="badge">period?</span>')+
(f.supported===false?' <span class="badge">unsupported</span>':'')+
`</span><span class="muted">${f.bytes?((f.bytes/1024).toFixed(0)+' KB'):''}</span></div>`;
}).join('')+'</div>';
}
if(skipped.length)
html+=`<div style="margin-top:8px"><span class="chip warn">skipped (no company): ${esc(skipped.join(', '))}</span></div>`;
el.innerHTML=html||'<div class="muted">Inbox empty.</div>';
}
// ------------------------------------------------------------------ polling
let lastPhase=null;
async function refresh(){
try{
const s = await getJSON('/api/status');
const rt = s.runtime||{};
const ph = document.getElementById('phase');
ph.textContent = rt.phase||'idle'; ph.className = 'pill '+(rt.phase||'idle');
const phase = rt.phase||'idle';
ph.textContent = phase;
ph.className = 'pill '+(phase==='idle'||phase==='done'?phase==='done'?'done':'idle':phase==='error'?'error':'busy');
if(lastPhase && lastPhase!==phase && (phase==='done'||phase==='idle')) loadCompanies();
lastPhase=phase;
document.getElementById('netmode').textContent =
(s.networkMode==='airgapped'?'air-gapped':'local-services')
+ (s.synthesis?' · synthesis on':'') + (s.autoRunOnDrop?' · auto-run':'');
+ (s.adjudicator?' · adjudicator on':'') + (s.autoRunOnDrop?' · auto-run':'');
document.getElementById('inbox').innerHTML = (s.inbox||[]).length
? (s.inbox||[]).map(f=>`<div class="kv"><span>${f.name} ${f.supported?'':'<span class="badge">unsupported</span>'}</span><span class="muted">${(f.bytes/1024).toFixed(0)} KB</span></div>`).join('')
: '<div class="muted">Inbox empty.</div>';
renderInbox(s.inbox);
document.getElementById('ready').innerHTML =
dot(s.configured.sparks,'Sparks configured') +
dot(s.configured.models>0,`${s.configured.models} model(s)`) +
dot(s.configured.reviewers>0,`${s.configured.reviewers} reviewer(s)`) +
`<div class="kv"><span class="k">network mode</span><span>${s.networkMode}</span></div>` +
dot(s.configured.graders>0,`${s.configured.graders} grader(s)`) +
`<div class="kv"><span class="k">network mode</span><span>${esc(s.networkMode)}</span></div>` +
`<div class="kv"><span class="k">wipe docs after</span><span>${s.wipeRemoteDocs?'yes':'no'}</span></div>` +
(s.models||[]).map(m=>`<div class="kv"><span>${m.alias}</span><span class="muted">${m.hfModel} · ${m.spark}</span></div>`).join('');
(s.models||[]).map(m=>`<div class="kv"><span>${esc(m.alias)}</span><span class="muted">${esc(m.hfModel)} · ${esc(m.spark)}</span></div>`).join('');
document.getElementById('panel').innerHTML = (s.panel||[]).length
? (s.panel||[]).map(r=>`<div class="kv"><span>${r.name} ${r.known?'':'<span class="badge">unknown model</span>'}</span><span class="muted">${r.model}${r.persona?' · persona ✓':''}</span></div>`).join('')
: '<div class="muted">No reviewers configured.</div>';
? (s.panel||[]).map(g=>`<div class="kv"><span>${esc(g.name)} ${g.known?'':'<span class="badge">unknown model</span>'}</span><span class="muted">${esc(g.model)}${g.persona?' · persona ✓':''}</span></div>`).join('')
: '<div class="muted">No graders configured.</div>';
const panelRt=(rt.panel||[]);
const deckChips=(rt.decks||[]).map(d=>{
const st=d.status||'pending';
const cls=st==='done'?'done':st==='error'?'error':(st==='pending'?'':'busy');
const tail=d.composite!=null?(' · '+(+d.composite).toFixed(1)):(d.error?' · error':'');
return `<span class="chip ${cls}">${esc(d.company)}${d.period?' '+esc(d.period):''}${tail}</span>`;
}).join('');
const curCo = rt.company||rt.current_company, curPer = rt.period||rt.current_period;
document.getElementById('job').innerHTML =
`<div class="kv"><span class="k">job</span><span>${rt.job_id||'—'}</span></div>` +
(rt.waves_total?`<div class="kv"><span class="k">wave</span><span>${rt.wave_index}/${rt.waves_total}</span></div>`:'') +
(rt.message?`<div class="muted" style="margin-top:6px">${rt.message}</div>`:'') +
panelRt.map(p=>`<div class="kv"><span>${p.name}</span><span class="muted">${p.status||''}</span></div>`).join('');
`<div class="kv"><span class="k">job</span><span>${esc(rt.job_id||'—')}</span></div>` +
(rt.decks_total?`<div class="kv"><span class="k">deck</span><span>${esc(rt.deck_index)}/${esc(rt.decks_total)}${curCo?' · '+esc(curCo)+(curPer?' '+esc(curPer):''):''}</span></div>`:'') +
(rt.waves_total?`<div class="kv"><span class="k">wave</span><span>${esc(rt.wave_index)}/${esc(rt.waves_total)}</span></div>`:'') +
(rt.message?`<div class="muted" style="margin-top:6px">${esc(rt.message)}</div>`:'') +
(deckChips?`<div style="margin-top:8px">${deckChips}</div>`:'') +
(rt.panel||[]).map(p=>`<div class="kv"><span>${esc(p.name)}</span><span class="muted">${esc(p.status||'')}</span></div>`).join('');
const ev = await getJSON('/api/events');
document.getElementById('log').textContent = (ev.events||[]).slice(-200).join('\n') || '(no activity yet)';
try{ const rep = await (await fetch('/api/report')).text();
document.getElementById('report').textContent = rep; }catch(e){}
}catch(e){ document.getElementById('log').textContent = 'status error: '+e; }
}
refresh(); setInterval(refresh, 6000);
refresh(); loadCompanies();
setInterval(refresh, 6000);
setInterval(loadCompanies, 60000);
</script>
</body>
</html>
orchestrator/tests/fixtures/extraction_q1.json
Vendored
+29
View File
@@ -0,0 +1,29 @@
{
"schema_version": 1,
"deck": {
"company_hint": "Acme Robotics",
"period": "2026-Q1",
"meeting_date": "2026-04-15",
"title": "Acme Robotics — Q1 2026 Board Deck",
"truncated": false
},
"kpis": [
{"name": "ARR", "canonical_name": "arr", "actual": 10.0, "unit": "$M", "period": "2026-Q1", "direction": "gte", "profitability": false, "target_in_deck": 9.5, "source": "slide 3, financial summary", "notes": ""},
{"name": "EBITDA Margin", "canonical_name": "ebitda_margin", "actual": -5.0, "unit": "%", "period": "2026-Q1", "direction": "gte", "profitability": true, "target_in_deck": -6.0, "source": "slide 4, P&L bridge", "notes": ""},
{"name": "Logo Churn", "canonical_name": "churn_rate", "actual": 4.0, "unit": "%", "period": "2026-Q1", "direction": "lte", "profitability": false, "target_in_deck": 5.0, "source": "slide 5, retention", "notes": ""},
{"name": "Cash Balance", "canonical_name": "cash_balance", "actual": 12.0, "unit": "$M", "period": "2026-Q1", "direction": "gte", "profitability": true, "target_in_deck": null, "source": "slide 4, balance sheet", "notes": ""}
],
"forward_targets": [
{"name": "ARR", "canonical_name": "arr", "target": 12.0, "unit": "$M", "target_period": "2026-Q2", "direction": "gte", "profitability": false, "source": "slide 9, guidance"},
{"name": "Logo Churn", "canonical_name": "churn_rate", "target": 4.0, "unit": "%", "target_period": "2026-Q2", "direction": "lte", "profitability": false, "source": "slide 9, guidance"},
{"name": "EBITDA Margin", "canonical_name": "ebitda_margin", "target": -2.0, "unit": "%", "target_period": "2026-Q2", "direction": "gte", "profitability": true, "source": "slide 9, guidance"},
{"name": "Qualified Pipeline", "canonical_name": "qualified_pipeline", "target": 30.0, "unit": "$M", "target_period": "2026-Q2", "direction": "gte", "profitability": false, "source": "slide 10, pipeline build"}
],
"red_flag_candidates": [
{"code": "hockey_stick_forecast", "description": "H2 revenue ramp shown with no downside case or stated falsifiers", "severity": 3, "evidence": "slide 9 guidance chart"}
],
"narrative": {
"summary": "Solid Q1: ARR beat plan at $10.0M, EBITDA margin improved to -5%, churn under plan. The H2 story rests entirely on the $30M qualified pipeline building as projected.",
"asks": ["Approve $2M expansion of the Austin integration facility"]
}
}
orchestrator/tests/fixtures/extraction_q2.json
Vendored
+26
View File
@@ -0,0 +1,26 @@
{
"schema_version": 1,
"deck": {
"company_hint": "Acme Robotics",
"period": "2026-Q2",
"meeting_date": "2026-07-14",
"title": "Acme Robotics — Q2 2026 Board Deck",
"truncated": false
},
"kpis": [
{"name": "ARR", "canonical_name": "arr", "actual": 11.0, "unit": "$M", "period": "2026-Q2", "direction": "gte", "profitability": false, "target_in_deck": null, "source": "slide 3, financial summary", "notes": ""},
{"name": "Logo Churn", "canonical_name": "churn_rate", "actual": 3.5, "unit": "%", "period": "2026-Q2", "direction": "lte", "profitability": false, "target_in_deck": null, "source": "slide 5, retention", "notes": ""},
{"name": "EBITDA Margin", "canonical_name": "ebitda_margin", "actual": -3.0, "unit": "%", "period": "2026-Q2", "direction": "gte", "profitability": true, "target_in_deck": null, "source": "slide 4, P&L bridge", "notes": ""},
{"name": "Cash Balance", "canonical_name": "cash_balance", "actual": 13.0, "unit": "$M", "period": "2026-Q2", "direction": "gte", "profitability": true, "target_in_deck": null, "source": "slide 4, balance sheet", "notes": ""}
],
"forward_targets": [
{"name": "ARR", "canonical_name": "arr", "target": 14.0, "unit": "$M", "target_period": "2026-Q3", "direction": "gte", "profitability": false, "source": "slide 9, guidance"}
],
"red_flag_candidates": [
{"code": "adjusted_metrics", "description": "EBITDA presented on an adjusted basis with no bridge to GAAP", "severity": 2, "evidence": "slide 4 footnote"}
],
"narrative": {
"summary": "Mixed Q2: ARR missed guidance at $11.0M vs $12.0M, churn beat, margin improved but missed the -2% target. Pipeline metric no longer reported.",
"asks": ["Approve revised FY2026 hiring plan"]
}
}
orchestrator/tests/fixtures/grade_a.json
Vendored
+103
View File
@@ -0,0 +1,103 @@
{
"schema_version": 1,
"grader": "grader-a",
"categories": [
{
"id": "A",
"score": 4,
"evidence": [
{
"quote": "Our compensation plan ties 60 percent of executive bonus to three-year ARR retention and free-cash-flow milestones rather than annual bookings, and every vice president now holds equity vesting over four years with a one-year cliff, which we believe aligns the team with long-term owners.",
"location": "slide 7"
}
],
"rationale": "Category A: specific, quantified disclosure with owner-aligned framing."
},
{
"id": "B",
"score": 4,
"evidence": [
{
"quote": "Our compensation plan ties 60 percent of executive bonus to three-year ARR retention and free-cash-flow milestones rather than annual bookings, and every vice president now holds equity vesting over four years with a one-year cliff, which we believe aligns the team with long-term owners.",
"location": "slide 7"
}
],
"rationale": "Category B: specific, quantified disclosure with owner-aligned framing."
},
{
"id": "C",
"score": 4,
"evidence": [
{
"quote": "Our compensation plan ties 60 percent of executive bonus to three-year ARR retention and free-cash-flow milestones rather than annual bookings, and every vice president now holds equity vesting over four years with a one-year cliff, which we believe aligns the team with long-term owners.",
"location": "slide 7"
}
],
"rationale": "Category C: specific, quantified disclosure with owner-aligned framing."
},
{
"id": "D",
"score": 4,
"evidence": [
{
"quote": "Our compensation plan ties 60 percent of executive bonus to three-year ARR retention and free-cash-flow milestones rather than annual bookings, and every vice president now holds equity vesting over four years with a one-year cliff, which we believe aligns the team with long-term owners.",
"location": "slide 7"
}
],
"rationale": "Category D: specific, quantified disclosure with owner-aligned framing."
},
{
"id": "E",
"score": 4,
"evidence": [
{
"quote": "Our compensation plan ties 60 percent of executive bonus to three-year ARR retention and free-cash-flow milestones rather than annual bookings, and every vice president now holds equity vesting over four years with a one-year cliff, which we believe aligns the team with long-term owners.",
"location": "slide 7"
}
],
"rationale": "Category E: specific, quantified disclosure with owner-aligned framing."
},
{
"id": "F",
"score": 4,
"evidence": [
{
"quote": "Our compensation plan ties 60 percent of executive bonus to three-year ARR retention and free-cash-flow milestones rather than annual bookings, and every vice president now holds equity vesting over four years with a one-year cliff, which we believe aligns the team with long-term owners.",
"location": "slide 7"
}
],
"rationale": "Category F: specific, quantified disclosure with owner-aligned framing."
},
{
"id": "G",
"score": 4,
"evidence": [
{
"quote": "Our compensation plan ties 60 percent of executive bonus to three-year ARR retention and free-cash-flow milestones rather than annual bookings, and every vice president now holds equity vesting over four years with a one-year cliff, which we believe aligns the team with long-term owners.",
"location": "slide 7"
}
],
"rationale": "Category G: specific, quantified disclosure with owner-aligned framing."
},
{
"id": "H",
"score": 2,
"evidence": [
{
"quote": "We ask the board to approve the revised hiring plan as presented; supporting detail is available from management upon request after the meeting, and we recommend approval without further discussion given the compressed agenda for this session.",
"location": "slide 11"
}
],
"rationale": "Asks are listed without recommendations or the inversion of the decision."
}
],
"red_flags": [
{
"code": "governance_gap",
"description": "succession and incentive redesign get one bullet while product minutiae fill nine slides",
"severity": 2,
"evidence": "slides 12-20"
}
],
"overall_comment": "Strong disclosure discipline overall; governance asks remain the weak spot."
}
orchestrator/tests/fixtures/grade_b.json
Vendored
+98
View File
@@ -0,0 +1,98 @@
{
"schema_version": 1,
"grader": "grader-b",
"categories": [
{
"id": "A",
"score": 4,
"evidence": [
{
"quote": "Our compensation plan ties 60 percent of executive bonus to three-year ARR retention and free-cash-flow milestones rather than annual bookings, and every vice president now holds equity vesting over four years with a one-year cliff, which we believe aligns the team with long-term owners.",
"location": "slide 7"
}
],
"rationale": "Category A: specific, quantified disclosure with owner-aligned framing."
},
{
"id": "B",
"score": 4,
"evidence": [
{
"quote": "Our compensation plan ties 60 percent of executive bonus to three-year ARR retention and free-cash-flow milestones rather than annual bookings, and every vice president now holds equity vesting over four years with a one-year cliff, which we believe aligns the team with long-term owners.",
"location": "slide 7"
}
],
"rationale": "Category B: specific, quantified disclosure with owner-aligned framing."
},
{
"id": "C",
"score": 4,
"evidence": [
{
"quote": "Our compensation plan ties 60 percent of executive bonus to three-year ARR retention and free-cash-flow milestones rather than annual bookings, and every vice president now holds equity vesting over four years with a one-year cliff, which we believe aligns the team with long-term owners.",
"location": "slide 7"
}
],
"rationale": "Category C: specific, quantified disclosure with owner-aligned framing."
},
{
"id": "D",
"score": 4,
"evidence": [
{
"quote": "Our compensation plan ties 60 percent of executive bonus to three-year ARR retention and free-cash-flow milestones rather than annual bookings, and every vice president now holds equity vesting over four years with a one-year cliff, which we believe aligns the team with long-term owners.",
"location": "slide 7"
}
],
"rationale": "Category D: specific, quantified disclosure with owner-aligned framing."
},
{
"id": "E",
"score": 4,
"evidence": [
{
"quote": "Our compensation plan ties 60 percent of executive bonus to three-year ARR retention and free-cash-flow milestones rather than annual bookings, and every vice president now holds equity vesting over four years with a one-year cliff, which we believe aligns the team with long-term owners.",
"location": "slide 7"
}
],
"rationale": "Category E: specific, quantified disclosure with owner-aligned framing."
},
{
"id": "F",
"score": 4,
"evidence": [
{
"quote": "Our compensation plan ties 60 percent of executive bonus to three-year ARR retention and free-cash-flow milestones rather than annual bookings, and every vice president now holds equity vesting over four years with a one-year cliff, which we believe aligns the team with long-term owners.",
"location": "slide 7"
}
],
"rationale": "Category F: specific, quantified disclosure with owner-aligned framing."
},
{
"id": "G",
"score": 4,
"evidence": [
{
"quote": "Our compensation plan ties 60 percent of executive bonus to three-year ARR retention and free-cash-flow milestones rather than annual bookings, and every vice president now holds equity vesting over four years with a one-year cliff, which we believe aligns the team with long-term owners.",
"location": "slide 7"
}
],
"rationale": "Category G: specific, quantified disclosure with owner-aligned framing."
},
{
"id": "H",
"score": 3,
"evidence": [],
"rationale": "Asks are listed without recommendations or the inversion of the decision."
}
],
"red_flags": [
{
"code": "governance_gap",
"description": "board asks lack recommendations and inversion",
"severity": 3,
"evidence": "slide 11"
}
],
"overall_comment": "Strong disclosure discipline overall; governance asks remain the weak spot."
}
orchestrator/tests/fixtures/grade_c.json
Vendored
+96
View File
@@ -0,0 +1,96 @@
{
"schema_version": 1,
"grader": "grader-c",
"categories": [
{
"id": "A",
"score": 4,
"evidence": [
{
"quote": "Our compensation plan ties 60 percent of executive bonus to three-year ARR retention and free-cash-flow milestones rather than annual bookings, and every vice president now holds equity vesting over four years with a one-year cliff, which we believe aligns the team with long-term owners.",
"location": "slide 7"
}
],
"rationale": "Category A: specific, quantified disclosure with owner-aligned framing."
},
{
"id": "B",
"score": 4,
"evidence": [
{
"quote": "Our compensation plan ties 60 percent of executive bonus to three-year ARR retention and free-cash-flow milestones rather than annual bookings, and every vice president now holds equity vesting over four years with a one-year cliff, which we believe aligns the team with long-term owners.",
"location": "slide 7"
}
],
"rationale": "Category B: specific, quantified disclosure with owner-aligned framing."
},
{
"id": "C",
"score": 4,
"evidence": [
{
"quote": "Our compensation plan ties 60 percent of executive bonus to three-year ARR retention and free-cash-flow milestones rather than annual bookings, and every vice president now holds equity vesting over four years with a one-year cliff, which we believe aligns the team with long-term owners.",
"location": "slide 7"
}
],
"rationale": "Category C: specific, quantified disclosure with owner-aligned framing."
},
{
"id": "D",
"score": 4,
"evidence": [
{
"quote": "Our compensation plan ties 60 percent of executive bonus to three-year ARR retention and free-cash-flow milestones rather than annual bookings, and every vice president now holds equity vesting over four years with a one-year cliff, which we believe aligns the team with long-term owners.",
"location": "slide 7"
}
],
"rationale": "Category D: specific, quantified disclosure with owner-aligned framing."
},
{
"id": "E",
"score": 4,
"evidence": [
{
"quote": "Our compensation plan ties 60 percent of executive bonus to three-year ARR retention and free-cash-flow milestones rather than annual bookings, and every vice president now holds equity vesting over four years with a one-year cliff, which we believe aligns the team with long-term owners.",
"location": "slide 7"
}
],
"rationale": "Category E: specific, quantified disclosure with owner-aligned framing."
},
{
"id": "F",
"score": 4,
"evidence": [
{
"quote": "Our compensation plan ties 60 percent of executive bonus to three-year ARR retention and free-cash-flow milestones rather than annual bookings, and every vice president now holds equity vesting over four years with a one-year cliff, which we believe aligns the team with long-term owners.",
"location": "slide 7"
}
],
"rationale": "Category F: specific, quantified disclosure with owner-aligned framing."
},
{
"id": "G",
"score": 4,
"evidence": [
{
"quote": "Our compensation plan ties 60 percent of executive bonus to three-year ARR retention and free-cash-flow milestones rather than annual bookings, and every vice president now holds equity vesting over four years with a one-year cliff, which we believe aligns the team with long-term owners.",
"location": "slide 7"
}
],
"rationale": "Category G: specific, quantified disclosure with owner-aligned framing."
},
{
"id": "H",
"score": 2,
"evidence": [
{
"quote": "We ask the board to approve the revised hiring plan as presented; supporting detail is available from management upon request after the meeting, and we recommend approval without further discussion given the compressed agenda for this session.",
"location": "slide 11"
}
],
"rationale": "Asks are listed without recommendations or the inversion of the decision."
}
],
"red_flags": [],
"overall_comment": "Strong disclosure discipline overall; governance asks remain the weak spot."
}
orchestrator/tests/test_decks.py
+118
View File
@@ -0,0 +1,118 @@
"""Tests for decks.py: slugify, period parsing/sorting, inbox discovery."""
import os
import sys
import tempfile
import unittest
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import decks
class TestSlugify(unittest.TestCase):
def test_basic(self):
self.assertEqual(decks.slugify("Acme Robotics"), "acme-robotics")
self.assertEqual(decks.slugify(" Acme, Inc. (US) "), "acme-inc-us")
self.assertEqual(decks.slugify("ALLCAPS"), "allcaps")
self.assertEqual(decks.slugify(""), "company")
self.assertEqual(decks.slugify("---"), "company")
class TestParsePeriod(unittest.TestCase):
def test_quarters(self):
self.assertEqual(decks.parse_period_from_name("acme_2026-Q2_board.pdf"), "2026-Q2")
self.assertEqual(decks.parse_period_from_name("2026Q4 deck.pptx"), "2026-Q4")
self.assertEqual(decks.parse_period_from_name("Q3 2025 update.pptx"), "2025-Q3")
self.assertEqual(decks.parse_period_from_name("q1_2024_board.docx"), "2024-Q1")
def test_halves(self):
self.assertEqual(decks.parse_period_from_name("board-2026-H1.docx"), "2026-H1")
self.assertEqual(decks.parse_period_from_name("2025H2-review.pdf"), "2025-H2")
def test_months(self):
self.assertEqual(decks.parse_period_from_name("acme 2026-05 board.pdf"), "2026-05")
self.assertEqual(decks.parse_period_from_name("2026_12_flash.txt"), "2026-12")
self.assertIsNone(decks.parse_period_from_name("2026-13 notes.pdf"))
self.assertIsNone(decks.parse_period_from_name("2026-00 notes.pdf"))
def test_fiscal_year(self):
self.assertEqual(decks.parse_period_from_name("FY2025 review.pdf"), "FY2025")
self.assertEqual(decks.parse_period_from_name("fy-2024 plan.txt"), "FY2024")
self.assertEqual(decks.parse_period_from_name("FY 2026 budget.docx"), "FY2026")
def test_no_period(self):
self.assertIsNone(decks.parse_period_from_name("notes.txt"))
self.assertIsNone(decks.parse_period_from_name("budget_2027.xlsx"))
self.assertIsNone(decks.parse_period_from_name("Q5 2026.pdf"))
def test_quarter_wins_over_month(self):
# "2026-Q2" must not be misread; Q pattern is checked before YYYY-MM.
self.assertEqual(decks.parse_period_from_name("2026-Q2 and 2026-05.pdf"), "2026-Q2")
def test_not_inside_digit_runs(self):
self.assertIsNone(decks.parse_period_from_name("doc-20261-05.pdf"))
class TestPeriodSortKey(unittest.TestCase):
def test_ordering_mixed_granularities(self):
ordered = ["FY2025", "2025-Q4", "2026-H1", "2026-Q1", "2026-01",
"2026-Q2", "2026-05", "2026-H2", "2026-Q4"]
self.assertEqual(sorted(ordered, key=decks.period_sort_key), ordered)
def test_start_months(self):
self.assertEqual(decks.period_sort_key("2026-Q2")[:2], (2026, 4))
self.assertEqual(decks.period_sort_key("2026-H2")[:2], (2026, 7))
self.assertEqual(decks.period_sort_key("2026-11")[:2], (2026, 11))
self.assertEqual(decks.period_sort_key("FY2026")[:2], (2026, 1))
def test_unknown_sorts_last(self):
keys = [decks.period_sort_key(p) for p in ("2026-Q4", None, "garbage", "FY2026")]
self.assertEqual(max(keys), decks.period_sort_key(None))
self.assertEqual(decks.period_sort_key(None), decks.period_sort_key("garbage"))
self.assertGreater(decks.period_sort_key(None), decks.period_sort_key("2099-Q4"))
class TestDiscover(unittest.TestCase):
def _touch(self, *parts):
path = os.path.join(*parts)
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "w") as f:
f.write("x")
def test_discover(self):
with tempfile.TemporaryDirectory() as tmp:
inbox = os.path.join(tmp, "inbox")
self._touch(inbox, "Acme Robotics", "acme-2026-Q1.pdf")
self._touch(inbox, "Acme Robotics", "acme-2026-Q1-appendix.txt")
self._touch(inbox, "Acme Robotics", "acme-2026-Q2.pptx")
self._touch(inbox, "Acme Robotics", "chart-2026-Q2.png")
self._touch(inbox, "Acme Robotics", "notes.txt")
self._touch(inbox, "Acme Robotics", ".DS_Store")
self._touch(inbox, "beta-corp", "deck 2026-H1.docx")
self._touch(inbox, "stray.pdf")
out = decks.discover(inbox)
self.assertEqual(out["skipped"], ["stray.pdf"])
units = out["units"]
keys = [(u["company_slug"], u["period"], u["period_source"]) for u in units]
self.assertEqual(keys, [
("acme-robotics", "2026-Q1", "filename"),
("acme-robotics", "2026-Q2", "filename"),
("acme-robotics", None, "unknown"),
("beta-corp", "2026-H1", "filename"),
])
q1 = units[0]
self.assertEqual([os.path.basename(f) for f in q1["files"]],
["acme-2026-Q1-appendix.txt", "acme-2026-Q1.pdf"])
self.assertTrue(all(os.path.isabs(f) for f in q1["files"]))
q2 = units[1]
self.assertEqual([os.path.basename(f) for f in q2["files"]], ["acme-2026-Q2.pptx"])
self.assertEqual(q2["ignored"], ["chart-2026-Q2.png"])
self.assertEqual([os.path.basename(f) for f in units[2]["files"]], ["notes.txt"])
def test_missing_inbox(self):
self.assertEqual(decks.discover("/nonexistent/inbox"), {"units": [], "skipped": []})
if __name__ == "__main__":
unittest.main()
orchestrator/tests/test_ledger.py
+116
View File
@@ -0,0 +1,116 @@
"""Tests for ledger.py: company lifecycle, deck records, forward targets."""
import glob
import os
import sys
import tempfile
import unittest
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import ledger as ledger_mod
def _record(deck_id, period, composite=70.0):
return {"schema_version": 1, "deck_id": deck_id, "period": period,
"composite": composite, "graded_at": "2026-07-01T00:00:00Z"}
def _ft(canonical, target, target_period, direction="gte"):
return {"name": canonical, "canonical_name": canonical, "target": target,
"unit": "", "target_period": target_period, "direction": direction,
"profitability": False, "source": "slide 9"}
class TestLedger(unittest.TestCase):
def setUp(self):
self._tmp = tempfile.TemporaryDirectory()
self.ledger = ledger_mod.Ledger(os.path.join(self._tmp.name, "ledger"))
def tearDown(self):
self._tmp.cleanup()
def test_ensure_company_auto_created(self):
c = self.ledger.ensure_company("acme")
self.assertTrue(c["auto_created"])
self.assertEqual(c["name"], "acme")
c2 = self.ledger.ensure_company("acme", name="Acme Robotics")
self.assertEqual(c2["name"], "acme") # existing entry wins
named = self.ledger.ensure_company("beta", name="Beta Corp")
self.assertFalse(named["auto_created"])
self.assertEqual(named["name"], "Beta Corp")
self.assertEqual(self.ledger.all_slugs(), ["acme", "beta"])
self.assertEqual(len(self.ledger.all_companies()), 2)
def test_merge_config_companies(self):
self.ledger.ensure_company("acme")
self.ledger.merge_config_companies([{
"slug": "acme", "name": "Acme Robotics",
"kpiAliases": "arr=annual recurring revenue;run_rate_arr\nchurn_rate=logo_churn",
"pinnedTargets": [{"kpi": "cash_balance", "target": 12.0, "unit": "$M",
"direction": "gte", "profitability": True}],
}])
c = self.ledger.get_company("acme")
self.assertFalse(c["auto_created"])
self.assertEqual(c["name"], "Acme Robotics")
self.assertEqual(c["kpi_aliases"],
{"arr": ["annual recurring revenue", "run_rate_arr"],
"churn_rate": ["logo_churn"]})
self.assertEqual(c["pinned_targets"][0]["kpi"], "cash_balance")
# slug derived from name when absent
self.ledger.merge_config_companies([{"name": "Beta Corp", "kpiAliases": "",
"pinnedTargets": []}])
self.assertIsNotNone(self.ledger.get_company("beta-corp"))
def test_record_supersede_prior_targets_roundtrip(self):
path = self.ledger.record_deck("acme", _record("2026-Q1", "2026-Q1", 82.7),
[_ft("arr", 12.0, "2026-Q2"),
_ft("arr", 15.0, "2026-Q3")])
self.assertTrue(os.path.isfile(path))
self.assertEqual(
[t["target"] for t in self.ledger.prior_targets("acme", "2026-Q2")], [12.0])
self.assertEqual(self.ledger.prior_targets("acme", "2026-Q4"), [])
self.assertEqual(self.ledger.prior_targets("nobody", "2026-Q2"), [])
# Re-grade the same deck: old record superseded (renamed), one live record.
self.ledger.record_deck("acme", _record("2026-Q1", "2026-Q1", 80.0),
[_ft("arr", 12.5, "2026-Q2")])
ddir = os.path.dirname(path)
self.assertEqual(len(glob.glob(os.path.join(ddir, "*.superseded-*.json"))), 1)
live = self.ledger.deck_records("acme")
self.assertEqual(len(live), 1)
self.assertEqual(live[0]["composite"], 80.0)
self.assertEqual(
[t["target"] for t in self.ledger.prior_targets("acme", "2026-Q2")], [12.5])
# history keeps one entry per period
c = self.ledger.get_company("acme")
self.assertEqual([h["period"] for h in c["history"]], ["2026-Q1"])
self.assertEqual(c["history"][0]["composite"], 80.0)
def test_newer_deck_replaces_targets_older_does_not(self):
self.ledger.record_deck("acme", _record("2026-Q1", "2026-Q1"),
[_ft("arr", 15.0, "2026-Q3")])
self.ledger.record_deck("acme", _record("2026-Q2", "2026-Q2"),
[_ft("arr", 16.0, "2026-Q3"),
_ft("churn_rate", 3.0, "2026-Q3", "lte")])
targets = self.ledger.prior_targets("acme", "2026-Q3")
self.assertEqual(sorted(t["target"] for t in targets), [3.0, 16.0])
c = self.ledger.get_company("acme")
self.assertEqual(c["extracted_targets"]["2026-Q3"]["from_deck"], "2026-Q2")
# Re-recording the OLDER deck must not clobber the newer deck's targets.
self.ledger.record_deck("acme", _record("2026-Q1", "2026-Q1"),
[_ft("arr", 15.0, "2026-Q3")])
targets = self.ledger.prior_targets("acme", "2026-Q3")
self.assertEqual(sorted(t["target"] for t in targets), [3.0, 16.0])
# History is sorted oldest first.
c = self.ledger.get_company("acme")
self.assertEqual([h["period"] for h in c["history"]], ["2026-Q1", "2026-Q2"])
def test_deck_record_lookup(self):
self.ledger.record_deck("acme", _record("2026-Q1", "2026-Q1"), [])
self.assertEqual(self.ledger.deck_record("acme", "2026-Q1")["period"], "2026-Q1")
self.assertIsNone(self.ledger.deck_record("acme", "2026-Q9"))
self.assertEqual(self.ledger.deck_records("nobody"), [])
if __name__ == "__main__":
unittest.main()
orchestrator/tests/test_scoring.py
+423
View File
@@ -0,0 +1,423 @@
"""Tests for scoring.py (pure scorer), validate.py, and the fixture-driven
Q1 -> Q2 end-to-end flow through the ledger and scorecard renderers."""
import json
import os
import sys
import tempfile
import unittest
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import ledger as ledger_mod
import scorecard
import scoring
import validate
FIXTURES = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures")
WEIGHTS = {
"profitabilityKpi": 30, "otherKpi": 20, "forecastIntegrity": 10,
"qualCategoryMax": 5, "redFlagCap": 15, "kpiCreditFloor": 0.5,
"droppedKpiPenalty": 2, "droppedKpiMax": 3, "evidenceFullCredit": 400,
"singleSourceFlagFactor": 0.5,
}
PINNED_CASH = [{"kpi": "cash_balance", "target": 12.0, "unit": "$M",
"direction": "gte", "profitability": True}]
def _fixture(name):
with open(os.path.join(FIXTURES, name), encoding="utf-8") as f:
return json.load(f)
def _kpi(canonical, actual, direction="gte", prof=False, tid=None, name=None, unit=""):
return {"name": name or canonical, "canonical_name": canonical, "actual": actual,
"unit": unit, "period": None, "direction": direction, "profitability": prof,
"target_in_deck": tid, "source": "slide 1", "notes": ""}
def _ft(canonical, target, direction="gte", target_period="2026-Q2", prof=False):
return {"name": canonical, "canonical_name": canonical, "target": target,
"unit": "", "target_period": target_period, "direction": direction,
"profitability": prof, "source": "slide 9"}
def _extraction(kpis=None, forward=None, flags=None, period="2026-Q2"):
return {"schema_version": 1, "deck": {"period": period},
"kpis": kpis or [], "forward_targets": forward or [],
"red_flag_candidates": flags or [],
"narrative": {"summary": "test deck", "asks": []}}
def _grade(grader="grader-a", score=3, quote_chars=0, red_flags=None, overrides=None):
cats = []
for cid in "ABCDEFGH":
s, qc = score, quote_chars
if overrides and cid in overrides:
s, qc = overrides[cid]
ev = [{"quote": "q" * qc, "location": "slide 1"}] if qc else []
cats.append({"id": cid, "score": s, "evidence": ev, "rationale": f"cat {cid}"})
return {"schema_version": 1, "grader": grader, "categories": cats,
"red_flags": red_flags or [], "overall_comment": "ok"}
def _meta(period="2026-Q2", deck_id="d1"):
return {"company": "acme", "period": period, "deck_id": deck_id, "job_id": "job-1",
"graded_at": "2026-07-06T12:00:00Z",
"panel": [{"rid": "grader-a", "model": "grader-a", "valid": True}],
"artifacts": {"extraction": "extraction.json"}}
def _score(extraction, grades=None, pinned=None, prior=None, aliases=None, meta=None):
return scoring.score_deck(extraction, grades if grades is not None else [_grade()],
pinned or [], prior or [], aliases or {}, WEIGHTS,
meta or _meta())
def _flag(rec, code):
return [f for f in rec["penalties"]["flags"] if f["code"] == code]
class TestMatchKpi(unittest.TestCase):
def test_exact(self):
cand, via = scoring.match_kpi("arr", [{"canonical_name": "arr", "name": "ARR"}], {})
self.assertEqual(via, "exact")
self.assertEqual(cand["name"], "ARR")
def test_alias_forward_and_reverse(self):
aliases = {"arr": ["Annual Recurring Revenue", "run_rate_arr"]}
cand, via = scoring.match_kpi(
"arr", [{"canonical_name": "revenue_annualized",
"name": "Annual Recurring Revenue"}], aliases)
self.assertEqual(via, "alias")
cand, via = scoring.match_kpi(
"run_rate_arr", [{"canonical_name": "arr", "name": "ARR"}], aliases)
self.assertEqual(via, "alias")
def test_fuzzy(self):
cand, via = scoring.match_kpi(
"ebitda_margin", [{"canonical_name": "ebitda_margins", "name": "x"}], {})
self.assertEqual(via, "fuzzy")
def test_no_match(self):
self.assertEqual(
scoring.match_kpi("arr", [{"canonical_name": "cash_balance", "name": "Cash"}], {}),
(None, None))
self.assertEqual(scoring.match_kpi("", [{"canonical_name": "arr"}], {}), (None, None))
class TestCredit(unittest.TestCase):
def test_lte_credit(self):
rec = _score(_extraction([_kpi("churn_rate", 6.0, "lte", tid=5.0)]))
self.assertAlmostEqual(rec["kpi_results"][0]["credit"], 0.6667, places=4)
rec = _score(_extraction([_kpi("churn_rate", 4.0, "lte", tid=5.0)]))
self.assertEqual(rec["kpi_results"][0]["credit"], 1.0)
def test_floor(self):
rec = _score(_extraction([_kpi("arr", 4.0, tid=10.0)])) # r=0.4 < floor
self.assertEqual(rec["kpi_results"][0]["credit"], 0.0)
rec = _score(_extraction([_kpi("arr", 7.5, tid=10.0)])) # r=0.75 -> 0.5
self.assertAlmostEqual(rec["kpi_results"][0]["credit"], 0.5, places=4)
def test_guards(self):
self.assertEqual(scoring._credit(5, 0, "gte", 0.5), 1.0) # zero target, passes
self.assertEqual(scoring._credit(-5, 0, "gte", 0.5), 0.0) # zero target, fails
self.assertEqual(scoring._credit(-1, 1, "gte", 0.5), 0.0) # sign mismatch, fails
self.assertEqual(scoring._credit(1, -1, "gte", 0.5), 1.0) # sign mismatch, passes
self.assertEqual(scoring._credit(0, 5, "lte", 0.5), 1.0) # lte zero actual
def test_negative_targets(self):
# EBITDA margin: target -2, actual -3 -> two thirds of the way -> 0.3333
self.assertAlmostEqual(scoring._credit(-3, -2, "gte", 0.5), 1 / 3, places=4)
self.assertEqual(scoring._credit(-1, -2, "gte", 0.5), 1.0)
class TestQuantBuckets(unittest.TestCase):
def test_first_deck_renormalization(self):
# No prior targets -> forecast NA -> its 10 points redistribute 36/24.
rec = _score(_extraction([_kpi("ebitda_margin", 5.0, prof=True, tid=5.0),
_kpi("arr", 10.0, tid=10.0)]))
q = rec["quant"]
self.assertTrue(q["forecast_integrity"]["na"])
self.assertAlmostEqual(q["profitability"]["weight"], 36.0)
self.assertAlmostEqual(q["profitability"]["score"], 36.0)
self.assertAlmostEqual(q["other"]["weight"], 24.0)
self.assertAlmostEqual(q["other"]["score"], 24.0)
self.assertAlmostEqual(q["score"], 60.0)
self.assertEqual(rec["penalties"]["flags"], [])
self.assertAlmostEqual(rec["composite"], 84.0) # 60 quant + 24 qual (all 3s)
def test_forecast_integrity_second_deck(self):
prior = [_ft("arr", 12.0), _ft("churn_rate", 4.0, "lte")]
rec = _score(_extraction([_kpi("arr", 11.0), _kpi("churn_rate", 3.5, "lte"),
_kpi("fcf", 1.0, prof=True, tid=1.0)]),
prior=prior)
fi = rec["quant"]["forecast_integrity"]
self.assertFalse(fi["na"])
self.assertEqual(fi["weight"], 10.0)
self.assertEqual(fi["kpi_count"], 2)
accs = {f["canonical_name"]: f["accuracy"] for f in rec["forecast_results"]}
self.assertAlmostEqual(accs["arr"], 0.9167, places=4) # 1/12 undershoot
self.assertAlmostEqual(accs["churn_rate"], 0.9375, places=4) # overshoot halved
self.assertAlmostEqual(fi["score"], (0.9167 + 0.9375) / 2 * 10, places=3)
def test_no_profitability_flag_and_redistribution(self):
rec = _score(_extraction([_kpi("arr", 10.0, tid=10.0)]))
q = rec["quant"]
self.assertTrue(q["profitability"]["na"])
self.assertTrue(q["forecast_integrity"]["na"])
self.assertAlmostEqual(q["other"]["weight"], 60.0)
self.assertAlmostEqual(q["score"], 60.0)
flags = _flag(rec, "no_profitability_visibility")
self.assertEqual(len(flags), 1)
self.assertEqual(flags[0]["points"], 3.0) # scoring flags never damped
self.assertEqual(flags[0]["sources"], ["scoring"])
def test_profitability_kpis_without_targets_na_no_flag(self):
rec = _score(_extraction([_kpi("ebitda_margin", -5.0, prof=True),
_kpi("arr", 10.0, tid=10.0)]))
self.assertTrue(rec["quant"]["profitability"]["na"])
self.assertEqual(_flag(rec, "no_profitability_visibility"), [])
def test_all_quant_na_scales_qual(self):
rec = _score(_extraction([]))
# qual 24 (all 3s) scaled to 60, minus no_profitability(3) + no_quantitative(4)
self.assertTrue(all(rec["quant"][b]["na"] for b in
("profitability", "other", "forecast_integrity")))
self.assertEqual(len(_flag(rec, "no_quantitative_kpis")), 1)
self.assertAlmostEqual(rec["composite"], 53.0)
class TestTargetPrecedence(unittest.TestCase):
def test_pinned_beats_extracted_beats_in_deck(self):
kpis = [_kpi("arr", 11.0, tid=9.0)]
pinned = [{"kpi": "arr", "target": 10.0, "unit": "$M",
"direction": "gte", "profitability": False}]
prior = [_ft("arr", 12.0)]
r = _score(_extraction(kpis), pinned=pinned, prior=prior)["kpi_results"][0]
self.assertEqual((r["target"], r["target_source"], r["matched_via"]),
(10.0, "pinned", "exact"))
self.assertEqual(r["credit"], 1.0)
r = _score(_extraction(kpis), prior=prior)["kpi_results"][0]
self.assertEqual((r["target"], r["target_source"]), (12.0, "extracted"))
self.assertAlmostEqual(r["credit"], 0.8333, places=4)
r = _score(_extraction(kpis))["kpi_results"][0]
self.assertEqual((r["target"], r["target_source"], r["matched_via"]),
(9.0, "in_deck", None))
def test_untargeted_kpi_reported_with_none(self):
r = _score(_extraction([_kpi("nps", 40.0)]))["kpi_results"][0]
self.assertIsNone(r["target"])
self.assertIsNone(r["credit"])
self.assertIsNone(r["target_source"])
class TestQualitative(unittest.TestCase):
def test_evidence_regression_both_directions(self):
# Median 5 with no quotes regresses to 3; so does median 1.
rec = _score(_extraction([]), grades=[_grade(score=5, quote_chars=0)])
self.assertEqual(rec["qual"]["categories"]["A"]["adjusted"], 3.0)
rec = _score(_extraction([]), grades=[_grade(score=1, quote_chars=0)])
self.assertEqual(rec["qual"]["categories"]["A"]["adjusted"], 3.0)
self.assertAlmostEqual(rec["qual"]["score"], 24.0)
def test_full_evidence_keeps_extreme_scores(self):
# Per-quote chars cap at 200, so full credit (400) needs two quotes.
g = _grade(score=5, quote_chars=200)
for cat in g["categories"]:
cat["evidence"].append({"quote": "q" * 200, "location": "slide 2"})
rec = _score(_extraction([]), grades=[g])
cat = rec["qual"]["categories"]["A"]
self.assertEqual(cat["evidence_quality"], 1.0)
self.assertEqual(cat["adjusted"], 5.0)
self.assertEqual(cat["points"], 5.0)
def test_quote_chars_capped_at_200_each(self):
# One 1000-char quote counts as 200 -> e = 0.5 -> adjusted 4.
rec = _score(_extraction([]), grades=[_grade(score=5, quote_chars=1000)])
self.assertEqual(rec["qual"]["categories"]["A"]["evidence_quality"], 0.5)
self.assertEqual(rec["qual"]["categories"]["A"]["adjusted"], 4.0)
def test_panel_median_and_rationales(self):
grades = [_grade("g1", score=4, quote_chars=400),
_grade("g2", score=4, quote_chars=400),
_grade("g3", score=2, quote_chars=400)]
rec = _score(_extraction([]), grades=grades)
cat = rec["qual"]["categories"]["B"]
self.assertEqual(cat["panel_scores"], [4, 4, 2])
self.assertEqual(cat["median"], 4.0)
self.assertEqual(len(cat["rationales"]), 3)
self.assertEqual(cat["rationales"][0]["grader"], "g1")
class TestPenalties(unittest.TestCase):
def test_single_source_damping(self):
rec = _score(_extraction([], flags=[{"code": "adjusted_metrics",
"description": "d", "severity": 4}]))
f = _flag(rec, "adjusted_metrics")[0]
self.assertEqual(f["points"], 2.0)
self.assertEqual(f["sources"], ["extractor"])
def test_two_sources_full_severity_max_wins(self):
grades = [_grade("g1", red_flags=[{"code": "governance_gap",
"description": "weak", "severity": 2}]),
_grade("g2", red_flags=[{"code": "governance_gap",
"description": "worse", "severity": 3}])]
rec = _score(_extraction([]), grades=grades)
f = _flag(rec, "governance_gap")[0]
self.assertEqual(f["severity"], 3)
self.assertEqual(f["points"], 3.0)
self.assertEqual(f["sources"], ["g1", "g2"])
def test_penalty_cap(self):
codes = ["related_party", "channel_stuffing_risk", "suppressed_dissent",
"metric_redefinition"]
flags = [{"code": c, "description": c, "severity": 5} for c in codes]
rec = _score(_extraction([_kpi("fcf", 1.0, prof=True, tid=1.0)], flags=flags),
grades=[_grade("g1", red_flags=flags)])
self.assertEqual(rec["penalties"]["total"], 15.0) # 4x5=20 capped
def test_dropped_kpi_flags_capped(self):
prior = [_ft(c, 1.0) for c in ("alpha_metric", "beta_metric", "gamma_metric",
"delta_metric", "epsilon_metric")]
rec = _score(_extraction([]), prior=prior)
dropped = _flag(rec, "kpi_dropped")
self.assertEqual(len(dropped), 3) # droppedKpiMax
for f in dropped:
self.assertEqual(f["points"], 2.0) # droppedKpiPenalty, never damped
class TestValidate(unittest.TestCase):
def test_parse_json_text(self):
self.assertEqual(validate.parse_json_text('{"a": 1}'), {"a": 1})
salvaged = validate.parse_json_text(
'Sure! Here is the JSON:\n```json\n{"a": {"b": "}"}}\n```\ntrailing prose')
self.assertEqual(salvaged, {"a": {"b": "}"}})
self.assertIsNone(validate.parse_json_text("no json here"))
self.assertIsNone(validate.parse_json_text("[1, 2, 3]"))
self.assertIsNone(validate.parse_json_text(""))
def test_schemas_load_and_fixtures_validate(self):
self.assertIn("properties", validate.load_schema("extraction"))
self.assertIn("properties", validate.load_schema("grades"))
for name, schema in (("extraction_q1.json", "extraction"),
("extraction_q2.json", "extraction"),
("grade_a.json", "grades"), ("grade_b.json", "grades"),
("grade_c.json", "grades")):
err = validate.validate_obj(_fixture(name), schema)
self.assertIsNone(err, f"{name}: {err}")
def test_validate_obj_rejects_bad(self):
self.assertIsNotNone(validate.validate_obj({"schema_version": 1}, "grades"))
def test_validate_file(self):
obj, err = validate.validate_file(os.path.join(FIXTURES, "grade_a.json"), "grades")
self.assertIsNone(err)
self.assertEqual(obj["grader"], "grader-a")
obj, err = validate.validate_file("/nonexistent.json", "grades")
self.assertIsNone(obj)
self.assertIsNotNone(err)
class TestEndToEnd(unittest.TestCase):
"""Fixture-driven Q1 -> Q2 flow: score, ledger round-trip, rendering."""
def setUp(self):
self._tmp = tempfile.TemporaryDirectory()
self.ledger = ledger_mod.Ledger(os.path.join(self._tmp.name, "ledger"))
self.grades = [_fixture("grade_a.json"), _fixture("grade_b.json"),
_fixture("grade_c.json")]
self.q1 = _fixture("extraction_q1.json")
self.q2 = _fixture("extraction_q2.json")
def tearDown(self):
self._tmp.cleanup()
def _score_q1(self):
return scoring.score_deck(self.q1, self.grades, PINNED_CASH, [], {}, WEIGHTS,
_meta("2026-Q1", "2026-Q1"))
def test_q1_first_deck(self):
rec = self._score_q1()
q = rec["quant"]
self.assertTrue(q["forecast_integrity"]["na"])
self.assertAlmostEqual(q["score"], 60.0) # every KPI at/above target
# qual: A-G 3.5 pts each (median 4, evidence 0.5), H 2.6667
self.assertAlmostEqual(rec["qual"]["score"], 27.1667, places=3)
self.assertAlmostEqual(rec["qual"]["categories"]["H"]["points"], 2.6667, places=3)
# hockey_stick (extractor only, sev 3 -> 1.5) + governance_gap (2 graders -> 3)
self.assertAlmostEqual(rec["penalties"]["total"], 4.5)
self.assertAlmostEqual(rec["composite"], 82.7)
cash = next(k for k in rec["kpi_results"] if k["canonical_name"] == "cash_balance")
self.assertEqual(cash["target_source"], "pinned")
def test_q2_against_q1_targets(self):
rec1 = self._score_q1()
self.ledger.record_deck("acme", rec1, self.q1["forward_targets"])
prior = self.ledger.prior_targets("acme", "2026-Q2")
self.assertEqual(len(prior), 4)
rec2 = scoring.score_deck(self.q2, self.grades, PINNED_CASH, prior, {}, WEIGHTS,
_meta("2026-Q2", "2026-Q2"))
by_name = {k["canonical_name"]: k for k in rec2["kpi_results"]}
self.assertAlmostEqual(by_name["arr"]["credit"], 0.8333, places=4)
self.assertEqual(by_name["churn_rate"]["credit"], 1.0)
self.assertAlmostEqual(by_name["ebitda_margin"]["credit"], 0.3333, places=4)
self.assertEqual(by_name["cash_balance"]["target_source"], "pinned")
self.assertEqual(by_name["cash_balance"]["credit"], 1.0)
q = rec2["quant"]
self.assertAlmostEqual(q["profitability"]["score"], 20.0, places=2)
self.assertAlmostEqual(q["other"]["score"], 18.333, places=2)
self.assertAlmostEqual(q["forecast_integrity"]["score"], 7.847, places=2)
self.assertEqual(len(rec2["forecast_results"]), 3)
# qualified_pipeline guided in Q1 but not reported in Q2 -> dropped flag
dropped = _flag(rec2, "kpi_dropped")
self.assertEqual(len(dropped), 1)
self.assertIn("qualified_pipeline", dropped[0]["description"])
# adjusted_metrics 1.0 + governance_gap 3.0 + kpi_dropped 2.0
self.assertAlmostEqual(rec2["penalties"]["total"], 6.0)
self.assertAlmostEqual(rec2["composite"], 67.3)
self.assertAlmostEqual(
rec2["composite"],
round(q["score"] + rec2["qual"]["score"] - rec2["penalties"]["total"], 1))
# ledger round-trip + rendering
self.ledger.record_deck("acme", rec2, self.q2["forward_targets"])
records = self.ledger.deck_records("acme")
self.assertEqual([r["period"] for r in records], ["2026-Q1", "2026-Q2"])
report = scorecard.render_deck_report(rec2, self.q2, adjudication_md="Chair memo.")
self.assertIn("67.3", report)
self.assertIn("pinned", report)
self.assertIn("## Panel adjudication", report)
self.assertIn("Chair memo.", report)
self.assertIn("kpi_dropped", report)
card = scorecard.render_scorecard(self.ledger.get_company("acme"), records)
self.assertIn("2026-Q1", card)
self.assertIn("2026-Q2", card)
self.assertIn("↓", card) # composite fell Q1 -> Q2
self.assertIn("KPI hit-rate", card)
self.assertIn("arr", card)
def test_meta_passthrough_and_record_shape(self):
rec = self._score_q1()
self.assertEqual(rec["company"], "acme")
self.assertEqual(rec["deck_id"], "2026-Q1")
self.assertEqual(rec["job_id"], "job-1")
self.assertEqual(rec["panel"][0]["rid"], "grader-a")
self.assertEqual(rec["artifacts"], {"extraction": "extraction.json"})
self.assertEqual(rec["schema_version"], 1)
self.assertIn("summary", rec["narrative"])
for key in ("composite", "quant", "qual", "penalties", "kpi_results",
"forecast_results"):
self.assertIn(key, rec)
# the record must be JSON-serializable as produced
json.dumps(rec)
if __name__ == "__main__":
unittest.main()
orchestrator/validate.py
+89
View File
@@ -0,0 +1,89 @@
"""JSON parsing + schema validation for extractor/grader outputs.
Local models occasionally wrap their JSON in prose or fences; parse_json_text
salvages the first brace-balanced top-level object before we give up. Schemas
live in orchestrator/schemas/ and are the single contract between the sandbox
agents and the deterministic scorer.
"""
from __future__ import annotations
import json
import os
SCHEMAS_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "schemas")
_cache: dict[str, dict] = {}
def load_schema(name: str) -> dict:
"""Load "extraction" or "grades" schema (cached)."""
if name not in _cache:
path = os.path.join(SCHEMAS_DIR, f"{name}.schema.json")
with open(path, encoding="utf-8") as f:
_cache[name] = json.load(f)
return _cache[name]
def parse_json_text(text: str) -> dict | None:
"""Parse `text` as a JSON object; salvage the first balanced {...} block."""
if not text:
return None
try:
obj = json.loads(text)
return obj if isinstance(obj, dict) else None
except Exception:
pass
start = text.find("{")
while start != -1:
depth = 0
in_str = False
esc = False
for i in range(start, len(text)):
c = text[i]
if esc:
esc = False
elif in_str:
if c == "\\":
esc = True
elif c == '"':
in_str = False
elif c == '"':
in_str = True
elif c == "{":
depth += 1
elif c == "}":
depth -= 1
if depth == 0:
try:
obj = json.loads(text[start:i + 1])
if isinstance(obj, dict):
return obj
except Exception:
pass
break
start = text.find("{", start + 1)
return None
def validate_obj(obj, schema_name: str) -> str | None:
"""Validate against the named schema; error message or None if valid."""
import jsonschema
try:
jsonschema.validate(obj, load_schema(schema_name))
return None
except jsonschema.ValidationError as e:
path = ".".join(str(p) for p in e.absolute_path) or "(root)"
return f"{path}: {e.message}"[:500]
def validate_file(path: str, schema_name: str) -> tuple[dict | None, str | None]:
"""Read + parse (with salvage) + validate a JSON file -> (obj, error)."""
try:
with open(path, encoding="utf-8", errors="replace") as f:
text = f.read()
except Exception as e:
return None, f"read failed: {e}"
obj = parse_json_text(text)
if obj is None:
return None, "no parseable JSON object found"
return obj, validate_obj(obj, schema_name)