Implement BDEF v1.1 grading: scoring core, per-deck pipeline, ledger, dashboard, StartOS layer
- Deterministic scoring.py (quant 60 / qual 40 / flags -15, profitability heaviest) - Per-company JSON ledger with forecast-target chaining deck N-1 -> N - Single-shot sandbox agent with guided-JSON fallback ladder (no tool loop) - Portfolio dashboard with sparklines, KPI hit rates, BDEF category bars - 48 unit tests green; endpoints smoke-tested; npm check+build green Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Jonathan Kirkwood
co-authored by
Claude Fable 5
Claude Fable 5
parent
1dde915540
commit
b1d7aed9f4
@@ -4,26 +4,33 @@ import { configFile } from '../file-models/config'
|
||||
const { InputSpec, Value } = sdk
|
||||
|
||||
const inputSpec = InputSpec.of({
|
||||
reviewInstructions: Value.textarea({
|
||||
name: 'Review Rubric',
|
||||
bdefOverride: Value.textarea({
|
||||
name: 'BDEF Rubric Override (optional)',
|
||||
description:
|
||||
'What every reviewer should look for and produce. Layered above each ' +
|
||||
'reviewer\'s persona. Be concrete about the structure you want back.',
|
||||
required: true,
|
||||
'Leave EMPTY to grade against the baked-in BDEF v1.1 framework ' +
|
||||
'(Girdley + Munger/Buffett). Non-empty text replaces the rubric wholesale, ' +
|
||||
'so include the qualitative categories A-H if you customize it.',
|
||||
required: false,
|
||||
default: null,
|
||||
minRows: 5,
|
||||
maxRows: 20,
|
||||
placeholder:
|
||||
'Review the attached document(s). Produce: a short summary, key findings, ' +
|
||||
'risks/red flags, open questions, and recommendations. Cite the document and ' +
|
||||
'section for each point. Never invent facts not present in the documents.',
|
||||
maxRows: 24,
|
||||
placeholder: '(empty = the built-in BDEF v1.1 rubric)',
|
||||
}),
|
||||
extractorModel: Value.text({
|
||||
name: 'Extractor Model (optional)',
|
||||
description:
|
||||
'Catalog alias of the model that runs the stage-1 structured KPI ' +
|
||||
'extraction over each deck. Empty = use the first model.',
|
||||
required: false,
|
||||
default: null,
|
||||
placeholder: 'grader-a',
|
||||
}),
|
||||
networkMode: Value.select({
|
||||
name: 'Network Mode',
|
||||
description:
|
||||
'Air-gapped: reviewers reach ONLY the on-Spark model proxy — zero internet, ' +
|
||||
'documents never leave your hardware (models must be pre-pulled into the ' +
|
||||
'Spark HF cache, all on the head Spark). Local services: reviewers may also ' +
|
||||
'Air-gapped: graders reach ONLY the on-Spark model proxy — zero internet, ' +
|
||||
'board decks never leave your hardware (models must be pre-pulled into the ' +
|
||||
'Spark HF cache, all on the head Spark). Local services: graders may also ' +
|
||||
'reach LAN services like SearXNG and the second Spark (this network has ' +
|
||||
'egress unless you firewall it).',
|
||||
default: 'airgapped',
|
||||
@@ -34,59 +41,151 @@ const inputSpec = InputSpec.of({
|
||||
}),
|
||||
searxngUrl: Value.text({
|
||||
name: 'SearXNG URL (local-services only)',
|
||||
description: 'JSON-search endpoint to give reviewers a web_search tool. Ignored in air-gapped mode. Empty = no web search.',
|
||||
description: 'JSON-search endpoint to give graders a web_search tool. Ignored in air-gapped mode. Empty = no web search.',
|
||||
required: false,
|
||||
default: null,
|
||||
placeholder: 'https://searxng.local',
|
||||
}),
|
||||
synthesisEnabled: Value.toggle({
|
||||
name: 'Synthesize a Consolidated Report',
|
||||
adjudicatorEnabled: Value.toggle({
|
||||
name: 'Run an Adjudicator',
|
||||
description:
|
||||
'After the panel finishes, run a local "lead reviewer" that reads all the ' +
|
||||
'individual reports and writes one consolidated report (themes, conflicts, ' +
|
||||
'consensus, recommendation). No frontier model — stays on the Sparks.',
|
||||
'After the panel finishes, run a local "lead grader" that reads every ' +
|
||||
'grade sheet, reconciles disagreements, and settles the qualitative scores ' +
|
||||
'the deterministic composite uses. No frontier model — stays on the Sparks.',
|
||||
default: true,
|
||||
}),
|
||||
synthesisModel: Value.text({
|
||||
name: 'Lead Reviewer Model (optional)',
|
||||
description: 'Catalog alias of the model that writes the consolidated report. Empty = use the first model.',
|
||||
adjudicatorModel: Value.text({
|
||||
name: 'Adjudicator Model (optional)',
|
||||
description: 'Catalog alias of the model that adjudicates the panel. Empty = use the first model.',
|
||||
required: false,
|
||||
default: null,
|
||||
placeholder: 'reviewer-a',
|
||||
placeholder: 'grader-a',
|
||||
}),
|
||||
synthesisPersona: Value.textarea({
|
||||
name: 'Lead Reviewer Instructions (optional)',
|
||||
description: 'Override how the consolidated report is written. Empty = a sensible built-in default.',
|
||||
adjudicatorPersona: Value.textarea({
|
||||
name: 'Adjudicator Instructions (optional)',
|
||||
description: 'Override how the adjudicator reconciles the panel. Empty = a sensible built-in default.',
|
||||
required: false,
|
||||
default: null,
|
||||
minRows: 3,
|
||||
maxRows: 14,
|
||||
}),
|
||||
wipeRemoteDocs: Value.toggle({
|
||||
name: 'Wipe Documents From Sparks After Review',
|
||||
name: 'Wipe Decks From Sparks After Grading',
|
||||
description:
|
||||
'Delete the extracted document text from the Sparks when a job finishes. ' +
|
||||
'Reports are always kept on this StartOS box. Recommended for confidential material.',
|
||||
'Delete the extracted deck text from the Sparks when a job finishes. ' +
|
||||
'Scorecards and ledgers are always kept on this StartOS box. Recommended ' +
|
||||
'for confidential board material.',
|
||||
default: true,
|
||||
}),
|
||||
autoRunOnDrop: Value.toggle({
|
||||
name: 'Auto-run When Documents Are Dropped',
|
||||
name: 'Auto-grade When Decks Are Dropped',
|
||||
description:
|
||||
'Start a review automatically (after a short debounce) whenever new files ' +
|
||||
'land in the inbox. Off by default so you trigger reviews explicitly.',
|
||||
'Start grading automatically (after a short debounce) whenever new decks ' +
|
||||
'land in the inbox. Off by default so you trigger grading explicitly.',
|
||||
default: false,
|
||||
}),
|
||||
// --- Deterministic-scorer weights (composite = quant 60 + qual 40 - flags) ---
|
||||
profitabilityKpi: Value.number({
|
||||
name: 'Weight: Profitability KPIs',
|
||||
description: 'Points for the profitability KPI attainment bucket — the heaviest slice of the quant 60. Default 30.',
|
||||
required: true,
|
||||
default: 30,
|
||||
integer: false,
|
||||
min: 0,
|
||||
max: 100,
|
||||
}),
|
||||
otherKpi: Value.number({
|
||||
name: 'Weight: Other KPIs',
|
||||
description: 'Points for the non-profitability measurable-KPI bucket. Default 20.',
|
||||
required: true,
|
||||
default: 20,
|
||||
integer: false,
|
||||
min: 0,
|
||||
max: 100,
|
||||
}),
|
||||
forecastIntegrity: Value.number({
|
||||
name: 'Weight: Forecast Integrity',
|
||||
description: 'Points for deck N actuals hitting what deck N-1 promised. Default 10.',
|
||||
required: true,
|
||||
default: 10,
|
||||
integer: false,
|
||||
min: 0,
|
||||
max: 100,
|
||||
}),
|
||||
qualCategoryMax: Value.number({
|
||||
name: 'Weight: Max Per Qualitative Category',
|
||||
description: 'Max points per BDEF category A-H (8 categories x 5 = the qualitative 40). Default 5.',
|
||||
required: true,
|
||||
default: 5,
|
||||
integer: false,
|
||||
min: 0,
|
||||
max: 100,
|
||||
}),
|
||||
redFlagCap: Value.number({
|
||||
name: 'Red-flag Penalty Cap',
|
||||
description: 'Maximum total points red flags can subtract from the composite. Default 15.',
|
||||
required: true,
|
||||
default: 15,
|
||||
integer: false,
|
||||
min: 0,
|
||||
max: 100,
|
||||
}),
|
||||
kpiCreditFloor: Value.number({
|
||||
name: 'KPI Credit Floor',
|
||||
description: 'actual/target ratio below which a KPI earns zero credit (linear credit above it). Default 0.5.',
|
||||
required: true,
|
||||
default: 0.5,
|
||||
integer: false,
|
||||
min: 0,
|
||||
max: 1,
|
||||
}),
|
||||
droppedKpiPenalty: Value.number({
|
||||
name: 'Dropped-KPI Penalty',
|
||||
description: 'Penalty per KPI that silently disappeared from the deck. Default 2.',
|
||||
required: true,
|
||||
default: 2,
|
||||
integer: false,
|
||||
min: 0,
|
||||
max: 100,
|
||||
}),
|
||||
droppedKpiMax: Value.number({
|
||||
name: 'Dropped-KPI Flag Limit',
|
||||
description: 'Count at most this many dropped-KPI flags per deck. Default 3.',
|
||||
required: true,
|
||||
default: 3,
|
||||
integer: true,
|
||||
min: 0,
|
||||
max: 50,
|
||||
}),
|
||||
evidenceFullCredit: Value.number({
|
||||
name: 'Evidence Full-credit Threshold',
|
||||
description: 'Quote characters a qualitative finding needs for full weight (thinner evidence scales down). Default 400.',
|
||||
required: true,
|
||||
default: 400,
|
||||
integer: true,
|
||||
min: 0,
|
||||
max: 100000,
|
||||
}),
|
||||
singleSourceFlagFactor: Value.number({
|
||||
name: 'Single-source Flag Damping',
|
||||
description: 'Multiplier applied to red flags raised by only one grader. Default 0.5.',
|
||||
required: true,
|
||||
default: 0.5,
|
||||
integer: false,
|
||||
min: 0,
|
||||
max: 1,
|
||||
}),
|
||||
})
|
||||
|
||||
export const configureReview = sdk.Action.withInput(
|
||||
'configure-review',
|
||||
export const configureGrading = sdk.Action.withInput(
|
||||
'configure-grading',
|
||||
|
||||
async ({ effects }) => ({
|
||||
name: 'Configure Review',
|
||||
description: 'Set the rubric, air-gap mode, synthesis, and document retention.',
|
||||
name: 'Configure Grading',
|
||||
description: 'Set the BDEF rubric, air-gap mode, adjudication, retention, and the scoring weights.',
|
||||
warning: null,
|
||||
allowedStatuses: 'any',
|
||||
group: null,
|
||||
group: 'Grading',
|
||||
visibility: 'enabled',
|
||||
}),
|
||||
|
||||
@@ -96,36 +195,60 @@ export const configureReview = sdk.Action.withInput(
|
||||
const cfg = await configFile.read().const(effects)
|
||||
if (!cfg) return {}
|
||||
return {
|
||||
reviewInstructions: cfg.reviewInstructions,
|
||||
bdefOverride: cfg.bdefOverride || undefined,
|
||||
extractorModel: cfg.extractorModel || undefined,
|
||||
networkMode: cfg.networkMode,
|
||||
searxngUrl: cfg.searxngUrl || undefined,
|
||||
synthesisEnabled: cfg.synthesisEnabled,
|
||||
synthesisModel: cfg.synthesisModel || undefined,
|
||||
synthesisPersona: cfg.synthesisPersona || undefined,
|
||||
adjudicatorEnabled: cfg.adjudicatorEnabled,
|
||||
adjudicatorModel: cfg.adjudicatorModel || undefined,
|
||||
adjudicatorPersona: cfg.adjudicatorPersona || undefined,
|
||||
wipeRemoteDocs: cfg.wipeRemoteDocs,
|
||||
autoRunOnDrop: cfg.autoRunOnDrop,
|
||||
profitabilityKpi: cfg.weights.profitabilityKpi,
|
||||
otherKpi: cfg.weights.otherKpi,
|
||||
forecastIntegrity: cfg.weights.forecastIntegrity,
|
||||
qualCategoryMax: cfg.weights.qualCategoryMax,
|
||||
redFlagCap: cfg.weights.redFlagCap,
|
||||
kpiCreditFloor: cfg.weights.kpiCreditFloor,
|
||||
droppedKpiPenalty: cfg.weights.droppedKpiPenalty,
|
||||
droppedKpiMax: cfg.weights.droppedKpiMax,
|
||||
evidenceFullCredit: cfg.weights.evidenceFullCredit,
|
||||
singleSourceFlagFactor: cfg.weights.singleSourceFlagFactor,
|
||||
}
|
||||
},
|
||||
|
||||
async ({ effects, input }) => {
|
||||
await configFile.merge(effects, {
|
||||
reviewInstructions: input.reviewInstructions,
|
||||
bdefOverride: input.bdefOverride ?? '',
|
||||
extractorModel: input.extractorModel ?? '',
|
||||
networkMode: input.networkMode,
|
||||
searxngUrl: input.searxngUrl ?? '',
|
||||
synthesisEnabled: input.synthesisEnabled,
|
||||
synthesisModel: input.synthesisModel ?? '',
|
||||
synthesisPersona: input.synthesisPersona ?? '',
|
||||
adjudicatorEnabled: input.adjudicatorEnabled,
|
||||
adjudicatorModel: input.adjudicatorModel ?? '',
|
||||
adjudicatorPersona: input.adjudicatorPersona ?? '',
|
||||
wipeRemoteDocs: input.wipeRemoteDocs,
|
||||
autoRunOnDrop: input.autoRunOnDrop,
|
||||
weights: {
|
||||
profitabilityKpi: input.profitabilityKpi,
|
||||
otherKpi: input.otherKpi,
|
||||
forecastIntegrity: input.forecastIntegrity,
|
||||
qualCategoryMax: input.qualCategoryMax,
|
||||
redFlagCap: input.redFlagCap,
|
||||
kpiCreditFloor: input.kpiCreditFloor,
|
||||
droppedKpiPenalty: input.droppedKpiPenalty,
|
||||
droppedKpiMax: input.droppedKpiMax,
|
||||
evidenceFullCredit: input.evidenceFullCredit,
|
||||
singleSourceFlagFactor: input.singleSourceFlagFactor,
|
||||
},
|
||||
})
|
||||
|
||||
return {
|
||||
version: '1',
|
||||
title: 'Review Settings Saved',
|
||||
title: 'Grading Settings Saved',
|
||||
message:
|
||||
input.networkMode === 'airgapped'
|
||||
? 'Saved. Reviewers will run air-gapped (no internet). Ensure all models are on the head Spark and pre-pulled into its HF cache.'
|
||||
: 'Saved. Reviewers run in local-services mode and may reach the network — make sure that is acceptable for these documents.',
|
||||
? 'Saved. Graders will run air-gapped (no internet). Ensure all models are on the head Spark and pre-pulled into its HF cache.'
|
||||
: 'Saved. Graders run in local-services mode and may reach the network — make sure that is acceptable for these board decks.',
|
||||
result: { type: 'single', value: input.networkMode, copyable: false, qr: false, masked: false },
|
||||
}
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user