Implement BDEF v1.1 grading: scoring core, per-deck pipeline, ledger, dashboard, StartOS layer
- Deterministic scoring.py (quant 60 / qual 40 / flags -15, profitability heaviest) - Per-company JSON ledger with forecast-target chaining deck N-1 -> N - Single-shot sandbox agent with guided-JSON fallback ladder (no tool loop) - Portfolio dashboard with sparklines, KPI hit rates, BDEF category bars - 48 unit tests green; endpoints smoke-tested; npm check+build green Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Jonathan Kirkwood
co-authored by
Claude Fable 5
Claude Fable 5
parent
1dde915540
commit
b1d7aed9f4
@@ -0,0 +1,185 @@
|
||||
import { sdk } from '../sdk'
|
||||
import { configFile } from '../file-models/config'
|
||||
|
||||
const { InputSpec, Value, List } = sdk
|
||||
|
||||
const inputSpec = InputSpec.of({
|
||||
companies: Value.list(
|
||||
List.obj(
|
||||
{
|
||||
name: 'Portfolio Companies',
|
||||
description:
|
||||
'One entry per portfolio company. The SLUG is the folder you drop its ' +
|
||||
'decks into (inbox/<slug>/2026-Q2-deck.pdf) and the key its running ' +
|
||||
'scorecard ledger lives under. Pinned targets are the KPIs the scorer ' +
|
||||
'holds the company to even when a deck goes quiet about them — pin the ' +
|
||||
'profitability thresholds especially, since they carry the heaviest ' +
|
||||
'weight (30 of the quant 60).',
|
||||
default: [],
|
||||
minLength: 0,
|
||||
maxLength: 64,
|
||||
},
|
||||
{
|
||||
uniqueBy: 'slug',
|
||||
displayAs: '{{slug}}',
|
||||
spec: InputSpec.of({
|
||||
slug: Value.text({
|
||||
name: 'Slug',
|
||||
description:
|
||||
'Folder name under the inbox and the ledger key. Stable — do not ' +
|
||||
'rename once decks have been graded.',
|
||||
required: true,
|
||||
default: null,
|
||||
placeholder: 'acme-widgets',
|
||||
patterns: [
|
||||
{ regex: '^[a-z0-9][a-z0-9-]{0,40}$',
|
||||
description: 'Lowercase letters, numbers, dashes (max 41 chars).' },
|
||||
],
|
||||
}),
|
||||
name: Value.text({
|
||||
name: 'Display Name (optional)',
|
||||
description: 'Shown on the dashboard and scorecards. Empty = the slug.',
|
||||
required: false,
|
||||
default: null,
|
||||
placeholder: 'Acme Widgets, Inc.',
|
||||
}),
|
||||
kpiAliases: Value.textarea({
|
||||
name: 'KPI Aliases (optional)',
|
||||
description:
|
||||
'One line per KPI: canonical=alias1;alias2 — maps the names a deck ' +
|
||||
'uses onto the canonical KPI name, so "Adj. EBITDA" and "EBITDA ' +
|
||||
'(adj)" both land on the same pinned target.',
|
||||
required: false,
|
||||
default: null,
|
||||
minRows: 2,
|
||||
maxRows: 12,
|
||||
placeholder: 'ebitda=Adj. EBITDA;EBITDA (adj)\nmrr=Monthly Recurring Revenue;MRR',
|
||||
}),
|
||||
pinnedTargets: Value.list(
|
||||
List.obj(
|
||||
{
|
||||
name: 'Pinned KPI Targets',
|
||||
description:
|
||||
'Targets graded every quarter regardless of what the deck ' +
|
||||
'chooses to show. Mark profitability KPIs (EBITDA, net margin, ' +
|
||||
'FCF...) so they score in the heavier profitability bucket.',
|
||||
default: [],
|
||||
minLength: 0,
|
||||
maxLength: 32,
|
||||
},
|
||||
{
|
||||
uniqueBy: 'kpi',
|
||||
displayAs: '{{kpi}} → {{target}}',
|
||||
spec: InputSpec.of({
|
||||
kpi: Value.text({
|
||||
name: 'KPI Name',
|
||||
description: 'Canonical KPI name (see the aliases field above).',
|
||||
required: true,
|
||||
default: null,
|
||||
placeholder: 'ebitda',
|
||||
}),
|
||||
target: Value.number({
|
||||
name: 'Target',
|
||||
description: 'The numeric target for the period.',
|
||||
required: true,
|
||||
default: null,
|
||||
integer: false,
|
||||
}),
|
||||
unit: Value.text({
|
||||
name: 'Unit (optional)',
|
||||
description: 'For display only, e.g. "USD", "%", "customers".',
|
||||
required: false,
|
||||
default: null,
|
||||
placeholder: 'USD',
|
||||
}),
|
||||
direction: Value.select({
|
||||
name: 'Direction',
|
||||
description: 'Whether hitting the target means being at-or-above it (revenue) or at-or-below it (churn, burn).',
|
||||
default: 'gte',
|
||||
values: {
|
||||
gte: 'At or above target',
|
||||
lte: 'At or below target',
|
||||
},
|
||||
}),
|
||||
profitability: Value.toggle({
|
||||
name: 'Profitability KPI',
|
||||
description: 'Score this KPI in the profitability bucket (heaviest weight) instead of the general KPI bucket.',
|
||||
default: false,
|
||||
}),
|
||||
}),
|
||||
},
|
||||
),
|
||||
),
|
||||
}),
|
||||
},
|
||||
),
|
||||
),
|
||||
})
|
||||
|
||||
export const configureCompanies = sdk.Action.withInput(
|
||||
'configure-companies',
|
||||
|
||||
async ({ effects }) => ({
|
||||
name: 'Configure Companies',
|
||||
description: 'Define the portfolio companies, their inbox slugs, KPI aliases, and pinned KPI targets.',
|
||||
warning: null,
|
||||
allowedStatuses: 'any',
|
||||
group: 'Grading',
|
||||
visibility: 'enabled',
|
||||
}),
|
||||
|
||||
inputSpec,
|
||||
|
||||
async ({ effects }) => {
|
||||
const cfg = await configFile.read().const(effects)
|
||||
if (!cfg) return {}
|
||||
return {
|
||||
companies: cfg.companies.map((c) => ({
|
||||
slug: c.slug,
|
||||
name: c.name || undefined,
|
||||
kpiAliases: c.kpiAliases || undefined,
|
||||
pinnedTargets: c.pinnedTargets.map((t) => ({
|
||||
kpi: t.kpi,
|
||||
target: t.target,
|
||||
unit: t.unit || undefined,
|
||||
direction: t.direction,
|
||||
profitability: t.profitability,
|
||||
})),
|
||||
})),
|
||||
}
|
||||
},
|
||||
|
||||
async ({ effects, input }) => {
|
||||
await configFile.merge(effects, {
|
||||
companies: input.companies.map((c) => ({
|
||||
slug: c.slug,
|
||||
name: c.name ?? '',
|
||||
kpiAliases: c.kpiAliases ?? '',
|
||||
pinnedTargets: c.pinnedTargets.map((t) => ({
|
||||
kpi: t.kpi,
|
||||
target: t.target,
|
||||
unit: t.unit ?? '',
|
||||
direction: t.direction,
|
||||
profitability: t.profitability,
|
||||
})),
|
||||
})),
|
||||
})
|
||||
|
||||
return {
|
||||
version: '1',
|
||||
title: 'Companies Configured',
|
||||
message:
|
||||
'Saved ' + input.companies.length + ' company(ies). Drop each company\'s ' +
|
||||
'decks into inbox/<slug>/ (e.g. inbox/' +
|
||||
(input.companies[0]?.slug || 'acme-widgets') +
|
||||
'/2026-Q2-deck.pdf), then run "Grade Decks".',
|
||||
result: {
|
||||
type: 'single',
|
||||
value: input.companies.map((c) => c.slug).join(', ') || '(none)',
|
||||
copyable: false,
|
||||
qr: false,
|
||||
masked: false,
|
||||
},
|
||||
}
|
||||
},
|
||||
)
|
||||
@@ -4,14 +4,15 @@ import { configFile } from '../file-models/config'
|
||||
const { InputSpec, Value, List } = sdk
|
||||
|
||||
const inputSpec = InputSpec.of({
|
||||
reviewers: Value.list(
|
||||
graders: Value.list(
|
||||
List.obj(
|
||||
{
|
||||
name: 'Review Panel',
|
||||
name: 'Grading Panel',
|
||||
description:
|
||||
'Who sits on the panel — one entry per review. Add as many as you like. ' +
|
||||
'Each reviewer is a model from your catalog plus a PERSONA: the lens it ' +
|
||||
'reads through, so the same document gets examined from different angles.',
|
||||
'Who grades the decks — one entry per grader. Add as many as you like. ' +
|
||||
'Each grader is a model from your catalog plus a PERSONA: the lens it ' +
|
||||
'grades through, so the same deck gets scored from different angles ' +
|
||||
'before the deterministic composite is computed.',
|
||||
default: [],
|
||||
minLength: 1,
|
||||
maxLength: 32,
|
||||
@@ -22,10 +23,10 @@ const inputSpec = InputSpec.of({
|
||||
spec: InputSpec.of({
|
||||
name: Value.text({
|
||||
name: 'Name',
|
||||
description: 'Unique reviewer name. Becomes its container and report filename.',
|
||||
description: 'Unique grader name. Becomes its container and grade-sheet filename.',
|
||||
required: true,
|
||||
default: null,
|
||||
placeholder: 'risk-counsel',
|
||||
placeholder: 'munger-lens',
|
||||
patterns: [
|
||||
{ regex: '^[A-Za-z0-9][A-Za-z0-9 _-]{0,40}$',
|
||||
description: 'Letters, numbers, spaces, dashes, underscores (max 41 chars).' },
|
||||
@@ -33,31 +34,32 @@ const inputSpec = InputSpec.of({
|
||||
}),
|
||||
model: Value.text({
|
||||
name: 'Model Alias',
|
||||
description: 'Which catalog model this reviewer uses (must match an alias from "Configure Models").',
|
||||
description: 'Which catalog model this grader uses (must match an alias from "Configure Models").',
|
||||
required: true,
|
||||
default: null,
|
||||
placeholder: 'reviewer-a',
|
||||
placeholder: 'grader-a',
|
||||
}),
|
||||
persona: Value.textarea({
|
||||
name: 'Persona / Lens',
|
||||
description:
|
||||
'How THIS reviewer should read the documents — its priorities and ' +
|
||||
'weighting. Injected into its system prompt. e.g. "You are skeptical ' +
|
||||
'legal counsel: weight liability, ambiguous obligations, and missing ' +
|
||||
'clauses above everything." Leave empty for a neutral reviewer.',
|
||||
'How THIS grader should read the deck — its priorities and ' +
|
||||
'weighting within the BDEF rubric. Injected into its system prompt. ' +
|
||||
'e.g. "You are a Munger-style inversion skeptic: ask what would have ' +
|
||||
'to be true for this deck to be hiding a deteriorating business." ' +
|
||||
'Leave empty for a neutral grader.',
|
||||
required: false,
|
||||
default: null,
|
||||
minRows: 3,
|
||||
maxRows: 16,
|
||||
placeholder:
|
||||
'You are a financial-controls reviewer. Focus on numbers that do not ' +
|
||||
'reconcile, unstated assumptions behind projections, and anything that ' +
|
||||
'would concern an auditor. Organize findings by severity.',
|
||||
'You are a Girdley-style operator. Weight unit economics, owner ' +
|
||||
'accountability, and whether the KPIs the board was promised last ' +
|
||||
'quarter are still being reported. Flag every silently dropped metric.',
|
||||
}),
|
||||
temperature: Value.number({
|
||||
name: 'Sampling Temperature (optional)',
|
||||
description:
|
||||
'Best-effort per-reviewer sampling temperature for extra diversity. ' +
|
||||
'Best-effort per-grader sampling temperature for extra diversity. ' +
|
||||
'Persona is the primary lever. Leave empty to use the model default.',
|
||||
required: false,
|
||||
default: null,
|
||||
@@ -71,15 +73,15 @@ const inputSpec = InputSpec.of({
|
||||
),
|
||||
})
|
||||
|
||||
export const configureReviewers = sdk.Action.withInput(
|
||||
'configure-reviewers',
|
||||
export const configureGraders = sdk.Action.withInput(
|
||||
'configure-graders',
|
||||
|
||||
async ({ effects }) => ({
|
||||
name: 'Configure Reviewers',
|
||||
description: 'Define the review panel: which models and which personas, and how many reviews.',
|
||||
name: 'Configure Graders',
|
||||
description: 'Define the grading panel: which models and which personas grade each deck.',
|
||||
warning: null,
|
||||
allowedStatuses: 'any',
|
||||
group: null,
|
||||
group: 'Grading',
|
||||
visibility: 'enabled',
|
||||
}),
|
||||
|
||||
@@ -88,22 +90,23 @@ export const configureReviewers = sdk.Action.withInput(
|
||||
async ({ effects }) => {
|
||||
const cfg = await configFile.read().const(effects)
|
||||
if (!cfg) return {}
|
||||
return { reviewers: cfg.reviewers }
|
||||
return { graders: cfg.graders }
|
||||
},
|
||||
|
||||
async ({ effects, input }) => {
|
||||
await configFile.merge(effects, { reviewers: input.reviewers })
|
||||
await configFile.merge(effects, { graders: input.graders })
|
||||
|
||||
return {
|
||||
version: '1',
|
||||
title: 'Panel Configured',
|
||||
message:
|
||||
'Saved a panel of ' + input.reviewers.length + ' reviewer(s). Each model ' +
|
||||
'alias must exist in "Configure Models". Set the rubric in "Configure ' +
|
||||
'Review", then drop documents and run a review.',
|
||||
'Saved a panel of ' + input.graders.length + ' grader(s). Each model ' +
|
||||
'alias must exist in "Configure Models". Set the scoring knobs in ' +
|
||||
'"Configure Grading", add companies in "Configure Companies", then drop ' +
|
||||
'decks into inbox/<company-slug>/ and run "Grade Decks".',
|
||||
result: {
|
||||
type: 'single',
|
||||
value: input.reviewers.map((r) => r.name).join(', '),
|
||||
value: input.graders.map((g) => g.name).join(', '),
|
||||
copyable: false,
|
||||
qr: false,
|
||||
masked: false,
|
||||
|
||||
@@ -4,26 +4,33 @@ import { configFile } from '../file-models/config'
|
||||
const { InputSpec, Value } = sdk
|
||||
|
||||
const inputSpec = InputSpec.of({
|
||||
reviewInstructions: Value.textarea({
|
||||
name: 'Review Rubric',
|
||||
bdefOverride: Value.textarea({
|
||||
name: 'BDEF Rubric Override (optional)',
|
||||
description:
|
||||
'What every reviewer should look for and produce. Layered above each ' +
|
||||
'reviewer\'s persona. Be concrete about the structure you want back.',
|
||||
required: true,
|
||||
'Leave EMPTY to grade against the baked-in BDEF v1.1 framework ' +
|
||||
'(Girdley + Munger/Buffett). Non-empty text replaces the rubric wholesale, ' +
|
||||
'so include the qualitative categories A-H if you customize it.',
|
||||
required: false,
|
||||
default: null,
|
||||
minRows: 5,
|
||||
maxRows: 20,
|
||||
placeholder:
|
||||
'Review the attached document(s). Produce: a short summary, key findings, ' +
|
||||
'risks/red flags, open questions, and recommendations. Cite the document and ' +
|
||||
'section for each point. Never invent facts not present in the documents.',
|
||||
maxRows: 24,
|
||||
placeholder: '(empty = the built-in BDEF v1.1 rubric)',
|
||||
}),
|
||||
extractorModel: Value.text({
|
||||
name: 'Extractor Model (optional)',
|
||||
description:
|
||||
'Catalog alias of the model that runs the stage-1 structured KPI ' +
|
||||
'extraction over each deck. Empty = use the first model.',
|
||||
required: false,
|
||||
default: null,
|
||||
placeholder: 'grader-a',
|
||||
}),
|
||||
networkMode: Value.select({
|
||||
name: 'Network Mode',
|
||||
description:
|
||||
'Air-gapped: reviewers reach ONLY the on-Spark model proxy — zero internet, ' +
|
||||
'documents never leave your hardware (models must be pre-pulled into the ' +
|
||||
'Spark HF cache, all on the head Spark). Local services: reviewers may also ' +
|
||||
'Air-gapped: graders reach ONLY the on-Spark model proxy — zero internet, ' +
|
||||
'board decks never leave your hardware (models must be pre-pulled into the ' +
|
||||
'Spark HF cache, all on the head Spark). Local services: graders may also ' +
|
||||
'reach LAN services like SearXNG and the second Spark (this network has ' +
|
||||
'egress unless you firewall it).',
|
||||
default: 'airgapped',
|
||||
@@ -34,59 +41,151 @@ const inputSpec = InputSpec.of({
|
||||
}),
|
||||
searxngUrl: Value.text({
|
||||
name: 'SearXNG URL (local-services only)',
|
||||
description: 'JSON-search endpoint to give reviewers a web_search tool. Ignored in air-gapped mode. Empty = no web search.',
|
||||
description: 'JSON-search endpoint to give graders a web_search tool. Ignored in air-gapped mode. Empty = no web search.',
|
||||
required: false,
|
||||
default: null,
|
||||
placeholder: 'https://searxng.local',
|
||||
}),
|
||||
synthesisEnabled: Value.toggle({
|
||||
name: 'Synthesize a Consolidated Report',
|
||||
adjudicatorEnabled: Value.toggle({
|
||||
name: 'Run an Adjudicator',
|
||||
description:
|
||||
'After the panel finishes, run a local "lead reviewer" that reads all the ' +
|
||||
'individual reports and writes one consolidated report (themes, conflicts, ' +
|
||||
'consensus, recommendation). No frontier model — stays on the Sparks.',
|
||||
'After the panel finishes, run a local "lead grader" that reads every ' +
|
||||
'grade sheet, reconciles disagreements, and settles the qualitative scores ' +
|
||||
'the deterministic composite uses. No frontier model — stays on the Sparks.',
|
||||
default: true,
|
||||
}),
|
||||
synthesisModel: Value.text({
|
||||
name: 'Lead Reviewer Model (optional)',
|
||||
description: 'Catalog alias of the model that writes the consolidated report. Empty = use the first model.',
|
||||
adjudicatorModel: Value.text({
|
||||
name: 'Adjudicator Model (optional)',
|
||||
description: 'Catalog alias of the model that adjudicates the panel. Empty = use the first model.',
|
||||
required: false,
|
||||
default: null,
|
||||
placeholder: 'reviewer-a',
|
||||
placeholder: 'grader-a',
|
||||
}),
|
||||
synthesisPersona: Value.textarea({
|
||||
name: 'Lead Reviewer Instructions (optional)',
|
||||
description: 'Override how the consolidated report is written. Empty = a sensible built-in default.',
|
||||
adjudicatorPersona: Value.textarea({
|
||||
name: 'Adjudicator Instructions (optional)',
|
||||
description: 'Override how the adjudicator reconciles the panel. Empty = a sensible built-in default.',
|
||||
required: false,
|
||||
default: null,
|
||||
minRows: 3,
|
||||
maxRows: 14,
|
||||
}),
|
||||
wipeRemoteDocs: Value.toggle({
|
||||
name: 'Wipe Documents From Sparks After Review',
|
||||
name: 'Wipe Decks From Sparks After Grading',
|
||||
description:
|
||||
'Delete the extracted document text from the Sparks when a job finishes. ' +
|
||||
'Reports are always kept on this StartOS box. Recommended for confidential material.',
|
||||
'Delete the extracted deck text from the Sparks when a job finishes. ' +
|
||||
'Scorecards and ledgers are always kept on this StartOS box. Recommended ' +
|
||||
'for confidential board material.',
|
||||
default: true,
|
||||
}),
|
||||
autoRunOnDrop: Value.toggle({
|
||||
name: 'Auto-run When Documents Are Dropped',
|
||||
name: 'Auto-grade When Decks Are Dropped',
|
||||
description:
|
||||
'Start a review automatically (after a short debounce) whenever new files ' +
|
||||
'land in the inbox. Off by default so you trigger reviews explicitly.',
|
||||
'Start grading automatically (after a short debounce) whenever new decks ' +
|
||||
'land in the inbox. Off by default so you trigger grading explicitly.',
|
||||
default: false,
|
||||
}),
|
||||
// --- Deterministic-scorer weights (composite = quant 60 + qual 40 - flags) ---
|
||||
profitabilityKpi: Value.number({
|
||||
name: 'Weight: Profitability KPIs',
|
||||
description: 'Points for the profitability KPI attainment bucket — the heaviest slice of the quant 60. Default 30.',
|
||||
required: true,
|
||||
default: 30,
|
||||
integer: false,
|
||||
min: 0,
|
||||
max: 100,
|
||||
}),
|
||||
otherKpi: Value.number({
|
||||
name: 'Weight: Other KPIs',
|
||||
description: 'Points for the non-profitability measurable-KPI bucket. Default 20.',
|
||||
required: true,
|
||||
default: 20,
|
||||
integer: false,
|
||||
min: 0,
|
||||
max: 100,
|
||||
}),
|
||||
forecastIntegrity: Value.number({
|
||||
name: 'Weight: Forecast Integrity',
|
||||
description: 'Points for deck N actuals hitting what deck N-1 promised. Default 10.',
|
||||
required: true,
|
||||
default: 10,
|
||||
integer: false,
|
||||
min: 0,
|
||||
max: 100,
|
||||
}),
|
||||
qualCategoryMax: Value.number({
|
||||
name: 'Weight: Max Per Qualitative Category',
|
||||
description: 'Max points per BDEF category A-H (8 categories x 5 = the qualitative 40). Default 5.',
|
||||
required: true,
|
||||
default: 5,
|
||||
integer: false,
|
||||
min: 0,
|
||||
max: 100,
|
||||
}),
|
||||
redFlagCap: Value.number({
|
||||
name: 'Red-flag Penalty Cap',
|
||||
description: 'Maximum total points red flags can subtract from the composite. Default 15.',
|
||||
required: true,
|
||||
default: 15,
|
||||
integer: false,
|
||||
min: 0,
|
||||
max: 100,
|
||||
}),
|
||||
kpiCreditFloor: Value.number({
|
||||
name: 'KPI Credit Floor',
|
||||
description: 'actual/target ratio below which a KPI earns zero credit (linear credit above it). Default 0.5.',
|
||||
required: true,
|
||||
default: 0.5,
|
||||
integer: false,
|
||||
min: 0,
|
||||
max: 1,
|
||||
}),
|
||||
droppedKpiPenalty: Value.number({
|
||||
name: 'Dropped-KPI Penalty',
|
||||
description: 'Penalty per KPI that silently disappeared from the deck. Default 2.',
|
||||
required: true,
|
||||
default: 2,
|
||||
integer: false,
|
||||
min: 0,
|
||||
max: 100,
|
||||
}),
|
||||
droppedKpiMax: Value.number({
|
||||
name: 'Dropped-KPI Flag Limit',
|
||||
description: 'Count at most this many dropped-KPI flags per deck. Default 3.',
|
||||
required: true,
|
||||
default: 3,
|
||||
integer: true,
|
||||
min: 0,
|
||||
max: 50,
|
||||
}),
|
||||
evidenceFullCredit: Value.number({
|
||||
name: 'Evidence Full-credit Threshold',
|
||||
description: 'Quote characters a qualitative finding needs for full weight (thinner evidence scales down). Default 400.',
|
||||
required: true,
|
||||
default: 400,
|
||||
integer: true,
|
||||
min: 0,
|
||||
max: 100000,
|
||||
}),
|
||||
singleSourceFlagFactor: Value.number({
|
||||
name: 'Single-source Flag Damping',
|
||||
description: 'Multiplier applied to red flags raised by only one grader. Default 0.5.',
|
||||
required: true,
|
||||
default: 0.5,
|
||||
integer: false,
|
||||
min: 0,
|
||||
max: 1,
|
||||
}),
|
||||
})
|
||||
|
||||
export const configureReview = sdk.Action.withInput(
|
||||
'configure-review',
|
||||
export const configureGrading = sdk.Action.withInput(
|
||||
'configure-grading',
|
||||
|
||||
async ({ effects }) => ({
|
||||
name: 'Configure Review',
|
||||
description: 'Set the rubric, air-gap mode, synthesis, and document retention.',
|
||||
name: 'Configure Grading',
|
||||
description: 'Set the BDEF rubric, air-gap mode, adjudication, retention, and the scoring weights.',
|
||||
warning: null,
|
||||
allowedStatuses: 'any',
|
||||
group: null,
|
||||
group: 'Grading',
|
||||
visibility: 'enabled',
|
||||
}),
|
||||
|
||||
@@ -96,36 +195,60 @@ export const configureReview = sdk.Action.withInput(
|
||||
const cfg = await configFile.read().const(effects)
|
||||
if (!cfg) return {}
|
||||
return {
|
||||
reviewInstructions: cfg.reviewInstructions,
|
||||
bdefOverride: cfg.bdefOverride || undefined,
|
||||
extractorModel: cfg.extractorModel || undefined,
|
||||
networkMode: cfg.networkMode,
|
||||
searxngUrl: cfg.searxngUrl || undefined,
|
||||
synthesisEnabled: cfg.synthesisEnabled,
|
||||
synthesisModel: cfg.synthesisModel || undefined,
|
||||
synthesisPersona: cfg.synthesisPersona || undefined,
|
||||
adjudicatorEnabled: cfg.adjudicatorEnabled,
|
||||
adjudicatorModel: cfg.adjudicatorModel || undefined,
|
||||
adjudicatorPersona: cfg.adjudicatorPersona || undefined,
|
||||
wipeRemoteDocs: cfg.wipeRemoteDocs,
|
||||
autoRunOnDrop: cfg.autoRunOnDrop,
|
||||
profitabilityKpi: cfg.weights.profitabilityKpi,
|
||||
otherKpi: cfg.weights.otherKpi,
|
||||
forecastIntegrity: cfg.weights.forecastIntegrity,
|
||||
qualCategoryMax: cfg.weights.qualCategoryMax,
|
||||
redFlagCap: cfg.weights.redFlagCap,
|
||||
kpiCreditFloor: cfg.weights.kpiCreditFloor,
|
||||
droppedKpiPenalty: cfg.weights.droppedKpiPenalty,
|
||||
droppedKpiMax: cfg.weights.droppedKpiMax,
|
||||
evidenceFullCredit: cfg.weights.evidenceFullCredit,
|
||||
singleSourceFlagFactor: cfg.weights.singleSourceFlagFactor,
|
||||
}
|
||||
},
|
||||
|
||||
async ({ effects, input }) => {
|
||||
await configFile.merge(effects, {
|
||||
reviewInstructions: input.reviewInstructions,
|
||||
bdefOverride: input.bdefOverride ?? '',
|
||||
extractorModel: input.extractorModel ?? '',
|
||||
networkMode: input.networkMode,
|
||||
searxngUrl: input.searxngUrl ?? '',
|
||||
synthesisEnabled: input.synthesisEnabled,
|
||||
synthesisModel: input.synthesisModel ?? '',
|
||||
synthesisPersona: input.synthesisPersona ?? '',
|
||||
adjudicatorEnabled: input.adjudicatorEnabled,
|
||||
adjudicatorModel: input.adjudicatorModel ?? '',
|
||||
adjudicatorPersona: input.adjudicatorPersona ?? '',
|
||||
wipeRemoteDocs: input.wipeRemoteDocs,
|
||||
autoRunOnDrop: input.autoRunOnDrop,
|
||||
weights: {
|
||||
profitabilityKpi: input.profitabilityKpi,
|
||||
otherKpi: input.otherKpi,
|
||||
forecastIntegrity: input.forecastIntegrity,
|
||||
qualCategoryMax: input.qualCategoryMax,
|
||||
redFlagCap: input.redFlagCap,
|
||||
kpiCreditFloor: input.kpiCreditFloor,
|
||||
droppedKpiPenalty: input.droppedKpiPenalty,
|
||||
droppedKpiMax: input.droppedKpiMax,
|
||||
evidenceFullCredit: input.evidenceFullCredit,
|
||||
singleSourceFlagFactor: input.singleSourceFlagFactor,
|
||||
},
|
||||
})
|
||||
|
||||
return {
|
||||
version: '1',
|
||||
title: 'Review Settings Saved',
|
||||
title: 'Grading Settings Saved',
|
||||
message:
|
||||
input.networkMode === 'airgapped'
|
||||
? 'Saved. Reviewers will run air-gapped (no internet). Ensure all models are on the head Spark and pre-pulled into its HF cache.'
|
||||
: 'Saved. Reviewers run in local-services mode and may reach the network — make sure that is acceptable for these documents.',
|
||||
? 'Saved. Graders will run air-gapped (no internet). Ensure all models are on the head Spark and pre-pulled into its HF cache.'
|
||||
: 'Saved. Graders run in local-services mode and may reach the network — make sure that is acceptable for these board decks.',
|
||||
result: { type: 'single', value: input.networkMode, copyable: false, qr: false, masked: false },
|
||||
}
|
||||
},
|
||||
|
||||
@@ -9,7 +9,7 @@ const inputSpec = InputSpec.of({
|
||||
{
|
||||
name: 'Model Catalog',
|
||||
description:
|
||||
'The local models this service can serve on your Sparks. Each reviewer ' +
|
||||
'The local models this service can serve on your Sparks. Each grader ' +
|
||||
'references one of these by its alias. The job runner loads models in ' +
|
||||
'waves so you can run a panel across more models than fit in GPU memory ' +
|
||||
'at once.',
|
||||
@@ -23,10 +23,10 @@ const inputSpec = InputSpec.of({
|
||||
spec: InputSpec.of({
|
||||
alias: Value.text({
|
||||
name: 'Alias',
|
||||
description: 'Short name reviewers use to pick this model (e.g. "qwen-32b").',
|
||||
description: 'Short name graders use to pick this model (e.g. "qwen-32b").',
|
||||
required: true,
|
||||
default: null,
|
||||
placeholder: 'reviewer-a',
|
||||
placeholder: 'grader-a',
|
||||
patterns: [
|
||||
{ regex: '^[a-z0-9][a-z0-9-]{0,30}$',
|
||||
description: 'Lowercase letters, numbers, dashes (max 31 chars).' },
|
||||
@@ -42,7 +42,7 @@ const inputSpec = InputSpec.of({
|
||||
spark: Value.select({
|
||||
name: 'Served On',
|
||||
description:
|
||||
'Which Spark serves this model. Air-gapped review mode requires the ' +
|
||||
'Which Spark serves this model. Air-gapped grading mode requires the ' +
|
||||
'head (primary) Spark; the secondary is used only in local-services mode.',
|
||||
default: 'primary',
|
||||
values: { primary: 'Primary (head) Spark', secondary: 'Secondary Spark' },
|
||||
@@ -68,7 +68,7 @@ const inputSpec = InputSpec.of({
|
||||
}),
|
||||
maxModelLen: Value.number({
|
||||
name: 'Max Model Length',
|
||||
description: 'vLLM --max-model-len (context window). Documents are chunked to fit.',
|
||||
description: 'vLLM --max-model-len (context window). Deck text is chunked to fit.',
|
||||
required: true,
|
||||
default: 32768,
|
||||
integer: true,
|
||||
@@ -77,7 +77,7 @@ const inputSpec = InputSpec.of({
|
||||
toolCallParser: Value.text({
|
||||
name: 'Tool-Call Parser',
|
||||
description:
|
||||
'vLLM tool-call parser for the reviewer\'s read-file tool loop. Match the ' +
|
||||
'vLLM tool-call parser for the grader\'s read-file tool loop. Match the ' +
|
||||
'served model family (Qwen3 → "hermes"). Empty disables native tool-calling.',
|
||||
required: false,
|
||||
default: 'hermes',
|
||||
@@ -112,7 +112,7 @@ export const configureModels = sdk.Action.withInput(
|
||||
description: 'Define the local model catalog served on your Sparks and the serving knobs.',
|
||||
warning: null,
|
||||
allowedStatuses: 'any',
|
||||
group: null,
|
||||
group: 'Setup',
|
||||
visibility: 'enabled',
|
||||
}),
|
||||
|
||||
@@ -146,7 +146,7 @@ export const configureModels = sdk.Action.withInput(
|
||||
title: 'Models Configured',
|
||||
message:
|
||||
'Saved ' + input.models.length + ' model(s). Make sure each is present in ' +
|
||||
'the Spark HF cache for air-gapped runs, then set "Configure Reviewers".',
|
||||
'the Spark HF cache for air-gapped runs, then set "Configure Graders".',
|
||||
result: {
|
||||
type: 'single',
|
||||
value: input.models.map((m) => m.alias).join(', '),
|
||||
|
||||
@@ -7,7 +7,7 @@ const { InputSpec, Value } = sdk
|
||||
const inputSpec = InputSpec.of({
|
||||
primarySparkHost: Value.text({
|
||||
name: 'Primary Spark Host',
|
||||
description: 'Hostname or IP of the head DGX Spark (reachable over SSH). Serves models, hosts the model proxy, and runs the reviewer panel.',
|
||||
description: 'Hostname or IP of the head DGX Spark (reachable over SSH). Serves models, hosts the model proxy, and runs the grading panel.',
|
||||
required: true,
|
||||
default: null,
|
||||
placeholder: 'spark-01.local',
|
||||
@@ -46,8 +46,8 @@ const inputSpec = InputSpec.of({
|
||||
name: 'Use Both Sparks',
|
||||
description:
|
||||
'Allow models to be served on a second Spark (over ConnectX/200GbE) for ' +
|
||||
'extra capacity. NOTE: in air-gapped review mode all models must run on the ' +
|
||||
'head Spark; the second Spark is used only in local-services mode.',
|
||||
'extra capacity. NOTE: in air-gapped grading mode all models must run on ' +
|
||||
'the head Spark; the second Spark is used only in local-services mode.',
|
||||
default: false,
|
||||
}),
|
||||
secondarySparkHost: Value.text({
|
||||
@@ -67,7 +67,7 @@ const inputSpec = InputSpec.of({
|
||||
}),
|
||||
remoteWorkDir: Value.text({
|
||||
name: 'Remote Work Directory',
|
||||
description: 'Absolute path on the head Spark for staged document text, the HF cache, and logs.',
|
||||
description: 'Absolute path on the head Spark for staged deck text, the HF cache, and logs.',
|
||||
required: true,
|
||||
default: '/home/nvidia/boardroom-map',
|
||||
}),
|
||||
@@ -78,8 +78,8 @@ const inputSpec = InputSpec.of({
|
||||
default: 'boardroom-vllm:latest',
|
||||
}),
|
||||
graderImage: Value.text({
|
||||
name: 'Reviewer Image Tag',
|
||||
description: 'The sandboxed reviewer image built on the head Spark from sandbox/build.sh.',
|
||||
name: 'Grader Image Tag',
|
||||
description: 'The sandboxed grader image built on the head Spark from sandbox/build.sh.',
|
||||
required: true,
|
||||
default: 'boardroom-grader:latest',
|
||||
}),
|
||||
@@ -103,7 +103,7 @@ export const configureSparks = sdk.Action.withInput(
|
||||
description: 'Set the DGX Spark connection details, SSH credentials, and image tags.',
|
||||
warning: null,
|
||||
allowedStatuses: 'any',
|
||||
group: null,
|
||||
group: 'Setup',
|
||||
visibility: 'enabled',
|
||||
}),
|
||||
|
||||
@@ -154,7 +154,7 @@ export const configureSparks = sdk.Action.withInput(
|
||||
title: 'Sparks Configured',
|
||||
message:
|
||||
'Saved. Use "Test Spark Connection" to verify SSH + GPU access, then set ' +
|
||||
'"Configure Models" and "Configure Reviewers".',
|
||||
'"Configure Models" and "Configure Graders".',
|
||||
result: { type: 'single', value: input.primarySparkHost, copyable: false, qr: false, masked: false },
|
||||
}
|
||||
},
|
||||
|
||||
@@ -2,17 +2,17 @@ import { startSdk } from '@start9labs/start-sdk'
|
||||
import { sdk } from '../sdk'
|
||||
|
||||
/**
|
||||
* Trigger a review of whatever is currently in the inbox. The running job-runner
|
||||
* Trigger grading of whatever is currently in the inbox. The running job-runner
|
||||
* thread polls for /data/state/run_request and starts a job when it appears, so
|
||||
* this action just drops that request file (decoupled from the daemon — no need
|
||||
* to reach its HTTP port from the action's one-shot container).
|
||||
*/
|
||||
export const runReview = sdk.Action.withoutInput(
|
||||
'run-review',
|
||||
export const gradeDecks = sdk.Action.withoutInput(
|
||||
'grade-decks',
|
||||
|
||||
async ({ effects }) => ({
|
||||
name: 'Run Review',
|
||||
description: 'Convene the panel now over the documents currently in the inbox.',
|
||||
name: 'Grade Decks',
|
||||
description: 'Grade all decks currently in the inbox (inbox/<company-slug>/...).',
|
||||
warning: null,
|
||||
allowedStatuses: 'only-running',
|
||||
group: null,
|
||||
@@ -36,23 +36,23 @@ export const runReview = sdk.Action.withoutInput(
|
||||
'sh',
|
||||
'-c',
|
||||
'mkdir -p /data/state && date +%s > /data/state/run_request && ' +
|
||||
'n=$(ls -1 /data/inbox 2>/dev/null | wc -l | tr -d " "); ' +
|
||||
'echo "Review requested. $n file(s) in the inbox."',
|
||||
'n=$(find /data/inbox -type f 2>/dev/null | wc -l | tr -d " "); ' +
|
||||
'echo "Grading requested. $n deck file(s) in the inbox."',
|
||||
],
|
||||
{ mounts, env: { BM_DATA_DIR: '/data' } },
|
||||
'run-review',
|
||||
'grade-decks',
|
||||
)
|
||||
output = (stdout?.toString() || '').trim() || 'Review requested.'
|
||||
output = (stdout?.toString() || '').trim() || 'Grading requested.'
|
||||
} catch (e: any) {
|
||||
output = 'Could not request a review: ' + (e?.message || String(e))
|
||||
output = 'Could not request grading: ' + (e?.message || String(e))
|
||||
}
|
||||
|
||||
return {
|
||||
version: '1',
|
||||
title: 'Review Requested',
|
||||
title: 'Grading Requested',
|
||||
message:
|
||||
output +
|
||||
' Watch the Web UI for progress; reports appear there and via "View Latest Report".',
|
||||
' Watch the Web UI for progress; scorecards appear there and via "View Latest Scorecard".',
|
||||
result: { type: 'single', value: output, copyable: false, qr: false, masked: false },
|
||||
}
|
||||
},
|
||||
|
||||
+15
-10
@@ -1,17 +1,22 @@
|
||||
import { sdk } from '../sdk'
|
||||
import { configureSparks } from './configure-sparks'
|
||||
import { configureModels } from './configure-models'
|
||||
import { configureReviewers } from './configure-reviewers'
|
||||
import { configureReview } from './configure-review'
|
||||
import { runReview } from './run-review'
|
||||
import { testConnection } from './test-connection'
|
||||
import { latestReport } from './latest-report'
|
||||
import { configureModels } from './configure-models'
|
||||
import { configureGraders } from './configure-graders'
|
||||
import { configureGrading } from './configure-grading'
|
||||
import { configureCompanies } from './configure-companies'
|
||||
import { gradeDecks } from './grade-decks'
|
||||
import { latestScorecard } from './latest-scorecard'
|
||||
|
||||
// Setup group: Sparks -> connection test -> model catalog.
|
||||
// Grading group: panel -> scoring knobs -> portfolio companies.
|
||||
// Ungrouped (day-to-day): grade the inbox, read the latest scorecard.
|
||||
export const actions = sdk.Actions.of()
|
||||
.addAction(configureSparks)
|
||||
.addAction(configureModels)
|
||||
.addAction(configureReviewers)
|
||||
.addAction(configureReview)
|
||||
.addAction(runReview)
|
||||
.addAction(testConnection)
|
||||
.addAction(latestReport)
|
||||
.addAction(configureModels)
|
||||
.addAction(configureGraders)
|
||||
.addAction(configureGrading)
|
||||
.addAction(configureCompanies)
|
||||
.addAction(gradeDecks)
|
||||
.addAction(latestScorecard)
|
||||
|
||||
@@ -2,17 +2,18 @@ import { startSdk } from '@start9labs/start-sdk'
|
||||
import { sdk } from '../sdk'
|
||||
|
||||
/**
|
||||
* Returns the latest report as a copyable result, so you can read it straight
|
||||
* from the StartOS service page without opening the Web UI. The job runner saves
|
||||
* the most recent report (consolidated if synthesis is on, else the panel
|
||||
* digest) to /data/reports/latest.md on the StartOS host — no Spark round-trip.
|
||||
* Returns the latest scorecard as a copyable result, so you can read it straight
|
||||
* from the StartOS service page without opening the dashboard. The job runner
|
||||
* saves the most recent scorecard to /data/reports/latest-scorecard.md (with
|
||||
* /data/reports/latest.md as the legacy fallback) on the StartOS host — no
|
||||
* Spark round-trip.
|
||||
*/
|
||||
export const latestReport = sdk.Action.withoutInput(
|
||||
'latest-report',
|
||||
export const latestScorecard = sdk.Action.withoutInput(
|
||||
'latest-scorecard',
|
||||
|
||||
async ({ effects }) => ({
|
||||
name: 'View Latest Report',
|
||||
description: 'Show the most recent review report produced by the panel.',
|
||||
name: 'View Latest Scorecard',
|
||||
description: 'Show the most recent deck scorecard produced by the grading panel.',
|
||||
warning: null,
|
||||
allowedStatuses: 'any',
|
||||
group: null,
|
||||
@@ -32,19 +33,25 @@ export const latestReport = sdk.Action.withoutInput(
|
||||
const { stdout } = await startSdk.runCommand<typeof sdk.manifest>(
|
||||
effects,
|
||||
{ imageId: 'main' },
|
||||
['sh', '-c', 'cat /data/reports/latest.md 2>/dev/null || echo "(no report yet — drop documents in the inbox and run a review)"'],
|
||||
[
|
||||
'sh',
|
||||
'-c',
|
||||
'cat /data/reports/latest-scorecard.md 2>/dev/null || ' +
|
||||
'cat /data/reports/latest.md 2>/dev/null || ' +
|
||||
'echo "(no scorecard yet — drop decks into inbox/<company-slug>/ and run Grade Decks)"',
|
||||
],
|
||||
{ mounts, env: { BM_DATA_DIR: '/data' } },
|
||||
'latest-report',
|
||||
'latest-scorecard',
|
||||
)
|
||||
report = (stdout?.toString() || '').trim() || '(no report yet)'
|
||||
report = (stdout?.toString() || '').trim() || '(no scorecard yet)'
|
||||
} catch (e: any) {
|
||||
report = 'Could not read report: ' + (e?.message || String(e))
|
||||
report = 'Could not read scorecard: ' + (e?.message || String(e))
|
||||
}
|
||||
|
||||
return {
|
||||
version: '1',
|
||||
title: 'Latest Boardroom Map Report',
|
||||
message: 'The panel\'s most recent review.',
|
||||
title: 'Latest Scorecard',
|
||||
message: 'The panel\'s most recent deck scorecard.',
|
||||
result: { type: 'single', value: report, copyable: true, qr: false, masked: false },
|
||||
}
|
||||
},
|
||||
|
||||
@@ -3,18 +3,19 @@ import { sdk } from '../sdk'
|
||||
|
||||
/**
|
||||
* Runs a one-shot in the orchestrator image that SSHes into the configured
|
||||
* Spark(s) and reports `nvidia-smi` plus whether the vLLM + reviewer images are
|
||||
* built. Reuses orchestrator/spark_client.py so SSH logic lives in one place.
|
||||
* Spark(s) and reports `nvidia-smi` plus whether the vLLM (boardroom-vllm) and
|
||||
* grader (boardroom-grader) images are built. Reuses orchestrator/spark_client.py
|
||||
* so SSH logic lives in one place.
|
||||
*/
|
||||
export const testConnection = sdk.Action.withoutInput(
|
||||
'test-connection',
|
||||
|
||||
async ({ effects }) => ({
|
||||
name: 'Test Spark Connection',
|
||||
description: 'SSH into the configured Spark(s) and verify GPU + serving/reviewer image access.',
|
||||
description: 'SSH into the configured Spark(s) and verify GPU + serving/grader image access.',
|
||||
warning: null,
|
||||
allowedStatuses: 'any',
|
||||
group: null,
|
||||
group: 'Setup',
|
||||
visibility: 'enabled',
|
||||
}),
|
||||
|
||||
|
||||
Reference in New Issue
Block a user