v0.1.8: two-Spark pipeline + kept-warm serving

- Secondary-Spark models now work in air-gapped mode: the LiteLLM proxy is
  dual-homed onto the default bridge (docker network connect) to reach the
  secondary's published vLLM port; grader containers stay on the --internal
  network with zero egress. The head-only enforcement is replaced by a
  secondary-configured check.
- Extraction runs in parallel with grading when the extractor's model and
  every grader model in the wave sit on different Sparks (separate GPUs).
- Keep-warm: single-wave jobs no longer tear the wave down between decks
  (was a ~6-min 31B reload per deck); a kept-warm wave that fails preflight
  is restarted once. Adjudicator reuses the live wave when its model is
  already serving instead of cycling the shared proxy.
- clear_resident_containers (preJobStopContainers) now stops names on every
  configured Spark; health() reports containers on both Sparks.
- Verified with a mocked dry-run of the full job loop (3 decks: one
  bring-up, zero mid-job teardowns, parallel overlap, stale-wave restart).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

startos/actions/configure-grading.ts

@@ -29,10 +29,11 @@ const inputSpec = InputSpec.of({
     name: 'Network Mode',
     description:
       'Air-gapped: graders reach ONLY the on-Spark model proxy — zero internet, ' +
-      'board decks never leave your hardware (models must be pre-pulled into the ' +
-      'Spark HF cache, all on the head Spark). Local services: graders may also ' +
-      'reach LAN services like SearXNG and the second Spark (this network has ' +
-      'egress unless you firewall it).',
+      'board decks never leave your hardware (models must be pre-pulled into ' +
+      'each Spark HF cache; models on either Spark are fine — the proxy routes ' +
+      'to the second Spark over your LAN). Local services: graders may also ' +
+      'reach LAN services like SearXNG (this network has egress unless you ' +
+      'firewall it).',
     default: 'airgapped',
     values: {
       airgapped: 'Air-gapped (no network, recommended)',

startos/file-models/config.ts

@@ -61,8 +61,10 @@ export const configShape = z.object({
       z.object({
         alias: z.string(),
         hfModel: z.string(),
-        // Which Spark serves this model. In `airgapped` network mode all models
-        // must be on the head Spark (see networkMode).
+        // Which Spark serves this model. Secondary-Spark models work in BOTH
+        // network modes (in airgapped mode the proxy is dual-homed onto the
+        // bridge to reach them; graders stay on the internal network). Panel
+        // traffic to a secondary model crosses the LAN between the Sparks.
         spark: z.enum(['primary', 'secondary']).default('primary'),
         port: z.number().int().positive().default(8001),
       }),
@@ -144,9 +146,10 @@ export const configShape = z.object({
   // Confidentiality posture for the grader containers:
   //   'airgapped'      — graders join an --internal Docker network: they can
   //                      reach ONLY the on-Spark model proxy, with zero internet
-  //                      egress. Models must be pre-pulled into the Spark's HF
-  //                      cache (no live download). All models must be on the head
-  //                      Spark. Strongest confidentiality.
+  //                      egress. Models must be pre-pulled into each Spark's HF
+  //                      cache (no live download). Secondary-Spark models are
+  //                      reached by the dual-homed proxy over the Spark-to-Spark
+  //                      LAN. Strongest confidentiality.
   //   'local_services' — graders may also reach configured LAN services
   //                      (e.g. SearXNG) and the second Spark. NOTE: this network
   //                      has egress unless you firewall it — use only when you
@@ -172,9 +175,10 @@ export const configShape = z.object({
   autoRunOnDrop: z.boolean().default(false),
   // Name of the per-job Docker network created on the head Spark.
   networkName: z.string().default('boardroom-net'),
-  // Space/comma-separated docker container names stopped on the head Spark at
-  // the start of every job to free GPU memory (e.g. an always-on vLLM another
-  // service runs). NOT restarted afterwards — their owner reloads them.
+  // Space/comma-separated docker container names stopped on EVERY configured
+  // Spark at the start of every job to free GPU memory (e.g. an always-on vLLM
+  // another service runs); names absent on a Spark are skipped. NOT restarted
+  // afterwards — their owner reloads them.
   preJobStopContainers: z.string().default(''),
 
   // --- Portfolio companies ---

startos/versions/index.ts

@@ -7,9 +7,10 @@ import { v_0_1_4 } from './v_0_1_4'
 import { v_0_1_5 } from './v_0_1_5'
 import { v_0_1_6 } from './v_0_1_6'
 import { v_0_1_7 } from './v_0_1_7'
+import { v_0_1_8 } from './v_0_1_8'
 
 /** The current version MUST be the first argument (`current`). */
 export const versions = VersionGraph.of({
-  current: v_0_1_7,
-  other: [v_0_1_6, v_0_1_5, v_0_1_4, v_0_1_3, v_0_1_2, v_0_1_1, v_0_1_0],
+  current: v_0_1_8,
+  other: [v_0_1_7, v_0_1_6, v_0_1_5, v_0_1_4, v_0_1_3, v_0_1_2, v_0_1_1, v_0_1_0],
 })

startos/versions/v_0_1_8.ts

@@ -0,0 +1,18 @@
+import { VersionInfo } from '@start9labs/start-sdk'
+
+/** Two-Spark pipeline + kept-warm serving. ExVer form `<upstream>:<downstream>`. */
+export const v_0_1_8 = VersionInfo.of({
+  version: '0.1.8:0',
+  releaseNotes:
+    'Two-Spark grading pipeline: models pinned to the secondary Spark now work ' +
+    'in air-gapped mode too (the model proxy is dual-homed onto the bridge to ' +
+    'reach them; grader containers keep zero egress), so the extractor can run ' +
+    'on one Spark while the grading panel runs on the other — and when they sit ' +
+    'on different Sparks, extraction and grading of a deck run in parallel. ' +
+    'Serving is also kept warm across decks: single-wave jobs no longer tear ' +
+    'the vLLMs down between decks (previously a ~6-minute 31B reload per deck), ' +
+    'with an automatic restart if a kept-warm model fails preflight. ' +
+    'preJobStopContainers now stops resident containers on every configured ' +
+    'Spark, and Serving/Job health shows containers on both Sparks.',
+  migrations: {},
+})