v0.1.8: two-Spark pipeline + kept-warm serving

- Secondary-Spark models now work in air-gapped mode: the LiteLLM proxy is
  dual-homed onto the default bridge (docker network connect) to reach the
  secondary's published vLLM port; grader containers stay on the --internal
  network with zero egress. The head-only enforcement is replaced by a
  secondary-configured check.
- Extraction runs in parallel with grading when the extractor's model and
  every grader model in the wave sit on different Sparks (separate GPUs).
- Keep-warm: single-wave jobs no longer tear the wave down between decks
  (was a ~6-min 31B reload per deck); a kept-warm wave that fails preflight
  is restarted once. Adjudicator reuses the live wave when its model is
  already serving instead of cycling the shared proxy.
- clear_resident_containers (preJobStopContainers) now stops names on every
  configured Spark; health() reports containers on both Sparks.
- Verified with a mocked dry-run of the full job loop (3 decks: one
  bring-up, zero mid-job teardowns, parallel overlap, stale-wave restart).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

orchestrator/serving.py

@@ -15,9 +15,11 @@ Network topology (the confidentiality boundary):
   * Head-Spark vLLMs join this network; the proxy reaches them by container name
     (bm-vllm-<alias>). Reviewers reach the proxy by name (boardroom-proxy).
   * Second-Spark vLLMs publish a host port; the proxy reaches them over the LAN.
-    This only works in local_services mode (an --internal network can't route to
-    the LAN), so airgapped jobs must keep all models on the head Spark — enforced
-    in preflight.
+    In airgapped mode the per-job network is --internal (no LAN route), so the
+    proxy gets a second leg on the default bridge (docker network connect) —
+    the proxy can then reach the secondary Spark while the grader containers
+    stay internal-only with zero egress. Panel traffic to a secondary-Spark
+    model crosses the LAN between the two Sparks in plaintext HTTP.
 
 Models are served from a pre-populated HF cache mounted from the Spark work dir,
 so airgapped serving needs no live download.
@@ -67,18 +69,19 @@ def remove_network(cfg: dict, log) -> None:
 
 # ---------------------------------------------------------- resident models
 def clear_resident_containers(cfg: dict, log) -> None:
-    """Stop the user-listed containers on the head Spark so a grading job gets
-    the GPU to itself (both Sparks normally run an always-on 31B vLLM).
+    """Stop the user-listed containers on EVERY configured Spark so a grading
+    job gets the GPUs to itself (both Sparks normally run an always-on 31B
+    vLLM). Names that don't exist on a given Spark are silently skipped.
 
     Deliberately NOT restarted after the job: whatever owns them is responsible
     for bringing them back (the Gazette's Fleet job reloads its own models)."""
     names = (cfg.get("preJobStopContainers") or "").replace(",", " ").split()
     if not names:
         return
-    head = sc.head(cfg)
     quoted = " ".join(shlex.quote(n) for n in names)
-    log(f"[serving] freeing the GPU on {head.host}: docker stop {' '.join(names)}")
-    sc.run(head, f"docker stop {quoted} 2>/dev/null; true", timeout=180)
+    for sp in sc.sparks(cfg):
+        log(f"[serving] freeing the GPU on {sp.host}: docker stop {' '.join(names)}")
+        sc.run(sp, f"docker stop {quoted} 2>/dev/null; true", timeout=180)
 
 
 # ------------------------------------------------------------------ vLLM
@@ -173,6 +176,15 @@ def bring_up_wave(cfg: dict, wave: list[dict], hf_token: str | None, log) -> Non
     if r.returncode != 0:
         raise RuntimeError(f"LiteLLM router launch failed: {r.stderr or r.stdout}")
 
+    if cfg.get("networkMode") == "airgapped" and \
+            any(m.get("spark") == "secondary" for m in wave):
+        # The --internal per-job network can't route to the LAN. Give the proxy
+        # a second leg on the default bridge so it can reach the secondary
+        # Spark's published vLLM port. The grader containers stay internal-only.
+        r = sc.run(head, f"docker network connect bridge {PROXY_NAME} 2>/dev/null; true",
+                   timeout=30)
+        log("[serving] proxy dual-homed onto the bridge (secondary-Spark routing)")
+
 
 def tear_down_wave(cfg: dict, wave: list[dict], log) -> None:
     names = " ".join(_vllm_name(m["alias"]) for m in wave)
@@ -226,7 +238,10 @@ def reviewer_proxy_base(cfg: dict) -> str:
 
 
 def health(cfg: dict) -> dict:
-    head = sc.head(cfg)
-    r = sc.run(head, "docker ps --filter name=bm-vllm- --filter name=boardroom-proxy "
-                     "--format '{{.Names}} {{.Status}}'", timeout=30)
-    return {"running": (r.stdout or "").strip().splitlines()}
+    """Serving containers across ALL configured Sparks (vLLMs may sit on either)."""
+    running: list[str] = []
+    for sp in sc.sparks(cfg):
+        r = sc.run(sp, "docker ps --filter name=bm-vllm- --filter name=boardroom-proxy "
+                       "--format '{{.Names}} {{.Status}}'", timeout=30)
+        running += (r.stdout or "").strip().splitlines()
+    return {"running": running}