Initial commit: Ten31 Signal Engine (ingest, scoring brain, corpus seeds)

signal_engine/extract/backends.py

@@ -0,0 +1,64 @@
+"""Pluggable extraction backends (§scaling).
+
+The §4.2 extractor calls a backend that turns chat messages into a JSON string. Default is the
+LOCAL Qwen via Spark Control (the ~95%-local design). The Gemini backend is the documented
+overflow/fallback for bulk back-cataloging at scale, or if the Sparks are unavailable — used for
+the PUBLIC corpus only, never conviction/exposure data (sovereignty boundary, §4.6).
+
+A backend exposes: complete_json(messages, max_tokens) -> str  (a JSON object string).
+"""
+from __future__ import annotations
+
+import logging
+
+log = logging.getLogger(__name__)
+
+
+class LocalQwenBackend:
+    name = "local"
+
+    def __init__(self, sc) -> None:
+        self.sc = sc
+
+    def complete_json(self, messages: list[dict], *, max_tokens: int = 4000) -> str:
+        resp = self.sc.chat(messages, json_object=True, temperature=0,
+                            enable_thinking=False, max_tokens=max_tokens)
+        return resp["choices"][0]["message"]["content"]
+
+
+class GeminiBackend:
+    """Gemini fallback/overflow. Implemented against the `google-genai` SDK. NOTE: untested until a
+    key is provided — validate end-to-end before relying on it for a real backfill. The async BATCH
+    API is the eventual scale path; this synchronous form is the drop-in fallback."""
+    name = "gemini"
+
+    def __init__(self, api_key: str, model: str = "gemini-2.5-flash") -> None:
+        from google import genai  # guarded import; pip install google-genai
+        self._genai = genai
+        self.client = genai.Client(api_key=api_key)
+        self.model = model
+
+    def complete_json(self, messages: list[dict], *, max_tokens: int = 4000) -> str:
+        from google.genai import types
+        system = "\n\n".join(m["content"] for m in messages if m["role"] == "system")
+        user = "\n\n".join(m["content"] for m in messages if m["role"] != "system")
+        resp = self.client.models.generate_content(
+            model=self.model,
+            contents=user,
+            config=types.GenerateContentConfig(
+                system_instruction=system or None,
+                temperature=0,
+                max_output_tokens=max_tokens,
+                response_mime_type="application/json",
+            ),
+        )
+        return resp.text or "{}"
+
+
+def from_config(cfg, sc) -> "LocalQwenBackend | GeminiBackend":
+    if cfg.extraction_backend == "gemini":
+        if not cfg.gemini_api_key:
+            log.warning("EXTRACTION_BACKEND=gemini but GEMINI_API_KEY missing — falling back to local")
+        else:
+            return GeminiBackend(cfg.gemini_api_key, cfg.gemini_model)
+    return LocalQwenBackend(sc)