From 85cb64104421869a4333e74e13979864c07ac809 Mon Sep 17 00:00:00 2001
From: Keysat <licensing@keysat.xyz>
Date: Fri, 8 May 2026 17:09:10 -0500
Subject: [PATCH] =?UTF-8?q?Module=20split:=20history=20storage=20+=20meta?=
 =?UTF-8?q?=20+=209=20routes=20=E2=86=92=20server/history.js?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

  • initHistory({ dataDir })          — boot setup; mkdir + path init
  • saveToHistory(...)                — write one summary file
  • loadMeta() / saveMeta(meta)       — _meta.json folder structure
  • setupHistoryRoutes(app, deps)     — registers GET /api/history,
                                        GET/PUT/DELETE /api/history/:id,
                                        PUT /api/history/:id/title,
                                        PUT /api/history/meta,
                                        POST/PUT/DELETE /api/history/folders[/:id],
                                        PUT /api/history/folders/:id/collapsed,
                                        PUT /api/history/move
  • getHistoryDir()                   — exposes the directory for code
                                        that hasn't been extracted yet
                                        (subscriptions / library / process
                                        pipeline)

The DELETE route needs to add the deleted videoId to the skip list so
subscriptions don't re-queue it. That's a cross-module concern, so
setupHistoryRoutes takes addToSkipList as a dependency. For now it's
late-bound to the still-local function in index.js (lambda captures
the scope, not the value); when subscriptions are extracted next, the
import flips cleanly.

server/index.js: 2300 → 2079 lines.

Smoke tested: /api/license-status, /api/health, and /api/history (402
license-gated for unlicensed) all respond as expected.
---
 server/history.js | 275 ++++++++++++++++++++++++++++++++++++++++++++++
 server/index.js   | 249 +++--------------------------------------
 2 files changed, 289 insertions(+), 235 deletions(-)
 create mode 100644 server/history.js

diff --git a/server/history.js b/server/history.js
new file mode 100644
index 0000000..8bea81e
--- /dev/null
+++ b/server/history.js
@@ -0,0 +1,275 @@
+// History storage + routes. Sessions are written as one JSON file per
+// summary in /data/history/<id>.json. Folder structure / ordering lives
+// in a sidecar `_meta.json`.
+//
+// Module-private state: just the historyDir path, set by initHistory().
+// The DELETE route needs to add the deleted videoId to the skip list
+// (so subscriptions don't re-queue it) — that's a cross-module concern,
+// so it's injected as a callback by setupHistoryRoutes.
+
+import fs from "fs/promises";
+import path from "path";
+
+let historyDir = null;
+let metaPath = null;
+
+// ── Initialization ──────────────────────────────────────────────────────────
+// Call once at boot. Creates the directory and stores the path. Idempotent.
+export async function initHistory({ dataDir }) {
+  historyDir = path.join(dataDir, "history");
+  metaPath = path.join(historyDir, "_meta.json");
+  await fs.mkdir(historyDir, { recursive: true }).catch(() => {});
+}
+
+// ── Storage ─────────────────────────────────────────────────────────────────
+// saveToHistory persists a completed summary. Returns the generated id.
+// Used by /api/process. The id encodes the timestamp + a content hint
+// (videoId for YouTube, base64-truncated guid/url for podcasts) so files
+// sort chronologically by name.
+export async function saveToHistory(videoId, url, title, chunks, entries, logs, uploadDate, type) {
+  const idSuffix = type === "podcast"
+    ? Buffer.from(videoId).toString("base64url").slice(0, 16)
+    : videoId;
+  const id = `${Date.now()}-${idSuffix}`;
+  const record = {
+    id,
+    videoId,
+    url,
+    title: title || "Untitled",
+    type: type || "youtube",
+    topicCount: chunks.length,
+    segmentCount: entries.length,
+    createdAt: new Date().toISOString(),
+    uploadDate: uploadDate || "",
+    chunks,
+    entries,
+    logs,
+  };
+  await fs.writeFile(path.join(historyDir, `${id}.json`), JSON.stringify(record));
+  return id;
+}
+
+// ── Meta ────────────────────────────────────────────────────────────────────
+// `_meta.json` shape: { folders: [{ id, name, order, collapsed,
+// items: [sessionId, ...] }], uncategorized: [sessionId, ...] }
+export async function loadMeta() {
+  try {
+    return JSON.parse(await fs.readFile(metaPath, "utf-8"));
+  } catch {
+    return { folders: [], uncategorized: [] };
+  }
+}
+
+export async function saveMeta(meta) {
+  await fs.writeFile(metaPath, JSON.stringify(meta, null, 2));
+}
+
+export function getHistoryDir() {
+  return historyDir;
+}
+
+// ── Routes ──────────────────────────────────────────────────────────────────
+// Pass `addToSkipList` so the DELETE route can suppress re-queueing of
+// videos the user has explicitly removed. Decoupled from subscriptions
+// to keep this module standalone.
+export function setupHistoryRoutes(app, { addToSkipList } = {}) {
+  // Get all history: sessions + folder structure
+  app.get("/api/history", async (req, res) => {
+    try {
+      const files = await fs.readdir(historyDir);
+      const sessionsMap = {};
+      // Skip the meta + state files; everything else is a session.
+      for (const file of files.filter(f =>
+        f.endsWith(".json") &&
+        !f.startsWith("_") &&
+        f !== "subscriptions.json" &&
+        f !== "skip-list.json" &&
+        f !== "seen-list.json" &&
+        f !== "auto-queue.json"
+      )) {
+        try {
+          const raw = await fs.readFile(path.join(historyDir, file), "utf-8");
+          const data = JSON.parse(raw);
+          sessionsMap[data.id] = {
+            id: data.id, videoId: data.videoId, url: data.url,
+            title: data.title, topicCount: data.topicCount,
+            type: data.type || "youtube",
+            segmentCount: data.segmentCount, createdAt: data.createdAt,
+            uploadDate: data.uploadDate || "",
+          };
+        } catch {}
+      }
+
+      const meta = await loadMeta();
+
+      // Clean up: remove references to deleted sessions
+      for (const folder of meta.folders) {
+        folder.items = folder.items.filter(id => sessionsMap[id]);
+      }
+      meta.uncategorized = meta.uncategorized.filter(id => sessionsMap[id]);
+
+      // Add any sessions not in meta (newly created)
+      const allReferenced = new Set([
+        ...meta.uncategorized,
+        ...meta.folders.flatMap(f => f.items),
+      ]);
+      const allIds = Object.keys(sessionsMap);
+      const orphans = allIds.filter(id => !allReferenced.has(id))
+        .sort((a, b) => new Date(sessionsMap[b].createdAt) - new Date(sessionsMap[a].createdAt));
+      meta.uncategorized = [...orphans, ...meta.uncategorized];
+
+      await saveMeta(meta);
+      res.json({ sessions: sessionsMap, meta });
+    } catch (err) {
+      res.json({ sessions: {}, meta: { folders: [], uncategorized: [] } });
+    }
+  });
+
+  // Get a single session (full data)
+  app.get("/api/history/:id", async (req, res) => {
+    try {
+      const raw = await fs.readFile(path.join(historyDir, `${req.params.id}.json`), "utf-8");
+      res.json(JSON.parse(raw));
+    } catch {
+      res.status(404).json({ error: "Session not found" });
+    }
+  });
+
+  // Rename a session title
+  app.put("/api/history/:id/title", async (req, res) => {
+    try {
+      const filePath = path.join(historyDir, `${req.params.id}.json`);
+      const raw = await fs.readFile(filePath, "utf-8");
+      const data = JSON.parse(raw);
+      data.title = req.body.title || data.title;
+      await fs.writeFile(filePath, JSON.stringify(data));
+      res.json({ ok: true, title: data.title });
+    } catch {
+      res.status(404).json({ error: "Session not found" });
+    }
+  });
+
+  // Delete a session — also adds the videoId to the skip list so any
+  // subscriptions don't re-queue it.
+  app.delete("/api/history/:id", async (req, res) => {
+    try {
+      const filePath = path.join(historyDir, `${req.params.id}.json`);
+      let videoId = null;
+      try {
+        const raw = await fs.readFile(filePath, "utf-8");
+        videoId = JSON.parse(raw).videoId;
+      } catch {}
+
+      await fs.unlink(filePath);
+
+      if (videoId && typeof addToSkipList === "function") {
+        await addToSkipList(videoId);
+      }
+
+      const meta = await loadMeta();
+      meta.uncategorized = meta.uncategorized.filter(id => id !== req.params.id);
+      for (const folder of meta.folders) {
+        folder.items = folder.items.filter(id => id !== req.params.id);
+      }
+      await saveMeta(meta);
+      res.json({ ok: true });
+    } catch {
+      res.status(404).json({ error: "Session not found" });
+    }
+  });
+
+  // Update meta (folders, ordering) — the frontend sends the full structure
+  app.put("/api/history/meta", async (req, res) => {
+    try {
+      const meta = req.body;
+      await saveMeta(meta);
+      res.json({ ok: true });
+    } catch (err) {
+      res.status(500).json({ error: err.message });
+    }
+  });
+
+  app.post("/api/history/folders", async (req, res) => {
+    try {
+      const meta = await loadMeta();
+      const folder = { id: `folder-${Date.now()}`, name: req.body.name || "New Folder", collapsed: false, items: [] };
+      meta.folders.push(folder);
+      await saveMeta(meta);
+      res.json(folder);
+    } catch (err) {
+      res.status(500).json({ error: err.message });
+    }
+  });
+
+  app.put("/api/history/folders/:id", async (req, res) => {
+    try {
+      const meta = await loadMeta();
+      const folder = meta.folders.find(f => f.id === req.params.id);
+      if (!folder) return res.status(404).json({ error: "Folder not found" });
+      folder.name = req.body.name || folder.name;
+      await saveMeta(meta);
+      res.json(folder);
+    } catch (err) {
+      res.status(500).json({ error: err.message });
+    }
+  });
+
+  app.put("/api/history/folders/:id/collapsed", async (req, res) => {
+    try {
+      const meta = await loadMeta();
+      const folder = meta.folders.find(f => f.id === req.params.id);
+      if (!folder) return res.status(404).json({ error: "Folder not found" });
+      folder.collapsed = !!req.body.collapsed;
+      await saveMeta(meta);
+      res.json({ ok: true, collapsed: folder.collapsed });
+    } catch (err) {
+      res.status(500).json({ error: err.message });
+    }
+  });
+
+  // Delete a folder — items move back to uncategorized
+  app.delete("/api/history/folders/:id", async (req, res) => {
+    try {
+      const meta = await loadMeta();
+      const idx = meta.folders.findIndex(f => f.id === req.params.id);
+      if (idx === -1) return res.status(404).json({ error: "Folder not found" });
+      const [folder] = meta.folders.splice(idx, 1);
+      meta.uncategorized = [...folder.items, ...meta.uncategorized];
+      await saveMeta(meta);
+      res.json({ ok: true });
+    } catch (err) {
+      res.status(500).json({ error: err.message });
+    }
+  });
+
+  // Move a session to a folder (or uncategorized if folderId is null)
+  app.put("/api/history/move", async (req, res) => {
+    try {
+      const { sessionId, folderId, index } = req.body;
+      const meta = await loadMeta();
+
+      // Remove from current location
+      meta.uncategorized = meta.uncategorized.filter(id => id !== sessionId);
+      for (const folder of meta.folders) {
+        folder.items = folder.items.filter(id => id !== sessionId);
+      }
+
+      // Add to new location
+      if (folderId) {
+        const folder = meta.folders.find(f => f.id === folderId);
+        if (folder) {
+          const i = typeof index === "number" ? index : folder.items.length;
+          folder.items.splice(i, 0, sessionId);
+        }
+      } else {
+        const i = typeof index === "number" ? index : 0;
+        meta.uncategorized.splice(i, 0, sessionId);
+      }
+
+      await saveMeta(meta);
+      res.json({ ok: true });
+    } catch (err) {
+      res.status(500).json({ error: err.message });
+    }
+  });
+}
diff --git a/server/index.js b/server/index.js
index 0ff1af8..3be5ccb 100644
--- a/server/index.js
+++ b/server/index.js
@@ -46,6 +46,14 @@ import {
   tryAcquireFreeSlot,
   releaseFreeSlot,
 } from "./license-middleware.js";
+import {
+  initHistory,
+  saveToHistory,
+  loadMeta,
+  saveMeta,
+  setupHistoryRoutes,
+  getHistoryDir,
+} from "./history.js";
 
 const execFileAsync = promisify(execFile);
 const app = express();
@@ -61,6 +69,8 @@ const configDir = path.join(DATA_DIR, "config");
 await fs.mkdir(historyDir, { recursive: true }).catch(() => {});
 await fs.mkdir(configDir, { recursive: true }).catch(() => {});
 
+await initHistory({ dataDir: DATA_DIR });
+
 // API key + live reload moved to ./config.js
 await config.initConfig({ dataDir: DATA_DIR });
 const envPath = config.getEnvPath();
@@ -82,29 +92,10 @@ setupLicenseMiddleware(app);
 setupLicenseRoutes(app);
 startLicenseRefresh();
 
-// ── History storage ───────────────────────────────────────────────────────
-
-async function saveToHistory(videoId, url, title, chunks, entries, logs, uploadDate, type) {
-  // For podcast episodes, videoId might be a long GUID/URL — use a short hash for the filename
-  const idSuffix = type === "podcast" ? Buffer.from(videoId).toString("base64url").slice(0, 16) : videoId;
-  const id = `${Date.now()}-${idSuffix}`;
-  const record = {
-    id,
-    videoId,
-    url,
-    title: title || "Untitled",
-    type: type || "youtube", // "youtube" or "podcast"
-    topicCount: chunks.length,
-    segmentCount: entries.length,
-    createdAt: new Date().toISOString(),
-    uploadDate: uploadDate || "",
-    chunks,
-    entries,
-    logs,
-  };
-  await fs.writeFile(path.join(historyDir, `${id}.json`), JSON.stringify(record));
-  return id;
-}
+// History storage + routes moved to ./history.js
+// (saveToHistory, loadMeta, saveMeta are imported above)
+// Late-bound addToSkipList — defined later in this file for now.
+setupHistoryRoutes(app, { addToSkipList: (id) => addToSkipList(id) });
 
 // Serve the frontend from ../public
 app.use(express.static(path.join(__dirname, "..", "public")));
@@ -165,218 +156,6 @@ app.post("/api/update-ytdlp", async (req, res) => {
 // /api/cookies/* routes registered via setupCookieRoutes (./cookies.js)
 setupCookieRoutes(app);
 
-// ── History endpoints ─────────────────────────────────────────────────────
-
-const metaPath = path.join(historyDir, "_meta.json");
-
-// meta.json structure: { folders: [ { id, name, order, collapsed, items: [sessionId, ...] } ], uncategorized: [sessionId, ...] }
-async function loadMeta() {
-  try {
-    return JSON.parse(await fs.readFile(metaPath, "utf-8"));
-  } catch {
-    return { folders: [], uncategorized: [] };
-  }
-}
-async function saveMeta(meta) {
-  await fs.writeFile(metaPath, JSON.stringify(meta, null, 2));
-}
-
-// Get all history: sessions + folder structure
-app.get("/api/history", async (req, res) => {
-  try {
-    const files = await fs.readdir(historyDir);
-    const sessionsMap = {};
-    for (const file of files.filter(f => f.endsWith(".json") && !f.startsWith("_") && f !== "subscriptions.json" && f !== "skip-list.json" && f !== "seen-list.json" && f !== "auto-queue.json")) {
-      try {
-        const raw = await fs.readFile(path.join(historyDir, file), "utf-8");
-        const data = JSON.parse(raw);
-        sessionsMap[data.id] = {
-          id: data.id, videoId: data.videoId, url: data.url,
-          title: data.title, topicCount: data.topicCount,
-          type: data.type || "youtube",
-          segmentCount: data.segmentCount, createdAt: data.createdAt,
-          uploadDate: data.uploadDate || "",
-        };
-      } catch {}
-    }
-
-    const meta = await loadMeta();
-
-    // Clean up: remove references to deleted sessions
-    for (const folder of meta.folders) {
-      folder.items = folder.items.filter(id => sessionsMap[id]);
-    }
-    meta.uncategorized = meta.uncategorized.filter(id => sessionsMap[id]);
-
-    // Add any sessions not in meta (newly created)
-    const allReferenced = new Set([
-      ...meta.uncategorized,
-      ...meta.folders.flatMap(f => f.items),
-    ]);
-    const allIds = Object.keys(sessionsMap);
-    const orphans = allIds.filter(id => !allReferenced.has(id))
-      .sort((a, b) => new Date(sessionsMap[b].createdAt) - new Date(sessionsMap[a].createdAt));
-    meta.uncategorized = [...orphans, ...meta.uncategorized];
-
-    await saveMeta(meta);
-    res.json({ sessions: sessionsMap, meta });
-  } catch (err) {
-    res.json({ sessions: {}, meta: { folders: [], uncategorized: [] } });
-  }
-});
-
-// Get a single session (full data)
-app.get("/api/history/:id", async (req, res) => {
-  try {
-    const raw = await fs.readFile(path.join(historyDir, `${req.params.id}.json`), "utf-8");
-    res.json(JSON.parse(raw));
-  } catch {
-    res.status(404).json({ error: "Session not found" });
-  }
-});
-
-// Rename a session title
-app.put("/api/history/:id/title", async (req, res) => {
-  try {
-    const filePath = path.join(historyDir, `${req.params.id}.json`);
-    const raw = await fs.readFile(filePath, "utf-8");
-    const data = JSON.parse(raw);
-    data.title = req.body.title || data.title;
-    await fs.writeFile(filePath, JSON.stringify(data));
-    res.json({ ok: true, title: data.title });
-  } catch {
-    res.status(404).json({ error: "Session not found" });
-  }
-});
-
-// Delete a session
-app.delete("/api/history/:id", async (req, res) => {
-  try {
-    // Read the file first to get the videoId for the skip list
-    const filePath = path.join(historyDir, `${req.params.id}.json`);
-    let videoId = null;
-    try {
-      const raw = await fs.readFile(filePath, "utf-8");
-      videoId = JSON.parse(raw).videoId;
-    } catch {}
-
-    await fs.unlink(filePath);
-
-    // Add to skip list so subscriptions don't re-queue it
-    if (videoId) {
-      await addToSkipList(videoId);
-    }
-
-    // Remove from meta
-    const meta = await loadMeta();
-    meta.uncategorized = meta.uncategorized.filter(id => id !== req.params.id);
-    for (const folder of meta.folders) {
-      folder.items = folder.items.filter(id => id !== req.params.id);
-    }
-    await saveMeta(meta);
-    res.json({ ok: true });
-  } catch {
-    res.status(404).json({ error: "Session not found" });
-  }
-});
-
-// Update meta (folders, ordering) — the frontend sends the full structure
-app.put("/api/history/meta", async (req, res) => {
-  try {
-    const meta = req.body;
-    await saveMeta(meta);
-    res.json({ ok: true });
-  } catch (err) {
-    res.status(500).json({ error: err.message });
-  }
-});
-
-// Create a folder
-app.post("/api/history/folders", async (req, res) => {
-  try {
-    const meta = await loadMeta();
-    const folder = { id: `folder-${Date.now()}`, name: req.body.name || "New Folder", collapsed: false, items: [] };
-    meta.folders.push(folder);
-    await saveMeta(meta);
-    res.json(folder);
-  } catch (err) {
-    res.status(500).json({ error: err.message });
-  }
-});
-
-// Rename a folder
-app.put("/api/history/folders/:id", async (req, res) => {
-  try {
-    const meta = await loadMeta();
-    const folder = meta.folders.find(f => f.id === req.params.id);
-    if (!folder) return res.status(404).json({ error: "Folder not found" });
-    folder.name = req.body.name || folder.name;
-    await saveMeta(meta);
-    res.json(folder);
-  } catch (err) {
-    res.status(500).json({ error: err.message });
-  }
-});
-
-// Update a folder's collapsed state
-app.put("/api/history/folders/:id/collapsed", async (req, res) => {
-  try {
-    const meta = await loadMeta();
-    const folder = meta.folders.find(f => f.id === req.params.id);
-    if (!folder) return res.status(404).json({ error: "Folder not found" });
-    folder.collapsed = !!req.body.collapsed;
-    await saveMeta(meta);
-    res.json({ ok: true, collapsed: folder.collapsed });
-  } catch (err) {
-    res.status(500).json({ error: err.message });
-  }
-});
-
-// Delete a folder (items move to uncategorized)
-app.delete("/api/history/folders/:id", async (req, res) => {
-  try {
-    const meta = await loadMeta();
-    const idx = meta.folders.findIndex(f => f.id === req.params.id);
-    if (idx === -1) return res.status(404).json({ error: "Folder not found" });
-    const [folder] = meta.folders.splice(idx, 1);
-    meta.uncategorized = [...folder.items, ...meta.uncategorized];
-    await saveMeta(meta);
-    res.json({ ok: true });
-  } catch (err) {
-    res.status(500).json({ error: err.message });
-  }
-});
-
-// Move a session to a folder (or uncategorized if folderId is null)
-app.put("/api/history/move", async (req, res) => {
-  try {
-    const { sessionId, folderId, index } = req.body;
-    const meta = await loadMeta();
-
-    // Remove from current location
-    meta.uncategorized = meta.uncategorized.filter(id => id !== sessionId);
-    for (const folder of meta.folders) {
-      folder.items = folder.items.filter(id => id !== sessionId);
-    }
-
-    // Add to new location
-    if (folderId) {
-      const folder = meta.folders.find(f => f.id === folderId);
-      if (folder) {
-        const i = typeof index === "number" ? index : folder.items.length;
-        folder.items.splice(i, 0, sessionId);
-      }
-    } else {
-      const i = typeof index === "number" ? index : 0;
-      meta.uncategorized.splice(i, 0, sessionId);
-    }
-
-    await saveMeta(meta);
-    res.json({ ok: true });
-  } catch (err) {
-    res.status(500).json({ error: err.message });
-  }
-});
 
 // ── Library export/import ────────────────────────────────────────────────