Save in-progress keysat integration and StartOS 0.4 work
Snapshot of the working tree before cleanup. Captures: - Keysat licensing: server/license.js, /api/license/* endpoints in server/index.js, activation modal in public/index.html, embedded Ed25519 issuer key (assets/issuer.pub). - StartOS 0.4 expansion: setApiKey action, version files v0.1.1 through v0.1.15, file-models/config.json.ts, manifest updates. - Self-hosted registry server (startos-registry/). - Build/deploy scripts (bin/bump-version.sh, bin/deploy.sh, vendored yt-dlp binary), .gitignore, .deploy.env.example. - Recent design docs (KEYSAT_INTEGRATION.md, UPGRADE-DESIGN.md) — retained here so they remain recoverable when removed in the follow-up cleanup commit.
This commit is contained in:
Keysat
+285
-96
@@ -9,6 +9,7 @@ import os from "os";
|
||||
import https from "https";
|
||||
import http from "http";
|
||||
import { GoogleGenAI } from "@google/genai";
|
||||
import * as license from "./license.js";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const app = express();
|
||||
@@ -73,11 +74,9 @@ function ytCookieArgs() {
|
||||
return [];
|
||||
}
|
||||
|
||||
// Extra yt-dlp args for robustness (browser impersonation, rate limiting)
|
||||
// Extra yt-dlp args for robustness (rate limiting)
|
||||
function ytExtraArgs() {
|
||||
const args = [];
|
||||
// Browser impersonation helps avoid TLS fingerprint detection on servers
|
||||
args.push("--impersonate", "chrome");
|
||||
// Sleep between requests to avoid rate limiting during batch operations
|
||||
args.push("--sleep-interval", "1", "--max-sleep-interval", "3");
|
||||
return args;
|
||||
@@ -98,6 +97,125 @@ function resolveApiKey(clientKey) {
|
||||
app.use(cors());
|
||||
app.use(express.json({ limit: "100mb" }));
|
||||
|
||||
// ── Keysat licensing (hard-gate / activate-screen flavor) ─────────────────
|
||||
// All /api/* routes return 402 until a valid license is activated, except
|
||||
// the allowlisted endpoints that exist precisely so the frontend can render
|
||||
// an activation UI. See server/license.js for the verifier.
|
||||
let LIC = license.checkLicense();
|
||||
console.log(
|
||||
`[license] state=${LIC.state} entitlements=[${[...LIC.entitlements].join(",")}]` +
|
||||
(LIC.reason ? ` reason=${LIC.reason}` : "")
|
||||
);
|
||||
|
||||
// Endpoints reachable without a license — kept intentionally minimal.
|
||||
const LICENSE_OPEN_PATHS = new Set([
|
||||
"/api/health",
|
||||
"/api/heartbeat",
|
||||
"/api/status",
|
||||
"/api/license-status",
|
||||
"/api/license/activate",
|
||||
"/api/license/deactivate",
|
||||
]);
|
||||
|
||||
// Activation-screen gate: any /api/* request without a valid license is
|
||||
// rejected with 402, except the allowlist above. Non-/api requests
|
||||
// (the static frontend, /assets, etc.) pass through so the UI can load.
|
||||
app.use((req, res, next) => {
|
||||
if (!req.path.startsWith("/api/")) return next();
|
||||
if (LICENSE_OPEN_PATHS.has(req.path)) return next();
|
||||
if (LIC.state === "licensed" && LIC.entitlements.has("core")) return next();
|
||||
return res.status(402).json({
|
||||
error: "license_required",
|
||||
message:
|
||||
LIC.state === "licensed"
|
||||
? "Your license is missing the 'core' entitlement. Contact the seller."
|
||||
: "This service requires a Keysat license. Activate to continue.",
|
||||
state: LIC.state,
|
||||
reason: LIC.reason,
|
||||
activate_url: "/#activate",
|
||||
keysat_base_url: license.KEYSAT_BASE_URL,
|
||||
product_slug: license.PRODUCT_SLUG,
|
||||
});
|
||||
});
|
||||
|
||||
// Pro-tier feature gates. Each entry maps URL prefixes → required
|
||||
// entitlement; first match wins. A licensed user without the right
|
||||
// entitlement gets a clean 402 feature_not_in_tier (vs. the generic
|
||||
// activation gate above).
|
||||
const PRO_FEATURE_GATES = [
|
||||
{
|
||||
prefixes: ["/api/subscriptions", "/api/auto-queue", "/api/sub-check-log"],
|
||||
entitlement: "subscriptions",
|
||||
feature: "subscriptions",
|
||||
message:
|
||||
"Channel subscriptions and auto-queue require a Pro license. Upgrade to unlock.",
|
||||
},
|
||||
{
|
||||
prefixes: ["/api/history"],
|
||||
entitlement: "history",
|
||||
feature: "history",
|
||||
message:
|
||||
"Summary history requires a Pro license. Upgrade to unlock.",
|
||||
},
|
||||
{
|
||||
prefixes: ["/api/library"],
|
||||
entitlement: "library",
|
||||
feature: "library",
|
||||
message:
|
||||
"Library import/export requires a Pro license. Upgrade to unlock.",
|
||||
},
|
||||
];
|
||||
app.use((req, res, next) => {
|
||||
for (const gate of PRO_FEATURE_GATES) {
|
||||
if (gate.prefixes.some((p) => req.path.startsWith(p))) {
|
||||
if (LIC.entitlements.has(gate.entitlement)) return next();
|
||||
return res.status(402).json({
|
||||
error: "feature_not_in_tier",
|
||||
feature: gate.feature,
|
||||
message: gate.message,
|
||||
keysat_base_url: license.KEYSAT_BASE_URL,
|
||||
product_slug: license.PRODUCT_SLUG,
|
||||
});
|
||||
}
|
||||
}
|
||||
next();
|
||||
});
|
||||
|
||||
// License management endpoints — kept open by LICENSE_OPEN_PATHS above.
|
||||
app.get("/api/license-status", (_req, res) => {
|
||||
res.json(license.publicView(LIC));
|
||||
});
|
||||
|
||||
app.post("/api/license/activate", (req, res) => {
|
||||
try {
|
||||
LIC = license.activate(req.body && req.body.license_key);
|
||||
} catch (e) {
|
||||
if (e && e.code === "bad_format") {
|
||||
return res.status(400).json({
|
||||
error: "bad_format",
|
||||
message: "Expected a license key starting with 'LIC1-'.",
|
||||
});
|
||||
}
|
||||
return res.status(500).json({ error: "activation_failed", message: e?.message });
|
||||
}
|
||||
if (LIC.state === "licensed") {
|
||||
return res.json({ ok: true, ...license.publicView(LIC) });
|
||||
}
|
||||
return res.status(400).json({
|
||||
ok: false,
|
||||
error: "invalid",
|
||||
...license.publicView(LIC),
|
||||
});
|
||||
});
|
||||
|
||||
app.post("/api/license/deactivate", async (_req, res) => {
|
||||
try {
|
||||
await fs.unlink(license.LICENSE_PATH).catch(() => {});
|
||||
} catch {}
|
||||
LIC = license.checkLicense();
|
||||
res.json({ ok: true, ...license.publicView(LIC) });
|
||||
});
|
||||
|
||||
// ── History storage ───────────────────────────────────────────────────────
|
||||
|
||||
async function saveToHistory(videoId, url, title, chunks, entries, logs, uploadDate, type) {
|
||||
@@ -401,95 +519,11 @@ app.get("/api/cookies/status", async (req, res) => {
|
||||
res.json(info);
|
||||
});
|
||||
|
||||
// ── OAuth2 YouTube authentication (headless / StartOS) ────────────────────
|
||||
// Uses yt-dlp's built-in OAuth2 device flow: user enters a code on any browser,
|
||||
// token is cached in yt-dlp's cache dir (persisted on /data volume).
|
||||
|
||||
let oauthInProgress = false;
|
||||
let oauthDeviceCode = null;
|
||||
let oauthVerifyUrl = null;
|
||||
|
||||
// Check if OAuth token already exists in yt-dlp cache
|
||||
app.get("/api/auth/oauth/status", async (req, res) => {
|
||||
// yt-dlp stores OAuth tokens in its cache dir
|
||||
const cacheDir = process.env.XDG_CACHE_HOME || path.join(DATA_DIR, "ytdlp-cache");
|
||||
let hasToken = false;
|
||||
try {
|
||||
const files = await fs.readdir(path.join(cacheDir, "yt-dlp"), { recursive: true }).catch(() => []);
|
||||
hasToken = files.some(f => f.includes("oauth") || f.includes("token"));
|
||||
} catch {}
|
||||
|
||||
res.json({
|
||||
hasToken,
|
||||
hasCookies: ytCookiesFileExists,
|
||||
cookieMethod: ytCookieMethod(),
|
||||
inProgress: oauthInProgress,
|
||||
deviceCode: oauthInProgress ? oauthDeviceCode : null,
|
||||
verifyUrl: oauthInProgress ? oauthVerifyUrl : null,
|
||||
});
|
||||
});
|
||||
|
||||
// Initiate OAuth2 device code flow
|
||||
app.post("/api/auth/oauth/start", async (req, res) => {
|
||||
if (oauthInProgress) {
|
||||
return res.json({ ok: true, message: "OAuth flow already in progress", deviceCode: oauthDeviceCode, verifyUrl: oauthVerifyUrl });
|
||||
}
|
||||
|
||||
oauthInProgress = true;
|
||||
oauthDeviceCode = null;
|
||||
oauthVerifyUrl = null;
|
||||
|
||||
res.json({ ok: true, message: "OAuth flow started. Check /api/auth/oauth/status for the device code." });
|
||||
|
||||
// Run yt-dlp in background — it will output the device code to stderr
|
||||
(async () => {
|
||||
try {
|
||||
const cacheDir = path.join(DATA_DIR, "ytdlp-cache");
|
||||
await fs.mkdir(cacheDir, { recursive: true });
|
||||
|
||||
const proc = execFile("yt-dlp", [
|
||||
"--username", "oauth",
|
||||
"--password", "",
|
||||
"--cache-dir", path.join(cacheDir, "yt-dlp"),
|
||||
"--print", "%(title)s",
|
||||
"--no-download",
|
||||
"https://www.youtube.com/watch?v=dQw4w9WgXcQ",
|
||||
], { timeout: 300000, env: { ...process.env, XDG_CACHE_HOME: cacheDir } });
|
||||
|
||||
let stderrBuf = "";
|
||||
proc.stderr?.on("data", (chunk) => {
|
||||
stderrBuf += chunk.toString();
|
||||
// Look for the device code pattern in yt-dlp output
|
||||
const codeMatch = stderrBuf.match(/go to\s+(https?:\/\/\S+)\s+.*?enter.*?code[:\s]+([A-Z0-9-]+)/i);
|
||||
if (codeMatch) {
|
||||
oauthVerifyUrl = codeMatch[1];
|
||||
oauthDeviceCode = codeMatch[2];
|
||||
console.log(` 🔑 OAuth device code: ${oauthDeviceCode} — verify at ${oauthVerifyUrl}`);
|
||||
}
|
||||
});
|
||||
|
||||
await new Promise((resolve, reject) => {
|
||||
proc.on("close", (code) => {
|
||||
if (code === 0) resolve();
|
||||
else reject(new Error(`yt-dlp exited with code ${code}: ${stderrBuf.slice(-300)}`));
|
||||
});
|
||||
proc.on("error", reject);
|
||||
});
|
||||
|
||||
console.log(" ✓ OAuth token cached successfully");
|
||||
} catch (err) {
|
||||
console.error(" ⚠ OAuth flow failed:", err.message);
|
||||
} finally {
|
||||
oauthInProgress = false;
|
||||
}
|
||||
})();
|
||||
});
|
||||
|
||||
// ── History endpoints ─────────────────────────────────────────────────────
|
||||
|
||||
const metaPath = path.join(historyDir, "_meta.json");
|
||||
|
||||
// meta.json structure: { folders: [ { id, name, order, items: [sessionId, ...] } ], uncategorized: [sessionId, ...] }
|
||||
// meta.json structure: { folders: [ { id, name, order, collapsed, items: [sessionId, ...] } ], uncategorized: [sessionId, ...] }
|
||||
async function loadMeta() {
|
||||
try {
|
||||
return JSON.parse(await fs.readFile(metaPath, "utf-8"));
|
||||
@@ -615,7 +649,7 @@ app.put("/api/history/meta", async (req, res) => {
|
||||
app.post("/api/history/folders", async (req, res) => {
|
||||
try {
|
||||
const meta = await loadMeta();
|
||||
const folder = { id: `folder-${Date.now()}`, name: req.body.name || "New Folder", items: [] };
|
||||
const folder = { id: `folder-${Date.now()}`, name: req.body.name || "New Folder", collapsed: false, items: [] };
|
||||
meta.folders.push(folder);
|
||||
await saveMeta(meta);
|
||||
res.json(folder);
|
||||
@@ -638,6 +672,20 @@ app.put("/api/history/folders/:id", async (req, res) => {
|
||||
}
|
||||
});
|
||||
|
||||
// Update a folder's collapsed state
|
||||
app.put("/api/history/folders/:id/collapsed", async (req, res) => {
|
||||
try {
|
||||
const meta = await loadMeta();
|
||||
const folder = meta.folders.find(f => f.id === req.params.id);
|
||||
if (!folder) return res.status(404).json({ error: "Folder not found" });
|
||||
folder.collapsed = !!req.body.collapsed;
|
||||
await saveMeta(meta);
|
||||
res.json({ ok: true, collapsed: folder.collapsed });
|
||||
} catch (err) {
|
||||
res.status(500).json({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Delete a folder (items move to uncategorized)
|
||||
app.delete("/api/history/folders/:id", async (req, res) => {
|
||||
try {
|
||||
@@ -684,6 +732,119 @@ app.put("/api/history/move", async (req, res) => {
|
||||
}
|
||||
});
|
||||
|
||||
// ── Library export/import ────────────────────────────────────────────────
|
||||
|
||||
app.get("/api/library/export", async (req, res) => {
|
||||
try {
|
||||
const meta = await loadMeta();
|
||||
const files = await fs.readdir(historyDir);
|
||||
const sessions = {};
|
||||
for (const file of files) {
|
||||
if (!file.endsWith(".json") || file === "_meta.json" || file === "subscriptions.json" || file === "auto-queue.json") continue;
|
||||
try {
|
||||
const raw = await fs.readFile(path.join(historyDir, file), "utf-8");
|
||||
const id = file.replace(".json", "");
|
||||
sessions[id] = JSON.parse(raw);
|
||||
} catch {}
|
||||
}
|
||||
// Load subscriptions
|
||||
let subscriptions = [];
|
||||
try {
|
||||
subscriptions = JSON.parse(await fs.readFile(path.join(historyDir, "subscriptions.json"), "utf-8")).subscriptions || [];
|
||||
} catch {}
|
||||
|
||||
const exportData = {
|
||||
version: 1,
|
||||
exportedAt: new Date().toISOString(),
|
||||
meta,
|
||||
sessions,
|
||||
subscriptions,
|
||||
};
|
||||
res.setHeader("Content-Type", "application/json");
|
||||
res.setHeader("Content-Disposition", 'attachment; filename="youtube-summarizer-library.json"');
|
||||
res.json(exportData);
|
||||
} catch (err) {
|
||||
res.status(500).json({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
app.post("/api/library/import", express.json({ limit: "200mb" }), async (req, res) => {
|
||||
try {
|
||||
const data = req.body;
|
||||
if (!data || !data.sessions) {
|
||||
return res.status(400).json({ error: "Invalid library file — missing sessions data" });
|
||||
}
|
||||
|
||||
let imported = 0;
|
||||
let skipped = 0;
|
||||
|
||||
// Import sessions
|
||||
for (const [id, session] of Object.entries(data.sessions)) {
|
||||
const filePath = path.join(historyDir, `${id}.json`);
|
||||
// Skip if session already exists (don't overwrite)
|
||||
try {
|
||||
await fs.access(filePath);
|
||||
skipped++;
|
||||
continue;
|
||||
} catch {}
|
||||
await fs.writeFile(filePath, JSON.stringify(session));
|
||||
imported++;
|
||||
}
|
||||
|
||||
// Merge meta (add imported sessions to uncategorized if not already placed)
|
||||
if (data.meta) {
|
||||
const existingMeta = await loadMeta();
|
||||
const allExistingIds = new Set([
|
||||
...existingMeta.uncategorized,
|
||||
...existingMeta.folders.flatMap(f => f.items),
|
||||
]);
|
||||
|
||||
// Import folders that don't exist
|
||||
if (data.meta.folders) {
|
||||
for (const folder of data.meta.folders) {
|
||||
const existingFolder = existingMeta.folders.find(f => f.id === folder.id);
|
||||
if (!existingFolder) {
|
||||
existingMeta.folders.push(folder);
|
||||
folder.items.forEach(id => allExistingIds.add(id));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Add any uncategorized items that aren't already placed
|
||||
if (data.meta.uncategorized) {
|
||||
for (const id of data.meta.uncategorized) {
|
||||
if (!allExistingIds.has(id)) {
|
||||
existingMeta.uncategorized.unshift(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await saveMeta(existingMeta);
|
||||
}
|
||||
|
||||
// Import subscriptions (merge, don't duplicate)
|
||||
if (data.subscriptions && data.subscriptions.length > 0) {
|
||||
let existingSubs = [];
|
||||
try {
|
||||
existingSubs = JSON.parse(await fs.readFile(path.join(historyDir, "subscriptions.json"), "utf-8")).subscriptions || [];
|
||||
} catch {}
|
||||
const existingUrls = new Set(existingSubs.map(s => s.url));
|
||||
let subsAdded = 0;
|
||||
for (const sub of data.subscriptions) {
|
||||
if (!existingUrls.has(sub.url)) {
|
||||
existingSubs.push(sub);
|
||||
subsAdded++;
|
||||
}
|
||||
}
|
||||
await fs.writeFile(path.join(historyDir, "subscriptions.json"), JSON.stringify({ subscriptions: existingSubs }));
|
||||
}
|
||||
|
||||
res.json({ ok: true, imported, skipped, total: Object.keys(data.sessions).length });
|
||||
} catch (err) {
|
||||
res.status(500).json({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
// ── Subscriptions ─────────────────────────────────────────────────────────
|
||||
|
||||
const subsPath = path.join(historyDir, "subscriptions.json");
|
||||
@@ -949,6 +1110,20 @@ async function fetchChannelName(url) {
|
||||
return "Unknown Channel";
|
||||
}
|
||||
|
||||
// ── In-process write serialization ────────────────────────────────────────
|
||||
// Per-file promise chain to prevent lost-update races on read-modify-write
|
||||
// state files (skip-list, seen-list). Without this, two concurrent DELETE
|
||||
// handlers can each load the same snapshot, add their own id, and the
|
||||
// second write overwrites the first — silently dropping entries.
|
||||
const _fileLocks = new Map();
|
||||
function withFileLock(key, fn) {
|
||||
const prev = _fileLocks.get(key) || Promise.resolve();
|
||||
const next = prev.then(fn, fn); // run fn whether prev resolved or rejected
|
||||
// Keep the chain alive but don't leak errors to the next caller
|
||||
_fileLocks.set(key, next.catch(() => {}));
|
||||
return next;
|
||||
}
|
||||
|
||||
// Skip list — videos deleted from history that subscriptions should not re-add
|
||||
const skipPath = path.join(historyDir, "skip-list.json");
|
||||
|
||||
@@ -961,9 +1136,11 @@ async function loadSkipList() {
|
||||
}
|
||||
|
||||
async function addToSkipList(videoId) {
|
||||
const skipIds = await loadSkipList();
|
||||
skipIds.add(videoId);
|
||||
await fs.writeFile(skipPath, JSON.stringify({ videoIds: [...skipIds] }));
|
||||
return withFileLock(skipPath, async () => {
|
||||
const skipIds = await loadSkipList();
|
||||
skipIds.add(videoId);
|
||||
await fs.writeFile(skipPath, JSON.stringify({ videoIds: [...skipIds] }));
|
||||
});
|
||||
}
|
||||
|
||||
// Seen list — videos already offered for approval (persists across restarts)
|
||||
@@ -978,9 +1155,11 @@ async function loadSeenList() {
|
||||
}
|
||||
|
||||
async function addToSeenList(videoIds) {
|
||||
const seen = await loadSeenList();
|
||||
for (const id of videoIds) seen.add(id);
|
||||
await fs.writeFile(seenPath, JSON.stringify({ videoIds: [...seen] }));
|
||||
return withFileLock(seenPath, async () => {
|
||||
const seen = await loadSeenList();
|
||||
for (const id of videoIds) seen.add(id);
|
||||
await fs.writeFile(seenPath, JSON.stringify({ videoIds: [...seen] }));
|
||||
});
|
||||
}
|
||||
|
||||
// Get all videoIds already processed in history
|
||||
@@ -1236,6 +1415,13 @@ async function checkSubscriptions() {
|
||||
}
|
||||
|
||||
async function _checkSubscriptionsInner() {
|
||||
// Pro-tier feature: skip silently when not entitled. The HTTP gate above
|
||||
// returns 402 to callers; this guards the background timer + manual paths.
|
||||
if (!LIC.entitlements.has("subscriptions")) {
|
||||
subCheckLog = [];
|
||||
subLog("Skipped: subscriptions require a Pro license.");
|
||||
return;
|
||||
}
|
||||
subCheckLog = []; // Clear logs for fresh check
|
||||
const subs = await loadSubscriptions();
|
||||
if (subs.length === 0) { subLog("No subscriptions found"); return; }
|
||||
@@ -1503,7 +1689,7 @@ function channelKeyFromUrl(url) {
|
||||
}
|
||||
|
||||
app.post("/api/subscriptions", async (req, res) => {
|
||||
const { url, since, type } = req.body;
|
||||
const { url, since, type, autoDownload } = req.body;
|
||||
if (!url) return res.status(400).json({ error: "Missing url" });
|
||||
|
||||
const isPodcast = type === "podcast" || isPodcastFeedUrl(url);
|
||||
@@ -1534,6 +1720,7 @@ app.post("/api/subscriptions", async (req, res) => {
|
||||
createdAt: cutoff,
|
||||
lastChecked: null,
|
||||
paused: false,
|
||||
autoDownload: autoDownload === true,
|
||||
};
|
||||
subs.push(sub);
|
||||
await saveSubscriptions(subs);
|
||||
@@ -1762,6 +1949,8 @@ app.get("/api/processing/log", (req, res) => {
|
||||
|
||||
app.post("/api/process", async (req, res) => {
|
||||
const { url, apiKey: clientKey, model, type: itemType, title: itemTitle, uploadDate: itemUploadDate, episodeId } = req.body;
|
||||
// BYO Gemini key is a Core-tier feature; the activation gate already
|
||||
// ensures the caller is licensed, so no further check is needed here.
|
||||
const apiKey = resolveApiKey(clientKey);
|
||||
|
||||
if (!url) {
|
||||
|
||||
Reference in New Issue
Block a user