From 099459b2f31976eb5b66dcb8b9c6bb804ebe3188 Mon Sep 17 00:00:00 2001 From: Jonathan Kirkwood Date: Wed, 1 Jul 2026 18:47:01 -0500 Subject: [PATCH] 0.2.25: batch historical eNAV backfill + collapsible investor chart Add POST /api/import/capital-accounts/batch: upload several of a fund's eNAV workbooks at once; each file's ALLOC SI roster is auto-matched to existing members (by fund-admin investor ID, else name/username) and their capital statement is saved at that file's own as-of date, building trend-lines without replacing the latest figures. Members not already in the portal are skipped and reported per file (never created). Capital statements only -- holdings/NAV are untouched. One bad file (wrong password, no ALLOC SI, unreadable date) is reported per-file and does not abort the rest. Import page gains a "Backfill historical capital" batch section (fund picker, multi-file .xlsx input, shared password, per-file results table). Investor portal "Capital over time" chart is now collapsed by default and expands per fund (first login opens clean); applies to InvestorHome and the admin Investor View via the shared component. Tests: backend/tests/test_capital_batch.py (2). Full suite 17 passed; frontend tsc clean. Co-Authored-By: Claude Opus 4.8 --- .../routers/capital_import_router.py | 116 +++++++++++++++ backend/ten31portal/schemas.py | 18 +++ backend/tests/test_capital_batch.py | 107 ++++++++++++++ deploy/startos/install/versions/index.ts | 5 +- deploy/startos/install/versions/v_0_2_25.ts | 13 ++ frontend/public/sw.js | 2 +- frontend/src/api.ts | 32 ++++ frontend/src/pages/Import.tsx | 139 +++++++++++++++++- frontend/src/portal/InvestorPortalView.tsx | 21 ++- frontend/src/version.ts | 2 +- 10 files changed, 447 insertions(+), 8 deletions(-) create mode 100644 backend/tests/test_capital_batch.py create mode 100644 deploy/startos/install/versions/v_0_2_25.ts diff --git a/backend/ten31portal/routers/capital_import_router.py b/backend/ten31portal/routers/capital_import_router.py index 25e48a5..b869755 100644 --- a/backend/ten31portal/routers/capital_import_router.py +++ b/backend/ten31portal/routers/capital_import_router.py @@ -27,6 +27,7 @@ from ten31portal.models import ( ) from ten31portal.routers.import_router import _open_workbook, _enav_as_of from ten31portal.schemas import ( + BatchCapitalFileResult, BatchCapitalImportResult, CapitalImportCommit, CapitalImportPreview, ImportInvestorPreview, ImportValueRow, ) @@ -346,3 +347,118 @@ def commit_import( "matched_accounts_updated": updated_accounts, "statements_written": statements, } + + +@router.post("/batch") +def batch_import( + files: list[UploadFile] = File(...), + entity_id: int = Form(...), + password: str | None = Form(None), + admin: User = Depends(require_internal_admin), + session: Session = Depends(get_session), +) -> BatchCapitalImportResult: + """Backfill several quarters of capital history for one fund from a batch of eNAV files. + + Each file is a full eNAV workbook; its ALLOC SI roster is auto-matched (by fund-admin + investor ID, else name/username) against existing members and each member's capital + statement is upserted at the file's own as-of date — so older files add historical points + without touching the latest. Members not already in the system are skipped and reported + (no account creation). One file failing (bad password, no ALLOC SI, unreadable date) is + reported per-file and does not abort the rest. + """ + entity = session.get(Entity, entity_id) + if entity is None: + raise HTTPException(status_code=404, detail="Entity not found") + + investors = session.exec(select(User).where(User.role == UserRole.investor)).all() + by_name = {u.name.strip().lower(): u for u in investors} + by_username = {u.username.strip().lower(): u for u in investors} + by_extid = {u.external_investor_id: u for u in investors if u.external_investor_id} + + def match_for(name: str, external_id: str | None) -> User | None: + if external_id and external_id in by_extid: + return by_extid[external_id] + return by_name.get(name.strip().lower()) or by_username.get(name.strip().lower()) + + cents = lambda d: round((d or 0.0) * 100) + results: list[BatchCapitalFileResult] = [] + total_statements = 0 + + for upload in files: + fname = upload.filename or "(unnamed)" + res = BatchCapitalFileResult(filename=fname) + try: + wb = _open_workbook(upload.file.read(), password) + if "ALLOC SI" not in wb.sheetnames: + raise HTTPException(status_code=422, detail="No ALLOC SI tab found in this workbook.") + as_of, roster = _parse_alloc_si(wb) + if as_of is None: + raise HTTPException(status_code=422, detail="Could not determine the as-of date from the file.") + res.as_of_date = as_of + + for inv in roster: + user = match_for(inv["name"], inv["external_id"]) + if user is None: + res.skipped.append(inv["name"]) + continue + + # Record the fund-admin ID on first sighting so later files match by ID too. + if inv["external_id"] and not user.external_investor_id: + user.external_investor_id = inv["external_id"] + by_extid[inv["external_id"]] = user + session.add(user) + + has_access = session.exec( + select(EntityAccess).where( + EntityAccess.user_id == user.id, EntityAccess.entity_id == entity_id + ) + ).first() + if has_access is None: + session.add(EntityAccess(user_id=user.id, entity_id=entity_id)) + + existing = session.exec( + select(CapitalAccountStatement).where( + CapitalAccountStatement.entity_id == entity_id, + CapitalAccountStatement.investor_user_id == user.id, + CapitalAccountStatement.as_of_date == as_of, + ) + ).first() + if existing: + existing.commitment_cents = cents(inv["commitment"]) + existing.contributions_cents = cents(inv["contributions"]) + existing.distributions_cents = cents(inv["distributions"]) + existing.ending_balance_cents = cents(inv["ending"]) + session.add(existing) + res.updated += 1 + else: + session.add(CapitalAccountStatement( + entity_id=entity_id, + investor_user_id=user.id, + as_of_date=as_of, + commitment_cents=cents(inv["commitment"]), + contributions_cents=cents(inv["contributions"]), + distributions_cents=cents(inv["distributions"]), + ending_balance_cents=cents(inv["ending"]), + )) + res.matched += 1 + res.statements_written += 1 + + record_audit(session, admin.id, "import_batch", "capital_account", entity_id, { + "file": fname, + "as_of_date": str(as_of), + "matched": res.matched, + "skipped": len(res.skipped), + }) + session.commit() + total_statements += res.statements_written + except HTTPException as e: + session.rollback() + res.error = e.detail + except Exception as e: # noqa: BLE001 — surface any parse failure per-file, keep going + session.rollback() + res.error = str(e) or "Could not process this file." + results.append(res) + + return BatchCapitalImportResult( + entity_id=entity_id, files=results, total_statements=total_statements + ) diff --git a/backend/ten31portal/schemas.py b/backend/ten31portal/schemas.py index 62c901a..86f5195 100644 --- a/backend/ten31portal/schemas.py +++ b/backend/ten31portal/schemas.py @@ -320,6 +320,24 @@ class CapitalImportCommit(BaseModel): investors: list[ImportCommitInvestor] +# --- Batch historical capital backfill (one eNAV file per quarter, auto-matched) --- + +class BatchCapitalFileResult(BaseModel): + filename: str + as_of_date: date | None = None + matched: int = 0 # existing members whose statement was written + statements_written: int = 0 # created + updated + updated: int = 0 # matched a statement already at this as-of date + skipped: list[str] = [] # roster names with no existing member (not created) + error: str | None = None # file-level failure (bad password, no ALLOC SI, etc.) + + +class BatchCapitalImportResult(BaseModel): + entity_id: int + files: list[BatchCapitalFileResult] + total_statements: int + + # --- Entity stakes (a GP/mgmt entity's interest in the funds it manages) --- class EntityStakeCreate(BaseModel): diff --git a/backend/tests/test_capital_batch.py b/backend/tests/test_capital_batch.py new file mode 100644 index 0000000..95945df --- /dev/null +++ b/backend/tests/test_capital_batch.py @@ -0,0 +1,107 @@ +"""Batch historical capital backfill: several eNAV files → per-quarter statements, auto-matched.""" + +import io +from datetime import date, datetime + +import openpyxl +from sqlmodel import select + +from ten31portal.models import CapitalAccountStatement, Entity, EntityType, User, UserRole +from tests.conftest import make_user + + +def _alloc_si_file(report: datetime, rows: list[dict]) -> bytes: + """Build a minimal eNAV workbook with an ALLOC SI roster and a report date in A1.""" + wb = openpyxl.Workbook() + ws = wb.active + ws.title = "ALLOC SI" + ws["A1"] = report # _enav_as_of reads the report date from the top-left cells + header = ["INVESTOR ID", "INVESTOR TYPE", "INVESTOR NAME", + "COMMITTED CAPITAL", "CONTRIBUTIONS", "(DISTRIBUTIONS)", "ENDING BALANCE"] + ws.append([None] * 7) # row 2 spacer + ws.append(header) # row 3 header + for r in rows: + ws.append([ + r.get("id"), "LP", r["name"], + r.get("commit", 0), r.get("contrib", 0), r.get("distrib", 0), r["ending"], + ]) + out = io.BytesIO() + wb.save(out) + return out.getvalue() + + +def test_batch_backfill_builds_history(auth_client, session): + fund = Entity(name="LTPF III", type=EntityType.fund) + session.add(fund) + session.commit() + session.refresh(fund) + + # One member matched by fund-admin ID, one by name; both already exist. + alice = make_user(session, username="alice", name="Alice Trust", + role=UserRole.investor, external_investor_id="INV-100") + make_user(session, username="bob", name="Bob Llc", role=UserRole.investor) + + q3 = _alloc_si_file(datetime(2025, 9, 30), [ + {"id": "INV-100", "name": "Alice Trust", "commit": 1_000_000, "contrib": 400_000, "ending": 420_000}, + {"id": "INV-200", "name": "Bob LLC", "commit": 500_000, "contrib": 200_000, "ending": 205_000}, + {"id": "INV-999", "name": "Ghost Capital", "commit": 999, "contrib": 999, "ending": 999}, # no account → skipped + ]) + q4 = _alloc_si_file(datetime(2025, 12, 31), [ + {"id": "INV-100", "name": "Alice Trust", "commit": 1_000_000, "contrib": 400_000, "ending": 455_000}, + {"id": "INV-200", "name": "Bob LLC", "commit": 500_000, "contrib": 200_000, "ending": 210_000}, + ]) + + resp = auth_client.post( + "/api/import/capital-accounts/batch", + data={"entity_id": fund.id}, + files=[ + ("files", ("LTPF_III_Q3.xlsx", q3, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")), + ("files", ("LTPF_III_Q4.xlsx", q4, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")), + ], + ) + assert resp.status_code == 200, resp.text + body = resp.json() + + assert body["total_statements"] == 4 # 2 members × 2 quarters + q3res = next(f for f in body["files"] if "Q3" in f["filename"]) + assert q3res["as_of_date"] == "2025-09-30" + assert q3res["matched"] == 2 + assert q3res["skipped"] == ["Ghost Capital"] # unknown member reported, not created + + # No account was created for the unknown roster name. + assert session.exec(select(User).where(User.name == "Ghost Capital")).first() is None + + # Alice now has two statements — a real trend line — and the latest (Q4) is intact. + stmts = session.exec( + select(CapitalAccountStatement) + .where(CapitalAccountStatement.investor_user_id == alice.id) + .order_by(CapitalAccountStatement.as_of_date) # type: ignore[arg-type] + ).all() + assert [s.as_of_date for s in stmts] == [date(2025, 9, 30), date(2025, 12, 31)] + assert [s.ending_balance_cents for s in stmts] == [42_000_000, 45_500_000] + + +def test_batch_reimport_updates_in_place(auth_client, session): + fund = Entity(name="LTPF IV", type=EntityType.fund) + session.add(fund) + session.commit() + session.refresh(fund) + carol = make_user(session, username="carol", name="Carol Ira", role=UserRole.investor) + + def one_file(ending: int) -> bytes: + return _alloc_si_file(datetime(2025, 12, 31), + [{"id": "INV-1", "name": "Carol Ira", "ending": ending}]) + + for ending in (300_000, 315_000): # corrected figure re-imported at the same quarter + resp = auth_client.post( + "/api/import/capital-accounts/batch", + data={"entity_id": fund.id}, + files=[("files", ("f.xlsx", one_file(ending), "application/octet-stream"))], + ) + assert resp.status_code == 200, resp.text + + stmts = session.exec( + select(CapitalAccountStatement).where(CapitalAccountStatement.investor_user_id == carol.id) + ).all() + assert len(stmts) == 1 # upsert, not a duplicate + assert stmts[0].ending_balance_cents == 31_500_000 diff --git a/deploy/startos/install/versions/index.ts b/deploy/startos/install/versions/index.ts index 28dc87d..2656f5a 100644 --- a/deploy/startos/install/versions/index.ts +++ b/deploy/startos/install/versions/index.ts @@ -1,4 +1,4 @@ -export { v_0_2_24 as current } from './v_0_2_24' +export { v_0_2_25 as current } from './v_0_2_25' import { v_0_1_0 } from './v_0_1_0' import { v_0_2_0 } from './v_0_2_0' import { v_0_2_1 } from './v_0_2_1' @@ -23,4 +23,5 @@ import { v_0_2_20 } from './v_0_2_20' import { v_0_2_21 } from './v_0_2_21' import { v_0_2_22 } from './v_0_2_22' import { v_0_2_23 } from './v_0_2_23' -export const other = [v_0_1_0, v_0_2_0, v_0_2_1, v_0_2_3, v_0_2_4, v_0_2_5, v_0_2_6, v_0_2_7, v_0_2_8, v_0_2_9, v_0_2_10, v_0_2_11, v_0_2_12, v_0_2_13, v_0_2_14, v_0_2_15, v_0_2_16, v_0_2_17, v_0_2_18, v_0_2_19, v_0_2_20, v_0_2_21, v_0_2_22, v_0_2_23] +import { v_0_2_24 } from './v_0_2_24' +export const other = [v_0_1_0, v_0_2_0, v_0_2_1, v_0_2_3, v_0_2_4, v_0_2_5, v_0_2_6, v_0_2_7, v_0_2_8, v_0_2_9, v_0_2_10, v_0_2_11, v_0_2_12, v_0_2_13, v_0_2_14, v_0_2_15, v_0_2_16, v_0_2_17, v_0_2_18, v_0_2_19, v_0_2_20, v_0_2_21, v_0_2_22, v_0_2_23, v_0_2_24] diff --git a/deploy/startos/install/versions/v_0_2_25.ts b/deploy/startos/install/versions/v_0_2_25.ts new file mode 100644 index 0000000..5954894 --- /dev/null +++ b/deploy/startos/install/versions/v_0_2_25.ts @@ -0,0 +1,13 @@ +import { VersionInfo } from '@start9labs/start-sdk' + +export const v_0_2_25 = VersionInfo.of({ + version: '0.2.25:0', + releaseNotes: { + en_US: + 'Load several past quarters of investor capital at once: the Import page now has a "Backfill historical capital" batch — drop a fund\'s eNAV workbooks and each file\'s members are matched to existing accounts and saved at that file\'s own as-of date, building trend-lines without replacing the latest figures (members not already in the portal are skipped, never created). Investors\' "Capital over time" chart is now collapsed by default and expands per fund.', + }, + migrations: { + up: async ({ effects }) => {}, + down: async ({ effects }) => {}, + }, +}) diff --git a/frontend/public/sw.js b/frontend/public/sw.js index 48bf62f..40b9e62 100644 --- a/frontend/public/sw.js +++ b/frontend/public/sw.js @@ -3,7 +3,7 @@ // - content-hashed /assets/* are cache-first (immutable, safe forever) // - /api/* is never cached // Bump CACHE on each release so old entries are purged. -const CACHE = 'ten31-portal-0.2.24' +const CACHE = 'ten31-portal-0.2.25' self.addEventListener('install', () => self.skipWaiting()) diff --git a/frontend/src/api.ts b/frontend/src/api.ts index 6aaf3fe..5d2913e 100644 --- a/frontend/src/api.ts +++ b/frontend/src/api.ts @@ -130,6 +130,22 @@ export interface CapitalImportResult { statements_written: number; } +export interface BatchCapitalFileResult { + filename: string; + as_of_date: string | null; + matched: number; + statements_written: number; + updated: number; + skipped: string[]; + error: string | null; +} + +export interface BatchCapitalImportResult { + entity_id: number; + files: BatchCapitalFileResult[]; + total_statements: number; +} + export interface CapitalAccount { id: number; entity_id: number; @@ -440,6 +456,22 @@ export const api = { method: "POST", body: JSON.stringify(data), }), + + // Batch backfill: several historical eNAV files → capital statements per quarter, auto-matched. + capitalImportBatch: async ( + entityId: number, + files: File[], + password?: string, + ): Promise => { + const form = new FormData(); + form.set("entity_id", String(entityId)); + if (password) form.set("password", password); + for (const f of files) form.append("files", f); + const res = await fetch("/api/import/capital-accounts/batch", { method: "POST", body: form }); + const data = await res.json().catch(() => ({ detail: res.statusText })); + if (!res.ok) throw new ApiError(res.status, data.detail || "Batch import failed"); + return data; + }, getUser: (id: number) => request(`/api/users/${id}`), createUser: (data: { name: string; diff --git a/frontend/src/pages/Import.tsx b/frontend/src/pages/Import.tsx index 8757efd..8510d72 100644 --- a/frontend/src/pages/Import.tsx +++ b/frontend/src/pages/Import.tsx @@ -1,7 +1,10 @@ import { useEffect, useState } from "react"; -import { api, isAdmin, type Entity, type EntityType, type CapitalImportPreview } from "../api"; +import { + api, isAdmin, + type Entity, type EntityType, type CapitalImportPreview, type BatchCapitalImportResult, +} from "../api"; import { useAuth } from "../context/AuthContext"; -import { formatMoneyExact } from "../format"; +import { formatDate, formatMoneyExact } from "../format"; type EntityMode = "existing" | "from_file"; @@ -377,6 +380,8 @@ export default function Import() { )} + {!memberPv && !result && } + {confirmReplace && (
@@ -406,3 +411,133 @@ export default function Import() {
); } + +// Backfill several past quarters of investor capital in one shot. Each eNAV file's ALLOC SI +// roster is auto-matched to existing members and their statement is written at the file's own +// as-of date, so older files add trend-line history without disturbing the latest figures. +// Members not already in the system are skipped (never created here) and reported per file. +function BatchBackfill({ entities }: { entities: Entity[] }) { + const [entityId, setEntityId] = useState(""); + const [files, setFiles] = useState([]); + const [password, setPassword] = useState(""); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(""); + const [result, setResult] = useState(null); + + const inputCls = "w-full px-3 py-2 border border-gray-300 rounded text-sm"; + + async function run() { + if (!entityId || files.length === 0) { + setError("Pick a fund and at least one eNAV file."); + return; + } + setError(""); + setResult(null); + setBusy(true); + try { + setResult(await api.capitalImportBatch(Number(entityId), files, password || undefined)); + } catch (e: any) { + setError(e.message || "Batch import failed"); + } finally { + setBusy(false); + } + } + + return ( +
+

Backfill historical capital

+

+ Load several past quarters at once to build investors' trend-lines. Drop the eNAV + workbooks for one fund; each file's members are matched to existing accounts and their + capital statement is saved at that file's own as-of date. The latest figures are never + replaced, and members not already in the portal are skipped (not created). +

+ + {error &&
{error}
} + +
+
+ + +
+
+
+ + { setFiles(Array.from(e.target.files ?? [])); setResult(null); }} + className="text-sm" + /> + {files.length > 0 &&

{files.length} file(s) selected

} +
+
+ + setPassword(e.target.value)} + /> +
+
+
+ +
+
+ + {result && ( +
+
+ {result.total_statements} statement(s) written across {result.files.length} file(s) +
+ + + + + + + + + + + {result.files.map((f, i) => ( + + + + + + + ))} + +
FileAs-ofLoadedSkipped (no account)
{f.filename}{f.as_of_date ? formatDate(f.as_of_date) : "—"} + {f.error ? "—" : f.statements_written} + + {f.error ? ( + {f.error} + ) : f.skipped.length === 0 ? ( + none + ) : ( + + {f.skipped.length}: {f.skipped.slice(0, 3).join(", ")} + {f.skipped.length > 3 ? "…" : ""} + + )} +
+
+ )} +
+ ); +} diff --git a/frontend/src/portal/InvestorPortalView.tsx b/frontend/src/portal/InvestorPortalView.tsx index 98d30ce..886e5a6 100644 --- a/frontend/src/portal/InvestorPortalView.tsx +++ b/frontend/src/portal/InvestorPortalView.tsx @@ -1,4 +1,4 @@ -import { useMemo } from "react"; +import { useMemo, useState } from "react"; import { api, type CapitalAccount, type Entity, type PortalDocument } from "../api"; import { categoryLabel, formatDate, formatMoneyExact } from "../format"; import CapitalChart, { type CapitalPoint } from "../components/CapitalChart"; @@ -125,6 +125,8 @@ function CapitalBlock({ paidIn: a.contributions_cents, distributions: a.distributions_cents, })); + // Collapsed by default so the portal opens clean; the investor expands the trend per fund. + const [showChart, setShowChart] = useState(false); return (
@@ -150,7 +152,20 @@ function CapitalBlock({ {history.length > 1 && (
-

Capital over time

+ + {showChart && ( +
@@ -172,6 +187,8 @@ function CapitalBlock({ ))}
+
+ )}
)}
diff --git a/frontend/src/version.ts b/frontend/src/version.ts index 96fe5e5..302fb97 100644 --- a/frontend/src/version.ts +++ b/frontend/src/version.ts @@ -1,4 +1,4 @@ // Bumped each release so the running build is visible in the UI. // If the number shown in the app doesn't match the installed s9pk version, // the new frontend isn't actually being served. -export const APP_VERSION = "0.2.20"; +export const APP_VERSION = "0.2.25";