0.2.25: batch historical eNAV backfill + collapsible investor chart

Add POST /api/import/capital-accounts/batch: upload several of a fund's
eNAV workbooks at once; each file's ALLOC SI roster is auto-matched to
existing members (by fund-admin investor ID, else name/username) and their
capital statement is saved at that file's own as-of date, building
trend-lines without replacing the latest figures. Members not already in
the portal are skipped and reported per file (never created). Capital
statements only -- holdings/NAV are untouched. One bad file (wrong
password, no ALLOC SI, unreadable date) is reported per-file and does not
abort the rest.

Import page gains a "Backfill historical capital" batch section (fund
picker, multi-file .xlsx input, shared password, per-file results table).

Investor portal "Capital over time" chart is now collapsed by default and
expands per fund (first login opens clean); applies to InvestorHome and
the admin Investor View via the shared component.

Tests: backend/tests/test_capital_batch.py (2). Full suite 17 passed;
frontend tsc clean.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Jonathan Kirkwood
2026-07-01 18:47:01 -05:00
co-authored by Claude Opus 4.8
parent 33776d42f4
commit 099459b2f3
10 changed files with 447 additions and 8 deletions
@@ -27,6 +27,7 @@ from ten31portal.models import (
) )
from ten31portal.routers.import_router import _open_workbook, _enav_as_of from ten31portal.routers.import_router import _open_workbook, _enav_as_of
from ten31portal.schemas import ( from ten31portal.schemas import (
BatchCapitalFileResult, BatchCapitalImportResult,
CapitalImportCommit, CapitalImportPreview, ImportInvestorPreview, ImportValueRow, CapitalImportCommit, CapitalImportPreview, ImportInvestorPreview, ImportValueRow,
) )
@@ -346,3 +347,118 @@ def commit_import(
"matched_accounts_updated": updated_accounts, "matched_accounts_updated": updated_accounts,
"statements_written": statements, "statements_written": statements,
} }
@router.post("/batch")
def batch_import(
files: list[UploadFile] = File(...),
entity_id: int = Form(...),
password: str | None = Form(None),
admin: User = Depends(require_internal_admin),
session: Session = Depends(get_session),
) -> BatchCapitalImportResult:
"""Backfill several quarters of capital history for one fund from a batch of eNAV files.
Each file is a full eNAV workbook; its ALLOC SI roster is auto-matched (by fund-admin
investor ID, else name/username) against existing members and each member's capital
statement is upserted at the file's own as-of date — so older files add historical points
without touching the latest. Members not already in the system are skipped and reported
(no account creation). One file failing (bad password, no ALLOC SI, unreadable date) is
reported per-file and does not abort the rest.
"""
entity = session.get(Entity, entity_id)
if entity is None:
raise HTTPException(status_code=404, detail="Entity not found")
investors = session.exec(select(User).where(User.role == UserRole.investor)).all()
by_name = {u.name.strip().lower(): u for u in investors}
by_username = {u.username.strip().lower(): u for u in investors}
by_extid = {u.external_investor_id: u for u in investors if u.external_investor_id}
def match_for(name: str, external_id: str | None) -> User | None:
if external_id and external_id in by_extid:
return by_extid[external_id]
return by_name.get(name.strip().lower()) or by_username.get(name.strip().lower())
cents = lambda d: round((d or 0.0) * 100)
results: list[BatchCapitalFileResult] = []
total_statements = 0
for upload in files:
fname = upload.filename or "(unnamed)"
res = BatchCapitalFileResult(filename=fname)
try:
wb = _open_workbook(upload.file.read(), password)
if "ALLOC SI" not in wb.sheetnames:
raise HTTPException(status_code=422, detail="No ALLOC SI tab found in this workbook.")
as_of, roster = _parse_alloc_si(wb)
if as_of is None:
raise HTTPException(status_code=422, detail="Could not determine the as-of date from the file.")
res.as_of_date = as_of
for inv in roster:
user = match_for(inv["name"], inv["external_id"])
if user is None:
res.skipped.append(inv["name"])
continue
# Record the fund-admin ID on first sighting so later files match by ID too.
if inv["external_id"] and not user.external_investor_id:
user.external_investor_id = inv["external_id"]
by_extid[inv["external_id"]] = user
session.add(user)
has_access = session.exec(
select(EntityAccess).where(
EntityAccess.user_id == user.id, EntityAccess.entity_id == entity_id
)
).first()
if has_access is None:
session.add(EntityAccess(user_id=user.id, entity_id=entity_id))
existing = session.exec(
select(CapitalAccountStatement).where(
CapitalAccountStatement.entity_id == entity_id,
CapitalAccountStatement.investor_user_id == user.id,
CapitalAccountStatement.as_of_date == as_of,
)
).first()
if existing:
existing.commitment_cents = cents(inv["commitment"])
existing.contributions_cents = cents(inv["contributions"])
existing.distributions_cents = cents(inv["distributions"])
existing.ending_balance_cents = cents(inv["ending"])
session.add(existing)
res.updated += 1
else:
session.add(CapitalAccountStatement(
entity_id=entity_id,
investor_user_id=user.id,
as_of_date=as_of,
commitment_cents=cents(inv["commitment"]),
contributions_cents=cents(inv["contributions"]),
distributions_cents=cents(inv["distributions"]),
ending_balance_cents=cents(inv["ending"]),
))
res.matched += 1
res.statements_written += 1
record_audit(session, admin.id, "import_batch", "capital_account", entity_id, {
"file": fname,
"as_of_date": str(as_of),
"matched": res.matched,
"skipped": len(res.skipped),
})
session.commit()
total_statements += res.statements_written
except HTTPException as e:
session.rollback()
res.error = e.detail
except Exception as e: # noqa: BLE001 — surface any parse failure per-file, keep going
session.rollback()
res.error = str(e) or "Could not process this file."
results.append(res)
return BatchCapitalImportResult(
entity_id=entity_id, files=results, total_statements=total_statements
)
+18
View File
@@ -320,6 +320,24 @@ class CapitalImportCommit(BaseModel):
investors: list[ImportCommitInvestor] investors: list[ImportCommitInvestor]
# --- Batch historical capital backfill (one eNAV file per quarter, auto-matched) ---
class BatchCapitalFileResult(BaseModel):
filename: str
as_of_date: date | None = None
matched: int = 0 # existing members whose statement was written
statements_written: int = 0 # created + updated
updated: int = 0 # matched a statement already at this as-of date
skipped: list[str] = [] # roster names with no existing member (not created)
error: str | None = None # file-level failure (bad password, no ALLOC SI, etc.)
class BatchCapitalImportResult(BaseModel):
entity_id: int
files: list[BatchCapitalFileResult]
total_statements: int
# --- Entity stakes (a GP/mgmt entity's interest in the funds it manages) --- # --- Entity stakes (a GP/mgmt entity's interest in the funds it manages) ---
class EntityStakeCreate(BaseModel): class EntityStakeCreate(BaseModel):
+107
View File
@@ -0,0 +1,107 @@
"""Batch historical capital backfill: several eNAV files → per-quarter statements, auto-matched."""
import io
from datetime import date, datetime
import openpyxl
from sqlmodel import select
from ten31portal.models import CapitalAccountStatement, Entity, EntityType, User, UserRole
from tests.conftest import make_user
def _alloc_si_file(report: datetime, rows: list[dict]) -> bytes:
"""Build a minimal eNAV workbook with an ALLOC SI roster and a report date in A1."""
wb = openpyxl.Workbook()
ws = wb.active
ws.title = "ALLOC SI"
ws["A1"] = report # _enav_as_of reads the report date from the top-left cells
header = ["INVESTOR ID", "INVESTOR TYPE", "INVESTOR NAME",
"COMMITTED CAPITAL", "CONTRIBUTIONS", "(DISTRIBUTIONS)", "ENDING BALANCE"]
ws.append([None] * 7) # row 2 spacer
ws.append(header) # row 3 header
for r in rows:
ws.append([
r.get("id"), "LP", r["name"],
r.get("commit", 0), r.get("contrib", 0), r.get("distrib", 0), r["ending"],
])
out = io.BytesIO()
wb.save(out)
return out.getvalue()
def test_batch_backfill_builds_history(auth_client, session):
fund = Entity(name="LTPF III", type=EntityType.fund)
session.add(fund)
session.commit()
session.refresh(fund)
# One member matched by fund-admin ID, one by name; both already exist.
alice = make_user(session, username="alice", name="Alice Trust",
role=UserRole.investor, external_investor_id="INV-100")
make_user(session, username="bob", name="Bob Llc", role=UserRole.investor)
q3 = _alloc_si_file(datetime(2025, 9, 30), [
{"id": "INV-100", "name": "Alice Trust", "commit": 1_000_000, "contrib": 400_000, "ending": 420_000},
{"id": "INV-200", "name": "Bob LLC", "commit": 500_000, "contrib": 200_000, "ending": 205_000},
{"id": "INV-999", "name": "Ghost Capital", "commit": 999, "contrib": 999, "ending": 999}, # no account → skipped
])
q4 = _alloc_si_file(datetime(2025, 12, 31), [
{"id": "INV-100", "name": "Alice Trust", "commit": 1_000_000, "contrib": 400_000, "ending": 455_000},
{"id": "INV-200", "name": "Bob LLC", "commit": 500_000, "contrib": 200_000, "ending": 210_000},
])
resp = auth_client.post(
"/api/import/capital-accounts/batch",
data={"entity_id": fund.id},
files=[
("files", ("LTPF_III_Q3.xlsx", q3, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")),
("files", ("LTPF_III_Q4.xlsx", q4, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")),
],
)
assert resp.status_code == 200, resp.text
body = resp.json()
assert body["total_statements"] == 4 # 2 members × 2 quarters
q3res = next(f for f in body["files"] if "Q3" in f["filename"])
assert q3res["as_of_date"] == "2025-09-30"
assert q3res["matched"] == 2
assert q3res["skipped"] == ["Ghost Capital"] # unknown member reported, not created
# No account was created for the unknown roster name.
assert session.exec(select(User).where(User.name == "Ghost Capital")).first() is None
# Alice now has two statements — a real trend line — and the latest (Q4) is intact.
stmts = session.exec(
select(CapitalAccountStatement)
.where(CapitalAccountStatement.investor_user_id == alice.id)
.order_by(CapitalAccountStatement.as_of_date) # type: ignore[arg-type]
).all()
assert [s.as_of_date for s in stmts] == [date(2025, 9, 30), date(2025, 12, 31)]
assert [s.ending_balance_cents for s in stmts] == [42_000_000, 45_500_000]
def test_batch_reimport_updates_in_place(auth_client, session):
fund = Entity(name="LTPF IV", type=EntityType.fund)
session.add(fund)
session.commit()
session.refresh(fund)
carol = make_user(session, username="carol", name="Carol Ira", role=UserRole.investor)
def one_file(ending: int) -> bytes:
return _alloc_si_file(datetime(2025, 12, 31),
[{"id": "INV-1", "name": "Carol Ira", "ending": ending}])
for ending in (300_000, 315_000): # corrected figure re-imported at the same quarter
resp = auth_client.post(
"/api/import/capital-accounts/batch",
data={"entity_id": fund.id},
files=[("files", ("f.xlsx", one_file(ending), "application/octet-stream"))],
)
assert resp.status_code == 200, resp.text
stmts = session.exec(
select(CapitalAccountStatement).where(CapitalAccountStatement.investor_user_id == carol.id)
).all()
assert len(stmts) == 1 # upsert, not a duplicate
assert stmts[0].ending_balance_cents == 31_500_000
+3 -2
View File
@@ -1,4 +1,4 @@
export { v_0_2_24 as current } from './v_0_2_24' export { v_0_2_25 as current } from './v_0_2_25'
import { v_0_1_0 } from './v_0_1_0' import { v_0_1_0 } from './v_0_1_0'
import { v_0_2_0 } from './v_0_2_0' import { v_0_2_0 } from './v_0_2_0'
import { v_0_2_1 } from './v_0_2_1' import { v_0_2_1 } from './v_0_2_1'
@@ -23,4 +23,5 @@ import { v_0_2_20 } from './v_0_2_20'
import { v_0_2_21 } from './v_0_2_21' import { v_0_2_21 } from './v_0_2_21'
import { v_0_2_22 } from './v_0_2_22' import { v_0_2_22 } from './v_0_2_22'
import { v_0_2_23 } from './v_0_2_23' import { v_0_2_23 } from './v_0_2_23'
export const other = [v_0_1_0, v_0_2_0, v_0_2_1, v_0_2_3, v_0_2_4, v_0_2_5, v_0_2_6, v_0_2_7, v_0_2_8, v_0_2_9, v_0_2_10, v_0_2_11, v_0_2_12, v_0_2_13, v_0_2_14, v_0_2_15, v_0_2_16, v_0_2_17, v_0_2_18, v_0_2_19, v_0_2_20, v_0_2_21, v_0_2_22, v_0_2_23] import { v_0_2_24 } from './v_0_2_24'
export const other = [v_0_1_0, v_0_2_0, v_0_2_1, v_0_2_3, v_0_2_4, v_0_2_5, v_0_2_6, v_0_2_7, v_0_2_8, v_0_2_9, v_0_2_10, v_0_2_11, v_0_2_12, v_0_2_13, v_0_2_14, v_0_2_15, v_0_2_16, v_0_2_17, v_0_2_18, v_0_2_19, v_0_2_20, v_0_2_21, v_0_2_22, v_0_2_23, v_0_2_24]
@@ -0,0 +1,13 @@
import { VersionInfo } from '@start9labs/start-sdk'
export const v_0_2_25 = VersionInfo.of({
version: '0.2.25:0',
releaseNotes: {
en_US:
'Load several past quarters of investor capital at once: the Import page now has a "Backfill historical capital" batch — drop a fund\'s eNAV workbooks and each file\'s members are matched to existing accounts and saved at that file\'s own as-of date, building trend-lines without replacing the latest figures (members not already in the portal are skipped, never created). Investors\' "Capital over time" chart is now collapsed by default and expands per fund.',
},
migrations: {
up: async ({ effects }) => {},
down: async ({ effects }) => {},
},
})
+1 -1
View File
@@ -3,7 +3,7 @@
// - content-hashed /assets/* are cache-first (immutable, safe forever) // - content-hashed /assets/* are cache-first (immutable, safe forever)
// - /api/* is never cached // - /api/* is never cached
// Bump CACHE on each release so old entries are purged. // Bump CACHE on each release so old entries are purged.
const CACHE = 'ten31-portal-0.2.24' const CACHE = 'ten31-portal-0.2.25'
self.addEventListener('install', () => self.skipWaiting()) self.addEventListener('install', () => self.skipWaiting())
+32
View File
@@ -130,6 +130,22 @@ export interface CapitalImportResult {
statements_written: number; statements_written: number;
} }
export interface BatchCapitalFileResult {
filename: string;
as_of_date: string | null;
matched: number;
statements_written: number;
updated: number;
skipped: string[];
error: string | null;
}
export interface BatchCapitalImportResult {
entity_id: number;
files: BatchCapitalFileResult[];
total_statements: number;
}
export interface CapitalAccount { export interface CapitalAccount {
id: number; id: number;
entity_id: number; entity_id: number;
@@ -440,6 +456,22 @@ export const api = {
method: "POST", method: "POST",
body: JSON.stringify(data), body: JSON.stringify(data),
}), }),
// Batch backfill: several historical eNAV files → capital statements per quarter, auto-matched.
capitalImportBatch: async (
entityId: number,
files: File[],
password?: string,
): Promise<BatchCapitalImportResult> => {
const form = new FormData();
form.set("entity_id", String(entityId));
if (password) form.set("password", password);
for (const f of files) form.append("files", f);
const res = await fetch("/api/import/capital-accounts/batch", { method: "POST", body: form });
const data = await res.json().catch(() => ({ detail: res.statusText }));
if (!res.ok) throw new ApiError(res.status, data.detail || "Batch import failed");
return data;
},
getUser: (id: number) => request<UserDetail>(`/api/users/${id}`), getUser: (id: number) => request<UserDetail>(`/api/users/${id}`),
createUser: (data: { createUser: (data: {
name: string; name: string;
+137 -2
View File
@@ -1,7 +1,10 @@
import { useEffect, useState } from "react"; import { useEffect, useState } from "react";
import { api, isAdmin, type Entity, type EntityType, type CapitalImportPreview } from "../api"; import {
api, isAdmin,
type Entity, type EntityType, type CapitalImportPreview, type BatchCapitalImportResult,
} from "../api";
import { useAuth } from "../context/AuthContext"; import { useAuth } from "../context/AuthContext";
import { formatMoneyExact } from "../format"; import { formatDate, formatMoneyExact } from "../format";
type EntityMode = "existing" | "from_file"; type EntityMode = "existing" | "from_file";
@@ -377,6 +380,8 @@ export default function Import() {
</div> </div>
)} )}
{!memberPv && !result && <BatchBackfill entities={entities} />}
{confirmReplace && ( {confirmReplace && (
<div className="fixed inset-0 bg-black/40 flex items-center justify-center p-4 z-50"> <div className="fixed inset-0 bg-black/40 flex items-center justify-center p-4 z-50">
<div className="bg-white rounded-lg p-6 max-w-md w-full"> <div className="bg-white rounded-lg p-6 max-w-md w-full">
@@ -406,3 +411,133 @@ export default function Import() {
</div> </div>
); );
} }
// Backfill several past quarters of investor capital in one shot. Each eNAV file's ALLOC SI
// roster is auto-matched to existing members and their statement is written at the file's own
// as-of date, so older files add trend-line history without disturbing the latest figures.
// Members not already in the system are skipped (never created here) and reported per file.
function BatchBackfill({ entities }: { entities: Entity[] }) {
const [entityId, setEntityId] = useState("");
const [files, setFiles] = useState<File[]>([]);
const [password, setPassword] = useState("");
const [busy, setBusy] = useState(false);
const [error, setError] = useState("");
const [result, setResult] = useState<BatchCapitalImportResult | null>(null);
const inputCls = "w-full px-3 py-2 border border-gray-300 rounded text-sm";
async function run() {
if (!entityId || files.length === 0) {
setError("Pick a fund and at least one eNAV file.");
return;
}
setError("");
setResult(null);
setBusy(true);
try {
setResult(await api.capitalImportBatch(Number(entityId), files, password || undefined));
} catch (e: any) {
setError(e.message || "Batch import failed");
} finally {
setBusy(false);
}
}
return (
<div className="mt-10 border-t border-gray-200 pt-8">
<h2 className="text-lg font-semibold text-gray-900 mb-1">Backfill historical capital</h2>
<p className="text-sm text-gray-500 mb-4 max-w-3xl">
Load several past quarters at once to build investors' trend-lines. Drop the eNAV
workbooks for one fund; each file's members are matched to existing accounts and their
capital statement is saved at that file's own as-of date. The latest figures are never
replaced, and members not already in the portal are skipped (not created).
</p>
{error && <div className="p-3 bg-red-50 border border-red-200 rounded text-sm text-red-700 mb-4">{error}</div>}
<div className="bg-white border border-gray-200 rounded-lg p-4 space-y-3">
<div>
<label className="block text-sm text-gray-700 mb-1">Fund</label>
<select value={entityId} onChange={(e) => setEntityId(e.target.value)} className={inputCls}>
<option value="">Select fund</option>
{entities.map((e) => <option key={e.id} value={e.id}>{e.name}</option>)}
</select>
</div>
<div className="grid grid-cols-2 gap-3">
<div>
<label className="block text-xs text-gray-500 mb-1">eNAV workbooks (.xlsx select several)</label>
<input
type="file"
accept=".xlsx"
multiple
onChange={(e) => { setFiles(Array.from(e.target.files ?? [])); setResult(null); }}
className="text-sm"
/>
{files.length > 0 && <p className="text-xs text-gray-400 mt-1">{files.length} file(s) selected</p>}
</div>
<div>
<label className="block text-xs text-gray-500 mb-1">Spreadsheet password</label>
<input
type="text"
className={inputCls}
value={password}
placeholder="Shared open password, if protected"
autoComplete="off"
onChange={(e) => setPassword(e.target.value)}
/>
</div>
</div>
<div className="flex justify-end">
<button
onClick={run}
disabled={busy || !entityId || files.length === 0}
className="px-3 py-1.5 bg-gray-900 text-white text-sm rounded hover:bg-gray-800 disabled:opacity-50"
>
{busy ? "Loading history…" : "Load history"}
</button>
</div>
</div>
{result && (
<div className="mt-4 bg-white border border-gray-200 rounded-lg overflow-hidden">
<div className="px-4 py-2 bg-gray-50 text-sm text-gray-600 border-b border-gray-100">
{result.total_statements} statement(s) written across {result.files.length} file(s)
</div>
<table className="w-full text-sm">
<thead className="text-gray-500 text-left">
<tr>
<th className="px-4 py-2 font-medium">File</th>
<th className="px-4 py-2 font-medium">As-of</th>
<th className="px-4 py-2 font-medium text-right">Loaded</th>
<th className="px-4 py-2 font-medium">Skipped (no account)</th>
</tr>
</thead>
<tbody>
{result.files.map((f, i) => (
<tr key={i} className="border-t border-gray-100 align-top">
<td className="px-4 py-2 text-gray-900">{f.filename}</td>
<td className="px-4 py-2 text-gray-600">{f.as_of_date ? formatDate(f.as_of_date) : "—"}</td>
<td className="px-4 py-2 text-right text-gray-900">
{f.error ? "—" : f.statements_written}
</td>
<td className="px-4 py-2 text-gray-500">
{f.error ? (
<span className="text-red-600">{f.error}</span>
) : f.skipped.length === 0 ? (
<span className="text-gray-400">none</span>
) : (
<span title={f.skipped.join(", ")}>
{f.skipped.length}: {f.skipped.slice(0, 3).join(", ")}
{f.skipped.length > 3 ? "…" : ""}
</span>
)}
</td>
</tr>
))}
</tbody>
</table>
</div>
)}
</div>
);
}
+19 -2
View File
@@ -1,4 +1,4 @@
import { useMemo } from "react"; import { useMemo, useState } from "react";
import { api, type CapitalAccount, type Entity, type PortalDocument } from "../api"; import { api, type CapitalAccount, type Entity, type PortalDocument } from "../api";
import { categoryLabel, formatDate, formatMoneyExact } from "../format"; import { categoryLabel, formatDate, formatMoneyExact } from "../format";
import CapitalChart, { type CapitalPoint } from "../components/CapitalChart"; import CapitalChart, { type CapitalPoint } from "../components/CapitalChart";
@@ -125,6 +125,8 @@ function CapitalBlock({
paidIn: a.contributions_cents, paidIn: a.contributions_cents,
distributions: a.distributions_cents, distributions: a.distributions_cents,
})); }));
// Collapsed by default so the portal opens clean; the investor expands the trend per fund.
const [showChart, setShowChart] = useState(false);
return ( return (
<div className={divider ? "mt-6 pt-5 border-t border-gray-100" : "mt-3"}> <div className={divider ? "mt-6 pt-5 border-t border-gray-100" : "mt-3"}>
@@ -150,7 +152,20 @@ function CapitalBlock({
{history.length > 1 && ( {history.length > 1 && (
<div className="mt-5"> <div className="mt-5">
<h3 className="text-xs font-medium text-gray-500 uppercase mb-2">Capital over time</h3> <button
type="button"
onClick={() => setShowChart((v) => !v)}
aria-expanded={showChart}
className="flex items-center gap-1.5 text-xs font-medium text-gray-500 uppercase hover:text-gray-700"
>
<span className={`transition-transform ${showChart ? "rotate-90" : ""}`}></span>
Capital over time
<span className="normal-case font-normal text-gray-400">
({history.length} quarters{showChart ? "" : " — show"})
</span>
</button>
{showChart && (
<div className="mt-3">
<CapitalChart points={chartPoints} /> <CapitalChart points={chartPoints} />
<table className="w-full text-sm mt-4"> <table className="w-full text-sm mt-4">
<thead className="text-gray-400 text-left"> <thead className="text-gray-400 text-left">
@@ -172,6 +187,8 @@ function CapitalBlock({
))} ))}
</tbody> </tbody>
</table> </table>
</div>
)}
</div> </div>
)} )}
</div> </div>
+1 -1
View File
@@ -1,4 +1,4 @@
// Bumped each release so the running build is visible in the UI. // Bumped each release so the running build is visible in the UI.
// If the number shown in the app doesn't match the installed s9pk version, // If the number shown in the app doesn't match the installed s9pk version,
// the new frontend isn't actually being served. // the new frontend isn't actually being served.
export const APP_VERSION = "0.2.20"; export const APP_VERSION = "0.2.25";