From ebafcf19d9a3e40f0832f2bff12968ed1dd24370 Mon Sep 17 00:00:00 2001 From: Jonathan Kirkwood Date: Tue, 11 Aug 2026 12:32:57 -0500 Subject: [PATCH] 0.2.43: historical NAV backfill without touching current holdings The batch history import now also records each quarter's NAV in the fund's valuation history: the old file's HLD rows are matched by issuer and security name against the book as it exists today, matched rows write that quarter's valuations, unmatched rows are counted and reported, and nothing outside the round is created or modified. A manually signed quarter is never overwritten. The single-file wizard automatically takes the same history-only path when the file is older than the fund's newest round. Previously that import would regress position cost basis to the old file's values and resurrect since-exited positions, corrupting the fund's Invested total. --- .../routers/capital_import_router.py | 27 +++- backend/ten31portal/routers/import_router.py | 129 +++++++++++++++ backend/ten31portal/schemas.py | 5 + backend/tests/test_history_nav.py | 152 ++++++++++++++++++ deploy/package.json | 2 +- deploy/startos/install/versions/index.ts | 5 +- deploy/startos/install/versions/v_0_2_43.ts | 18 +++ frontend/public/sw.js | 2 +- frontend/src/api.ts | 4 + frontend/src/pages/Import.tsx | 40 ++++- frontend/src/version.ts | 2 +- 11 files changed, 376 insertions(+), 10 deletions(-) create mode 100644 backend/tests/test_history_nav.py create mode 100644 deploy/startos/install/versions/v_0_2_43.ts diff --git a/backend/ten31portal/routers/capital_import_router.py b/backend/ten31portal/routers/capital_import_router.py index b1ef254..6a8961e 100644 --- a/backend/ten31portal/routers/capital_import_router.py +++ b/backend/ten31portal/routers/capital_import_router.py @@ -25,7 +25,9 @@ from ten31portal.database import get_session from ten31portal.models import ( CapitalAccountStatement, Entity, EntityAccess, User, UserRole, ) -from ten31portal.routers.import_router import _open_workbook, _enav_as_of +from ten31portal.routers.import_router import ( + _open_workbook, _enav_as_of, _parse_schedule_xlsx, upsert_history_round, +) from ten31portal.schemas import ( BatchCapitalFileResult, BatchCapitalImportResult, CapitalImportCommit, CapitalImportPreview, ImportInvestorPreview, ImportValueRow, @@ -433,7 +435,8 @@ def batch_import( fname = upload.filename or "(unnamed)" res = BatchCapitalFileResult(filename=fname) try: - wb = _open_workbook(storage.read_capped(upload), password) + file_bytes = storage.read_capped(upload) + wb = _open_workbook(file_bytes, password) if "ALLOC SI" not in wb.sheetnames: raise HTTPException(status_code=422, detail="No ALLOC SI tab found in this workbook.") as_of, roster = _parse_alloc_si(wb) @@ -488,11 +491,31 @@ def batch_import( res.matched += 1 res.statements_written += 1 + # NAV history leg: record this quarter's valuation round from the file's HLD + # sheet, matched against today's book only (holdings are never modified). An + # HLD problem must not lose the member statements above, so it runs in a + # savepoint and reports per-file. + try: + with session.begin_nested(): + _, _, _, positions_prev, _ = _parse_schedule_xlsx(file_bytes, password) + if positions_prev: + hist = upsert_history_round(entity_id, as_of, positions_prev, admin, session) + res.nav_status = hist["status"] + res.nav_matched = hist["matched"] + res.nav_unmatched = hist["unmatched"] + res.nav_cents = hist["nav_cents"] + else: + res.nav_status = "no-hld" + except Exception: # noqa: BLE001 — the savepoint rolled back; members still land + res.nav_status = "error" + record_audit(session, admin.id, "import_batch", "capital_account", entity_id, { "file": fname, "as_of_date": str(as_of), "matched": res.matched, "skipped": len(res.skipped), + "nav_status": res.nav_status, + "nav_cents": res.nav_cents, }) session.commit() total_statements += res.statements_written diff --git a/backend/ten31portal/routers/import_router.py b/backend/ten31portal/routers/import_router.py index 0f47fa5..a0ab469 100644 --- a/backend/ten31portal/routers/import_router.py +++ b/backend/ten31portal/routers/import_router.py @@ -331,6 +331,101 @@ def dedupe_entity(entity_id: int, session: Session) -> dict[str, int]: return {"removed_holdings": removed_holdings, "removed_positions": removed_positions} +def upsert_history_round( + entity_id: int, as_of: date, positions_preview: list[dict], user: User, session: Session, +) -> dict[str, Any]: + """Record a past quarter's NAV as a valuation round WITHOUT touching current holdings. + + Backfilling an old eNAV through the normal import would regress each position's cost + basis to the old file's values and resurrect positions the fund has since exited. This + path instead matches the file's rows by issuer + security name against the book as it + exists today and only writes that quarter's valuations: matched rows contribute to the + quarter's NAV, unmatched rows are counted and reported, nothing is created or modified + outside the round. A manually signed round for the quarter is left untouched. + """ + existing = session.exec( + select(ValuationRound).where( + ValuationRound.entity_id == entity_id, + ValuationRound.quarter_end == as_of, + ) + ).first() + if existing is not None and not existing.is_seed: + return {"status": "kept-signed", "matched": 0, "unmatched": 0, "nav_cents": 0} + + # Match first, so a file with no recognizable rows never leaves an empty $0 round. + matches: list[tuple[Position, int]] = [] + unmatched = 0 + for pp in positions_preview: + holding = session.exec( + select(Holding).where( + Holding.entity_id == entity_id, + Holding.company_name == pp["company_name"], + ) + ).first() + pos = None + if holding is not None: + pos = session.exec( + select(Position).where( + Position.holding_id == holding.id, + Position.security_name == pp["security_name"], + ) + ).first() + if pos is None: + unmatched += 1 + else: + matches.append((pos, pp["value_cents"] or 0)) + + if not matches: + return {"status": "no-match", "matched": 0, "unmatched": unmatched, "nav_cents": 0} + + round = existing + if round is None: + round = ValuationRound( + entity_id=entity_id, + quarter_end=as_of, + status=RoundStatus.approved, + is_seed=True, + approved_by=user.id, + approved_at=datetime.utcnow(), + ) + session.add(round) + session.flush() + + nav_cents = 0 + seen_position_ids: set[int] = set() + for pos, value_cents in matches: + val = session.exec( + select(Valuation).where( + Valuation.round_id == round.id, + Valuation.position_id == pos.id, + ) + ).first() + if val is None: + session.add(Valuation(round_id=round.id, position_id=pos.id, value_cents=value_cents)) + else: + val.value_cents = value_cents + session.add(val) + nav_cents += value_cents + seen_position_ids.add(pos.id) + + # Re-import of the same quarter: drop valuations for rows no longer in the file. + if existing is not None: + stale_q = select(Valuation).where( + Valuation.round_id == round.id, + col(Valuation.position_id).not_in(seen_position_ids), + ) + for stale_val in session.exec(stale_q).all(): + session.delete(stale_val) + + return { + "status": "updated" if existing is not None else "added", + "matched": len(matches), + "unmatched": unmatched, + "nav_cents": nav_cents, + "round_id": round.id, + } + + @router.post("/schedule") def import_schedule( file: UploadFile = File(...), @@ -450,6 +545,40 @@ def import_schedule( # import's upsert lands on a single clean copy and "Invested" stops double-counting. dedupe_entity(resolved_entity_id, session) + # A file OLDER than the fund's newest round is a history backfill: record the quarter's + # NAV against today's book without touching holdings — a full import here would regress + # cost basis to the old file and resurrect since-exited positions. (Replace mode wiped + # the rounds above, so an explicit replace still takes the full path.) + newest_round = session.exec( + select(ValuationRound) + .where(ValuationRound.entity_id == resolved_entity_id) + .order_by(col(ValuationRound.quarter_end).desc()) + ).first() + if newest_round is not None and newest_round.quarter_end > as_of: + hist = upsert_history_round(resolved_entity_id, as_of, positions_preview, user, session) + record_audit(session, user.id, "import_history", "entity", resolved_entity_id, { + "seed_quarter": str(as_of), **{k: v for k, v in hist.items() if k != "round_id"}, + }) + session.commit() + return { + "committed": True, + "history_only": True, + **hist, + "source_entity_name": source_entity_name, + "entity": { + "resolution": entity_resolution, + "entity_id": resolved_entity_id, + "entity_name": entity.name, + "entity_type": entity.type.value, + }, + "holdings_count": 0, + "positions_count": len(positions_preview), + "positions_created": 0, + "positions_updated": 0, + "seed_quarter": str(as_of), + "errors": errors, + } + # An existing round at this quarter: a NAV re-import should UPDATE it in place (refresh # to the latest file) instead of stacking a second round and doubling the totals. Only # import-created seed rounds are refreshable; a manually-signed valuation round is left diff --git a/backend/ten31portal/schemas.py b/backend/ten31portal/schemas.py index 337333e..b70985d 100644 --- a/backend/ten31portal/schemas.py +++ b/backend/ten31portal/schemas.py @@ -393,6 +393,11 @@ class BatchCapitalFileResult(BaseModel): updated: int = 0 # matched a statement already at this as-of date skipped: list[str] = [] # roster names with no existing member (not created) error: str | None = None # file-level failure (bad password, no ALLOC SI, etc.) + # NAV history leg: the quarter's valuation round written from the file's HLD sheet. + nav_status: str | None = None # added | updated | kept-signed | no-match | no-hld | error + nav_matched: int = 0 # HLD rows matched to positions in today's book + nav_unmatched: int = 0 # HLD rows with no current position (sold/renamed since) + nav_cents: int = 0 # the quarter's NAV as recorded (matched rows only) class BatchCapitalImportResult(BaseModel): diff --git a/backend/tests/test_history_nav.py b/backend/tests/test_history_nav.py new file mode 100644 index 0000000..18f8bd2 --- /dev/null +++ b/backend/tests/test_history_nav.py @@ -0,0 +1,152 @@ +"""NAV history backfill (0.2.43): old eNAV files add past quarters to valuation history +WITHOUT touching current holdings — no cost-basis regression, no resurrected positions.""" + +import io +from datetime import date, datetime + +import openpyxl +from sqlmodel import select + +from ten31portal.models import ( + Entity, EntityType, Holding, Position, RoundStatus, Valuation, ValuationRound, +) + + +def _enav_file(report: datetime, hld_rows: list[tuple], alloc_rows: list[dict] | None = None) -> bytes: + """Minimal eNAV workbook: an HLD sheet (and optionally an ALLOC SI roster). + + hld_rows: (security_name, quantity, cost, value) tuples. + """ + wb = openpyxl.Workbook() + ws = wb.active + ws.title = "HLD" + ws["A1"] = report # _enav_as_of scans the top-left cells for the report date + ws.append([None]) + ws.append(["SECURITY NAME", "QUANTITY", "COST BASIS - BOOK", "MARKET VALUE (BOOK)"]) + for name, qty, cost, value in hld_rows: + ws.append([name, qty, cost, value]) + if alloc_rows is not None: + alloc = wb.create_sheet("ALLOC SI") + alloc.append(["INVESTOR ID", "INVESTOR TYPE", "INVESTOR NAME", + "COMMITTED CAPITAL", "CONTRIBUTIONS", "(DISTRIBUTIONS)", "ENDING BALANCE"]) + for r in alloc_rows: + alloc.append([r.get("id"), "LP", r["name"], + r.get("commit", 0), r.get("contrib", 0), r.get("distrib", 0), r["ending"]]) + out = io.BytesIO() + wb.save(out) + return out.getvalue() + + +def _fund_with_current_book(session): + """A fund holding one position (cost $100) with its NAV already signed for Q1 2026.""" + fund = Entity(name="LTPF I", type=EntityType.fund) + session.add(fund) + session.commit() + session.refresh(fund) + holding = Holding(entity_id=fund.id, company_name="Acme") + session.add(holding) + session.commit() + pos = Position(holding_id=holding.id, security_name="Acme - Series A", + investment_date=date(2023, 1, 1), cost_cents=100_00) + session.add(pos) + session.commit() + session.refresh(pos) + rnd = ValuationRound(entity_id=fund.id, quarter_end=date(2026, 3, 31), + status=RoundStatus.approved, is_seed=True) + session.add(rnd) + session.commit() + session.refresh(rnd) + session.add(Valuation(round_id=rnd.id, position_id=pos.id, value_cents=900_00)) + session.commit() + return fund, pos + + +def test_old_file_becomes_history_round_without_touching_book(auth_client, session): + fund, pos = _fund_with_current_book(session) + old = _enav_file(datetime(2025, 9, 30), [ + ("Acme - Series A", 10, 123.0, 555.0), # matches today's book + ("Ghost - SAFE", 5, 999.0, 999.0), # sold since; must NOT be created + ]) + + resp = auth_client.post( + f"/api/import/schedule?entity_id={fund.id}&commit=true", + files={"file": ("old.xlsx", io.BytesIO(old), "application/octet-stream")}, + ) + assert resp.status_code == 200, resp.text + body = resp.json() + assert body["history_only"] is True + assert body["status"] == "added" + assert body["matched"] == 1 and body["unmatched"] == 1 + + session.expire_all() + # The quarter landed in valuation history with the matched row's value only. + hist_round = session.exec(select(ValuationRound).where( + ValuationRound.entity_id == fund.id, ValuationRound.quarter_end == date(2025, 9, 30) + )).first() + assert hist_round is not None + vals = session.exec(select(Valuation).where(Valuation.round_id == hist_round.id)).all() + assert [(v.position_id, v.value_cents) for v in vals] == [(pos.id, 555_00)] + # Today's book is untouched: cost basis kept, no ghost position resurrected. + assert session.get(Position, pos.id).cost_cents == 100_00 + assert session.exec(select(Holding).where(Holding.company_name == "Ghost")).first() is None + + +def test_signed_round_is_kept(auth_client, session): + fund, pos = _fund_with_current_book(session) + signed = ValuationRound(entity_id=fund.id, quarter_end=date(2025, 9, 30), + status=RoundStatus.approved, is_seed=False) + session.add(signed) + session.commit() + old = _enav_file(datetime(2025, 9, 30), [("Acme - Series A", 10, 123.0, 555.0)]) + + resp = auth_client.post( + f"/api/import/schedule?entity_id={fund.id}&commit=true", + files={"file": ("old.xlsx", io.BytesIO(old), "application/octet-stream")}, + ) + assert resp.status_code == 200, resp.text + assert resp.json()["status"] == "kept-signed" + session.expire_all() + assert session.exec(select(Valuation).join( + ValuationRound, Valuation.round_id == ValuationRound.id + ).where(ValuationRound.quarter_end == date(2025, 9, 30))).all() == [] + + +def test_batch_backfill_also_records_nav(auth_client, session): + from tests.conftest import make_user + from ten31portal.models import CapitalAccountStatement, EntityAccess, UserRole + + fund, pos = _fund_with_current_book(session) + lp = make_user(session, username="lp", name="Alice Trust", role=UserRole.investor) + session.add(EntityAccess(user_id=lp.id, entity_id=fund.id)) + session.commit() + + old = _enav_file( + datetime(2025, 9, 30), + [("Acme - Series A", 10, 123.0, 555.0), ("Ghost - SAFE", 5, 999.0, 999.0)], + alloc_rows=[{"id": "INV-1", "name": "Alice Trust", "commit": 1000, "contrib": 400, "ending": 420}], + ) + + for expected_status in ("added", "updated"): # second pass proves idempotency + resp = auth_client.post( + "/api/import/capital-accounts/batch", + data={"entity_id": str(fund.id)}, + files=[("files", ("old.xlsx", io.BytesIO(old), "application/octet-stream"))], + ) + assert resp.status_code == 200, resp.text + f = resp.json()["files"][0] + assert f["error"] is None and f["skipped"] == [] + assert f["statements_written"] == 1 + assert f["nav_status"] == expected_status + assert f["nav_matched"] == 1 and f["nav_unmatched"] == 1 + assert f["nav_cents"] == 555_00 + + session.expire_all() + stmts = session.exec(select(CapitalAccountStatement).where( + CapitalAccountStatement.entity_id == fund.id + )).all() + assert len(stmts) == 1 # upserted, not duplicated + rounds = session.exec(select(ValuationRound).where( + ValuationRound.entity_id == fund.id, ValuationRound.quarter_end == date(2025, 9, 30) + )).all() + assert len(rounds) == 1 + assert session.get(Position, pos.id).cost_cents == 100_00 diff --git a/deploy/package.json b/deploy/package.json index c28a2ae..01f30d5 100644 --- a/deploy/package.json +++ b/deploy/package.json @@ -1,6 +1,6 @@ { "name": "ten31portal-startos", - "version": "0.2.42", + "version": "0.2.43", "private": true, "scripts": { "build": "npm run check && rm -rf ./javascript && ncc build startos/index.ts -o ./javascript", diff --git a/deploy/startos/install/versions/index.ts b/deploy/startos/install/versions/index.ts index 42f724e..5b712a6 100644 --- a/deploy/startos/install/versions/index.ts +++ b/deploy/startos/install/versions/index.ts @@ -1,7 +1,8 @@ -export { v_0_2_42 as current } from './v_0_2_42' +export { v_0_2_43 as current } from './v_0_2_43' import { v_0_1_0 } from './v_0_1_0' import { v_0_2_40 } from './v_0_2_40' import { v_0_2_41 } from './v_0_2_41' +import { v_0_2_42 } from './v_0_2_42' import { v_0_2_0 } from './v_0_2_0' import { v_0_2_1 } from './v_0_2_1' import { v_0_2_3 } from './v_0_2_3' @@ -41,4 +42,4 @@ import { v_0_2_36 } from './v_0_2_36' import { v_0_2_37 } from './v_0_2_37' import { v_0_2_38 } from './v_0_2_38' import { v_0_2_39 } from './v_0_2_39' -export const other = [v_0_1_0, v_0_2_0, v_0_2_1, v_0_2_3, v_0_2_4, v_0_2_5, v_0_2_6, v_0_2_7, v_0_2_8, v_0_2_9, v_0_2_10, v_0_2_11, v_0_2_12, v_0_2_13, v_0_2_14, v_0_2_15, v_0_2_16, v_0_2_17, v_0_2_18, v_0_2_19, v_0_2_20, v_0_2_21, v_0_2_22, v_0_2_23, v_0_2_24, v_0_2_25, v_0_2_26, v_0_2_27, v_0_2_28, v_0_2_29, v_0_2_30, v_0_2_31, v_0_2_32, v_0_2_33, v_0_2_34, v_0_2_35, v_0_2_36, v_0_2_37, v_0_2_38, v_0_2_39, v_0_2_40, v_0_2_41] +export const other = [v_0_1_0, v_0_2_0, v_0_2_1, v_0_2_3, v_0_2_4, v_0_2_5, v_0_2_6, v_0_2_7, v_0_2_8, v_0_2_9, v_0_2_10, v_0_2_11, v_0_2_12, v_0_2_13, v_0_2_14, v_0_2_15, v_0_2_16, v_0_2_17, v_0_2_18, v_0_2_19, v_0_2_20, v_0_2_21, v_0_2_22, v_0_2_23, v_0_2_24, v_0_2_25, v_0_2_26, v_0_2_27, v_0_2_28, v_0_2_29, v_0_2_30, v_0_2_31, v_0_2_32, v_0_2_33, v_0_2_34, v_0_2_35, v_0_2_36, v_0_2_37, v_0_2_38, v_0_2_39, v_0_2_40, v_0_2_41, v_0_2_42] diff --git a/deploy/startos/install/versions/v_0_2_43.ts b/deploy/startos/install/versions/v_0_2_43.ts new file mode 100644 index 0000000..904e119 --- /dev/null +++ b/deploy/startos/install/versions/v_0_2_43.ts @@ -0,0 +1,18 @@ +import { VersionInfo } from '@start9labs/start-sdk' + +export const v_0_2_43 = VersionInfo.of({ + version: '0.2.43:0', + releaseNotes: { + en_US: + 'Historical NAV backfill: the batch history import now also records each ' + + "quarter's NAV in the fund's valuation history, matched against the current " + + 'book without ever modifying holdings or cost basis. The single-file import ' + + 'automatically treats a file older than the newest quarter the same safe way, ' + + 'fixing a path where importing an old workbook could regress cost basis and ' + + 'resurrect exited positions.', + }, + migrations: { + up: async ({ effects }) => {}, + down: async ({ effects }) => {}, + }, +}) diff --git a/frontend/public/sw.js b/frontend/public/sw.js index bd76c78..09e377d 100644 --- a/frontend/public/sw.js +++ b/frontend/public/sw.js @@ -3,7 +3,7 @@ // - content-hashed /assets/* are cache-first (immutable, safe forever) // - /api/* is never cached // Bump CACHE on each release so old entries are purged. -const CACHE = 'ten31-portal-0.2.42' +const CACHE = 'ten31-portal-0.2.43' self.addEventListener('install', () => self.skipWaiting()) diff --git a/frontend/src/api.ts b/frontend/src/api.ts index 85ee0f0..33fe187 100644 --- a/frontend/src/api.ts +++ b/frontend/src/api.ts @@ -155,6 +155,10 @@ export interface BatchCapitalFileResult { updated: number; skipped: string[]; error: string | null; + nav_status: "added" | "updated" | "kept-signed" | "no-match" | "no-hld" | "error" | null; + nav_matched: number; + nav_unmatched: number; + nav_cents: number; } export interface BatchCapitalImportResult { diff --git a/frontend/src/pages/Import.tsx b/frontend/src/pages/Import.tsx index 2f5dc28..f6f8171 100644 --- a/frontend/src/pages/Import.tsx +++ b/frontend/src/pages/Import.tsx @@ -165,13 +165,22 @@ export default function Import() { await api.capitalImportCommit({ entity_id: resolvedEntityId, ...memberPayload }); setStep(replaceExisting ? "Replacing holdings…" : "Loading holdings…"); try { - await api.scheduleImport(file, { + const sres = await api.scheduleImport(file, { commit: true, entityId: resolvedEntityId, password: password || undefined, asOf, replaceExisting, }); + if (sres.history_only) { + const total = (sres.matched ?? 0) + (sres.unmatched ?? 0); + holdingsNote = + sres.status === "kept-signed" + ? "This quarter already has a signed valuation round; it was left unchanged." + : sres.status === "no-match" + ? "Older file: none of its holdings match the current book, so no NAV was recorded for that quarter." + : `Older file: recorded as a historical quarter in valuation history (${sres.matched} of ${total} holdings matched). Current holdings and cost basis untouched.`; + } } catch (e: any) { if (String(e.message).toLowerCase().includes("already exists")) { holdingsNote = "A signed valuation round exists for this quarter — holdings left unchanged."; @@ -458,8 +467,10 @@ function BatchBackfill({ entities }: { entities: Entity[] }) {

Load several past quarters at once to build investors' trend-lines. Drop the eNAV workbooks for one fund; each file's members are matched to existing accounts and their - capital statement is saved at that file's own as-of date. The latest figures are never - replaced, and members not already in the portal are skipped (not created). + capital statement is saved at that file's own as-of date, and the quarter's NAV is + recorded in the fund's valuation history. The latest figures are never replaced, + current holdings are never modified, and members not already in the portal are + skipped (not created).

{error &&
{error}
} @@ -518,6 +529,7 @@ function BatchBackfill({ entities }: { entities: Entity[] }) { File As-of Loaded + NAV Skipped (no account) @@ -529,6 +541,28 @@ function BatchBackfill({ entities }: { entities: Entity[] }) { {f.error ? "—" : f.statements_written} + + {f.error || !f.nav_status ? ( + "—" + ) : f.nav_status === "added" || f.nav_status === "updated" ? ( + <> + {formatMoneyExact(f.nav_cents)} + {f.nav_unmatched > 0 && ( + + {f.nav_unmatched} row{f.nav_unmatched === 1 ? "" : "s"} not in current book + + )} + + ) : f.nav_status === "kept-signed" ? ( + signed, kept + ) : f.nav_status === "no-match" ? ( + no holdings matched + ) : f.nav_status === "no-hld" ? ( + no HLD tab + ) : ( + NAV failed + )} + {f.error ? ( {f.error} diff --git a/frontend/src/version.ts b/frontend/src/version.ts index 3ae03fb..e8cb285 100644 --- a/frontend/src/version.ts +++ b/frontend/src/version.ts @@ -1,4 +1,4 @@ // Bumped each release so the running build is visible in the UI. // If the number shown in the app doesn't match the installed s9pk version, // the new frontend isn't actually being served. -export const APP_VERSION = "0.2.42"; +export const APP_VERSION = "0.2.43";