0.2.25: batch historical eNAV backfill + collapsible investor chart
Add POST /api/import/capital-accounts/batch: upload several of a fund's eNAV workbooks at once; each file's ALLOC SI roster is auto-matched to existing members (by fund-admin investor ID, else name/username) and their capital statement is saved at that file's own as-of date, building trend-lines without replacing the latest figures. Members not already in the portal are skipped and reported per file (never created). Capital statements only -- holdings/NAV are untouched. One bad file (wrong password, no ALLOC SI, unreadable date) is reported per-file and does not abort the rest. Import page gains a "Backfill historical capital" batch section (fund picker, multi-file .xlsx input, shared password, per-file results table). Investor portal "Capital over time" chart is now collapsed by default and expands per fund (first login opens clean); applies to InvestorHome and the admin Investor View via the shared component. Tests: backend/tests/test_capital_batch.py (2). Full suite 17 passed; frontend tsc clean. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Jonathan Kirkwood
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
33776d42f4
commit
099459b2f3
@@ -27,6 +27,7 @@ from ten31portal.models import (
|
||||
)
|
||||
from ten31portal.routers.import_router import _open_workbook, _enav_as_of
|
||||
from ten31portal.schemas import (
|
||||
BatchCapitalFileResult, BatchCapitalImportResult,
|
||||
CapitalImportCommit, CapitalImportPreview, ImportInvestorPreview, ImportValueRow,
|
||||
)
|
||||
|
||||
@@ -346,3 +347,118 @@ def commit_import(
|
||||
"matched_accounts_updated": updated_accounts,
|
||||
"statements_written": statements,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/batch")
|
||||
def batch_import(
|
||||
files: list[UploadFile] = File(...),
|
||||
entity_id: int = Form(...),
|
||||
password: str | None = Form(None),
|
||||
admin: User = Depends(require_internal_admin),
|
||||
session: Session = Depends(get_session),
|
||||
) -> BatchCapitalImportResult:
|
||||
"""Backfill several quarters of capital history for one fund from a batch of eNAV files.
|
||||
|
||||
Each file is a full eNAV workbook; its ALLOC SI roster is auto-matched (by fund-admin
|
||||
investor ID, else name/username) against existing members and each member's capital
|
||||
statement is upserted at the file's own as-of date — so older files add historical points
|
||||
without touching the latest. Members not already in the system are skipped and reported
|
||||
(no account creation). One file failing (bad password, no ALLOC SI, unreadable date) is
|
||||
reported per-file and does not abort the rest.
|
||||
"""
|
||||
entity = session.get(Entity, entity_id)
|
||||
if entity is None:
|
||||
raise HTTPException(status_code=404, detail="Entity not found")
|
||||
|
||||
investors = session.exec(select(User).where(User.role == UserRole.investor)).all()
|
||||
by_name = {u.name.strip().lower(): u for u in investors}
|
||||
by_username = {u.username.strip().lower(): u for u in investors}
|
||||
by_extid = {u.external_investor_id: u for u in investors if u.external_investor_id}
|
||||
|
||||
def match_for(name: str, external_id: str | None) -> User | None:
|
||||
if external_id and external_id in by_extid:
|
||||
return by_extid[external_id]
|
||||
return by_name.get(name.strip().lower()) or by_username.get(name.strip().lower())
|
||||
|
||||
cents = lambda d: round((d or 0.0) * 100)
|
||||
results: list[BatchCapitalFileResult] = []
|
||||
total_statements = 0
|
||||
|
||||
for upload in files:
|
||||
fname = upload.filename or "(unnamed)"
|
||||
res = BatchCapitalFileResult(filename=fname)
|
||||
try:
|
||||
wb = _open_workbook(upload.file.read(), password)
|
||||
if "ALLOC SI" not in wb.sheetnames:
|
||||
raise HTTPException(status_code=422, detail="No ALLOC SI tab found in this workbook.")
|
||||
as_of, roster = _parse_alloc_si(wb)
|
||||
if as_of is None:
|
||||
raise HTTPException(status_code=422, detail="Could not determine the as-of date from the file.")
|
||||
res.as_of_date = as_of
|
||||
|
||||
for inv in roster:
|
||||
user = match_for(inv["name"], inv["external_id"])
|
||||
if user is None:
|
||||
res.skipped.append(inv["name"])
|
||||
continue
|
||||
|
||||
# Record the fund-admin ID on first sighting so later files match by ID too.
|
||||
if inv["external_id"] and not user.external_investor_id:
|
||||
user.external_investor_id = inv["external_id"]
|
||||
by_extid[inv["external_id"]] = user
|
||||
session.add(user)
|
||||
|
||||
has_access = session.exec(
|
||||
select(EntityAccess).where(
|
||||
EntityAccess.user_id == user.id, EntityAccess.entity_id == entity_id
|
||||
)
|
||||
).first()
|
||||
if has_access is None:
|
||||
session.add(EntityAccess(user_id=user.id, entity_id=entity_id))
|
||||
|
||||
existing = session.exec(
|
||||
select(CapitalAccountStatement).where(
|
||||
CapitalAccountStatement.entity_id == entity_id,
|
||||
CapitalAccountStatement.investor_user_id == user.id,
|
||||
CapitalAccountStatement.as_of_date == as_of,
|
||||
)
|
||||
).first()
|
||||
if existing:
|
||||
existing.commitment_cents = cents(inv["commitment"])
|
||||
existing.contributions_cents = cents(inv["contributions"])
|
||||
existing.distributions_cents = cents(inv["distributions"])
|
||||
existing.ending_balance_cents = cents(inv["ending"])
|
||||
session.add(existing)
|
||||
res.updated += 1
|
||||
else:
|
||||
session.add(CapitalAccountStatement(
|
||||
entity_id=entity_id,
|
||||
investor_user_id=user.id,
|
||||
as_of_date=as_of,
|
||||
commitment_cents=cents(inv["commitment"]),
|
||||
contributions_cents=cents(inv["contributions"]),
|
||||
distributions_cents=cents(inv["distributions"]),
|
||||
ending_balance_cents=cents(inv["ending"]),
|
||||
))
|
||||
res.matched += 1
|
||||
res.statements_written += 1
|
||||
|
||||
record_audit(session, admin.id, "import_batch", "capital_account", entity_id, {
|
||||
"file": fname,
|
||||
"as_of_date": str(as_of),
|
||||
"matched": res.matched,
|
||||
"skipped": len(res.skipped),
|
||||
})
|
||||
session.commit()
|
||||
total_statements += res.statements_written
|
||||
except HTTPException as e:
|
||||
session.rollback()
|
||||
res.error = e.detail
|
||||
except Exception as e: # noqa: BLE001 — surface any parse failure per-file, keep going
|
||||
session.rollback()
|
||||
res.error = str(e) or "Could not process this file."
|
||||
results.append(res)
|
||||
|
||||
return BatchCapitalImportResult(
|
||||
entity_id=entity_id, files=results, total_statements=total_statements
|
||||
)
|
||||
|
||||
@@ -320,6 +320,24 @@ class CapitalImportCommit(BaseModel):
|
||||
investors: list[ImportCommitInvestor]
|
||||
|
||||
|
||||
# --- Batch historical capital backfill (one eNAV file per quarter, auto-matched) ---
|
||||
|
||||
class BatchCapitalFileResult(BaseModel):
|
||||
filename: str
|
||||
as_of_date: date | None = None
|
||||
matched: int = 0 # existing members whose statement was written
|
||||
statements_written: int = 0 # created + updated
|
||||
updated: int = 0 # matched a statement already at this as-of date
|
||||
skipped: list[str] = [] # roster names with no existing member (not created)
|
||||
error: str | None = None # file-level failure (bad password, no ALLOC SI, etc.)
|
||||
|
||||
|
||||
class BatchCapitalImportResult(BaseModel):
|
||||
entity_id: int
|
||||
files: list[BatchCapitalFileResult]
|
||||
total_statements: int
|
||||
|
||||
|
||||
# --- Entity stakes (a GP/mgmt entity's interest in the funds it manages) ---
|
||||
|
||||
class EntityStakeCreate(BaseModel):
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
"""Batch historical capital backfill: several eNAV files → per-quarter statements, auto-matched."""
|
||||
|
||||
import io
|
||||
from datetime import date, datetime
|
||||
|
||||
import openpyxl
|
||||
from sqlmodel import select
|
||||
|
||||
from ten31portal.models import CapitalAccountStatement, Entity, EntityType, User, UserRole
|
||||
from tests.conftest import make_user
|
||||
|
||||
|
||||
def _alloc_si_file(report: datetime, rows: list[dict]) -> bytes:
|
||||
"""Build a minimal eNAV workbook with an ALLOC SI roster and a report date in A1."""
|
||||
wb = openpyxl.Workbook()
|
||||
ws = wb.active
|
||||
ws.title = "ALLOC SI"
|
||||
ws["A1"] = report # _enav_as_of reads the report date from the top-left cells
|
||||
header = ["INVESTOR ID", "INVESTOR TYPE", "INVESTOR NAME",
|
||||
"COMMITTED CAPITAL", "CONTRIBUTIONS", "(DISTRIBUTIONS)", "ENDING BALANCE"]
|
||||
ws.append([None] * 7) # row 2 spacer
|
||||
ws.append(header) # row 3 header
|
||||
for r in rows:
|
||||
ws.append([
|
||||
r.get("id"), "LP", r["name"],
|
||||
r.get("commit", 0), r.get("contrib", 0), r.get("distrib", 0), r["ending"],
|
||||
])
|
||||
out = io.BytesIO()
|
||||
wb.save(out)
|
||||
return out.getvalue()
|
||||
|
||||
|
||||
def test_batch_backfill_builds_history(auth_client, session):
|
||||
fund = Entity(name="LTPF III", type=EntityType.fund)
|
||||
session.add(fund)
|
||||
session.commit()
|
||||
session.refresh(fund)
|
||||
|
||||
# One member matched by fund-admin ID, one by name; both already exist.
|
||||
alice = make_user(session, username="alice", name="Alice Trust",
|
||||
role=UserRole.investor, external_investor_id="INV-100")
|
||||
make_user(session, username="bob", name="Bob Llc", role=UserRole.investor)
|
||||
|
||||
q3 = _alloc_si_file(datetime(2025, 9, 30), [
|
||||
{"id": "INV-100", "name": "Alice Trust", "commit": 1_000_000, "contrib": 400_000, "ending": 420_000},
|
||||
{"id": "INV-200", "name": "Bob LLC", "commit": 500_000, "contrib": 200_000, "ending": 205_000},
|
||||
{"id": "INV-999", "name": "Ghost Capital", "commit": 999, "contrib": 999, "ending": 999}, # no account → skipped
|
||||
])
|
||||
q4 = _alloc_si_file(datetime(2025, 12, 31), [
|
||||
{"id": "INV-100", "name": "Alice Trust", "commit": 1_000_000, "contrib": 400_000, "ending": 455_000},
|
||||
{"id": "INV-200", "name": "Bob LLC", "commit": 500_000, "contrib": 200_000, "ending": 210_000},
|
||||
])
|
||||
|
||||
resp = auth_client.post(
|
||||
"/api/import/capital-accounts/batch",
|
||||
data={"entity_id": fund.id},
|
||||
files=[
|
||||
("files", ("LTPF_III_Q3.xlsx", q3, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")),
|
||||
("files", ("LTPF_III_Q4.xlsx", q4, "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")),
|
||||
],
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
body = resp.json()
|
||||
|
||||
assert body["total_statements"] == 4 # 2 members × 2 quarters
|
||||
q3res = next(f for f in body["files"] if "Q3" in f["filename"])
|
||||
assert q3res["as_of_date"] == "2025-09-30"
|
||||
assert q3res["matched"] == 2
|
||||
assert q3res["skipped"] == ["Ghost Capital"] # unknown member reported, not created
|
||||
|
||||
# No account was created for the unknown roster name.
|
||||
assert session.exec(select(User).where(User.name == "Ghost Capital")).first() is None
|
||||
|
||||
# Alice now has two statements — a real trend line — and the latest (Q4) is intact.
|
||||
stmts = session.exec(
|
||||
select(CapitalAccountStatement)
|
||||
.where(CapitalAccountStatement.investor_user_id == alice.id)
|
||||
.order_by(CapitalAccountStatement.as_of_date) # type: ignore[arg-type]
|
||||
).all()
|
||||
assert [s.as_of_date for s in stmts] == [date(2025, 9, 30), date(2025, 12, 31)]
|
||||
assert [s.ending_balance_cents for s in stmts] == [42_000_000, 45_500_000]
|
||||
|
||||
|
||||
def test_batch_reimport_updates_in_place(auth_client, session):
|
||||
fund = Entity(name="LTPF IV", type=EntityType.fund)
|
||||
session.add(fund)
|
||||
session.commit()
|
||||
session.refresh(fund)
|
||||
carol = make_user(session, username="carol", name="Carol Ira", role=UserRole.investor)
|
||||
|
||||
def one_file(ending: int) -> bytes:
|
||||
return _alloc_si_file(datetime(2025, 12, 31),
|
||||
[{"id": "INV-1", "name": "Carol Ira", "ending": ending}])
|
||||
|
||||
for ending in (300_000, 315_000): # corrected figure re-imported at the same quarter
|
||||
resp = auth_client.post(
|
||||
"/api/import/capital-accounts/batch",
|
||||
data={"entity_id": fund.id},
|
||||
files=[("files", ("f.xlsx", one_file(ending), "application/octet-stream"))],
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
|
||||
stmts = session.exec(
|
||||
select(CapitalAccountStatement).where(CapitalAccountStatement.investor_user_id == carol.id)
|
||||
).all()
|
||||
assert len(stmts) == 1 # upsert, not a duplicate
|
||||
assert stmts[0].ending_balance_cents == 31_500_000
|
||||
Reference in New Issue
Block a user