Release 0.2.22: capital chart, Investor View, GP stakes, doc folders
Snapshot commit bringing the uncommitted phase-2 work into version control
together with four new features and the 0.2.22 version bump.
New features:
- Investor capital-over-time chart (value, paid-in, distributions per
quarter), rendered from existing capital-account history.
- Admin Investor View: read-only reconstruction of an investor's portal
(GET /api/users/{id}/investor-view), reusing the investor portal UI.
- Document upload scoped to the selected fund's own investors, with an
explicit upload-target confirmation to prevent mis-attaching.
- GP/mgmt entities gain an Assets tab listing their stakes in the funds
they manage (new entity_stakes table + /api/entities/{id}/stakes).
- Edit-entity form (change type/status/etc.), so GP entities can be
categorized correctly.
Verified: 11/11 backend tests pass; alembic upgrades to head b8c9d0e1f2a3;
frontend tsc + vite build clean; s9pk packs at 0.2.22:0 (x86_64).
Also: ignore .DS_Store and *.s9pk artifacts.
This commit is contained in:
Jonathan Kirkwood
@@ -1,88 +1,32 @@
|
||||
"""CSV/XLSX import endpoints for entities and schedule of investments."""
|
||||
"""XLSX/CSV import of a fund's holdings and NAV from a fund-administrator eNAV pack.
|
||||
|
||||
Reads the "HLD" (Holdings Report) sheet of an administrator eNAV workbook: each
|
||||
security row becomes a holding + position, and its market value (book) becomes the
|
||||
valuation for the quarter. Encrypted workbooks are decrypted with the open password.
|
||||
A plain holdings CSV is also accepted.
|
||||
"""
|
||||
|
||||
import csv
|
||||
import io
|
||||
import re
|
||||
from datetime import date, datetime
|
||||
from typing import Any
|
||||
|
||||
import msoffcrypto
|
||||
import openpyxl
|
||||
from fastapi import APIRouter, Depends, File, HTTPException, Query, UploadFile
|
||||
from sqlmodel import Session, select
|
||||
from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, UploadFile
|
||||
from sqlmodel import Session, select, col
|
||||
|
||||
from ten31portal.audit import record_audit
|
||||
from ten31portal.auth import require_role
|
||||
from ten31portal.database import get_session
|
||||
from ten31portal.models import (
|
||||
Entity, EntityStatus, EntityType, Holding, Position,
|
||||
Entity, EntityType, Holding, Position,
|
||||
User, UserRole, Valuation, ValuationRound, RoundStatus,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/api/import", tags=["import"])
|
||||
|
||||
|
||||
# --- Column maps ---
|
||||
# Confirmed against real Carta exports 2026-06-07.
|
||||
|
||||
# Entity CSV import (Issue 9) — still needs a real entity-level export to confirm.
|
||||
ENTITY_COLUMN_MAP: dict[str, str] = {
|
||||
# UNCONFIRMED: update after inspecting a real Carta entities export
|
||||
"Entity Name": "name",
|
||||
"Entity Type": "type",
|
||||
"Vintage Year": "vintage_year",
|
||||
"Fund Size": "fund_size_cents",
|
||||
}
|
||||
|
||||
ENTITY_TYPE_MAP: dict[str, EntityType] = {
|
||||
"Fund": EntityType.fund,
|
||||
"fund": EntityType.fund,
|
||||
"SPV": EntityType.spv,
|
||||
"spv": EntityType.spv,
|
||||
"GP": EntityType.gp,
|
||||
"gp": EntityType.gp,
|
||||
"Mgmt Co": EntityType.mgmt_co,
|
||||
"mgmt_co": EntityType.mgmt_co,
|
||||
"Management Company": EntityType.mgmt_co,
|
||||
}
|
||||
|
||||
# Schedule of Investments XLSX import (Issue 10)
|
||||
# CONFIRMED against Carta export: "Low Time Preference Fund I, LLC" 2026-06-07
|
||||
#
|
||||
# Carta XLSX layout:
|
||||
# Row 1: Entity name (e.g. "Low Time Preference Fund I, LLC")
|
||||
# Row 2: Metadata line ("As of MM/DD/YYYY • Generated by ...")
|
||||
# Row 3: Empty
|
||||
# Row 4: Headers
|
||||
# Row 5: Empty
|
||||
# Rows 6+: Data (company rows alternate with position rows, separated by empty rows)
|
||||
# Last data row: "Total" summary
|
||||
#
|
||||
# Column mapping (0-indexed from Row 4 headers):
|
||||
# A (0): "Investment" — company name on company/subtotal rows
|
||||
# B (1): "Asset" — security name on position rows
|
||||
# C (2): "Investment date" — datetime on position rows
|
||||
# D (3): "Shares" — number (0 for SAFEs/membership interests)
|
||||
# E (4): "Cost" — dollar amount (float)
|
||||
# F (5): "Value" — dollar amount (float)
|
||||
# G (6): "Last Valuation date" — datetime on position rows
|
||||
# H (7): "Gain/Loss" — derived, skip
|
||||
# I (8): "Cost per share" — derived, skip
|
||||
# J (9): "FMV per share" — derived, skip
|
||||
# K (10): "Percent of partners' capital" — skip (phase 2)
|
||||
#
|
||||
# Company rows: A has name, B is empty, D/E/F have subtotals
|
||||
# Position rows: A is empty, B has security name, all columns populated
|
||||
# SAFEs: shares = 0, cost per share = 0, FMV per share = 0
|
||||
|
||||
SCHEDULE_COLUMNS = {
|
||||
"investment": 0, # A — company name
|
||||
"asset": 1, # B — security name
|
||||
"inv_date": 2, # C — investment date
|
||||
"shares": 3, # D — share count
|
||||
"cost": 4, # E — cost in dollars
|
||||
"value": 5, # F — value in dollars
|
||||
"val_date": 6, # G — last valuation date
|
||||
}
|
||||
|
||||
|
||||
def _parse_money(raw: str) -> int | None:
|
||||
@@ -121,268 +65,304 @@ def _dollars_to_cents(val: float | int) -> int:
|
||||
return round(float(val) * 100)
|
||||
|
||||
|
||||
# --- Entity import (CSV) ---
|
||||
# --- eNAV holdings import (XLSX) ---
|
||||
|
||||
@router.post("/entities")
|
||||
def import_entities(
|
||||
file: UploadFile = File(...),
|
||||
commit: bool = Query(default=True),
|
||||
user: User = Depends(require_role(UserRole.approver, UserRole.cfo)),
|
||||
session: Session = Depends(get_session),
|
||||
) -> dict[str, Any]:
|
||||
content = file.file.read().decode("utf-8-sig")
|
||||
reader = csv.DictReader(io.StringIO(content))
|
||||
|
||||
results: list[dict] = []
|
||||
errors: list[dict] = []
|
||||
created = 0
|
||||
updated = 0
|
||||
|
||||
for i, row in enumerate(reader, start=2):
|
||||
mapped: dict[str, Any] = {}
|
||||
|
||||
for csv_col, model_field in ENTITY_COLUMN_MAP.items():
|
||||
val = row.get(csv_col)
|
||||
if val is None:
|
||||
for k, v in row.items():
|
||||
if k.strip().lower() == csv_col.lower():
|
||||
val = v
|
||||
break
|
||||
if val is not None:
|
||||
mapped[model_field] = val.strip()
|
||||
|
||||
parsed: dict[str, Any] = {}
|
||||
row_errors: list[str] = []
|
||||
|
||||
name = mapped.get("name")
|
||||
if not name:
|
||||
row_errors.append("Missing entity name")
|
||||
else:
|
||||
parsed["name"] = name
|
||||
|
||||
type_raw = mapped.get("type", "")
|
||||
entity_type = ENTITY_TYPE_MAP.get(type_raw)
|
||||
if entity_type is None and type_raw:
|
||||
row_errors.append(f"Unknown entity type: {type_raw}")
|
||||
elif entity_type:
|
||||
parsed["type"] = entity_type
|
||||
|
||||
vy = mapped.get("vintage_year")
|
||||
if vy:
|
||||
try:
|
||||
parsed["vintage_year"] = int(vy)
|
||||
except ValueError:
|
||||
row_errors.append(f"Invalid vintage year: {vy}")
|
||||
|
||||
fs = mapped.get("fund_size_cents")
|
||||
if fs:
|
||||
cents = _parse_money(fs)
|
||||
if cents is None:
|
||||
row_errors.append(f"Cannot parse fund size: {fs}")
|
||||
else:
|
||||
parsed["fund_size_cents"] = cents
|
||||
|
||||
if row_errors:
|
||||
errors.append({"row": i, "errors": row_errors, "raw": dict(row)})
|
||||
continue
|
||||
|
||||
if not parsed.get("name"):
|
||||
continue
|
||||
|
||||
existing = session.exec(select(Entity).where(Entity.name == parsed["name"])).first()
|
||||
action = "update" if existing else "create"
|
||||
|
||||
results.append({
|
||||
"row": i,
|
||||
"action": action,
|
||||
"name": parsed["name"],
|
||||
"type": parsed.get("type", EntityType.fund).value if parsed.get("type") else None,
|
||||
"vintage_year": parsed.get("vintage_year"),
|
||||
"fund_size_cents": parsed.get("fund_size_cents"),
|
||||
})
|
||||
|
||||
if commit:
|
||||
if existing:
|
||||
if "type" in parsed:
|
||||
existing.type = parsed["type"]
|
||||
if "vintage_year" in parsed:
|
||||
existing.vintage_year = parsed["vintage_year"]
|
||||
if "fund_size_cents" in parsed:
|
||||
existing.fund_size_cents = parsed["fund_size_cents"]
|
||||
session.add(existing)
|
||||
updated += 1
|
||||
else:
|
||||
entity = Entity(
|
||||
name=parsed["name"],
|
||||
type=parsed.get("type", EntityType.fund),
|
||||
vintage_year=parsed.get("vintage_year"),
|
||||
fund_size_cents=parsed.get("fund_size_cents"),
|
||||
)
|
||||
session.add(entity)
|
||||
created += 1
|
||||
|
||||
if commit:
|
||||
record_audit(session, user.id, "import_entities", "entity", None, {
|
||||
"created": created, "updated": updated, "errors": len(errors),
|
||||
})
|
||||
session.commit()
|
||||
|
||||
return {
|
||||
"committed": commit,
|
||||
"preview": results,
|
||||
"errors": errors,
|
||||
"summary": {"created": created, "updated": updated, "error_rows": len(errors)},
|
||||
}
|
||||
def _open_workbook(file_bytes: bytes, password: str | None):
|
||||
"""Load an xlsx workbook, decrypting an encrypted (password-protected) file if needed."""
|
||||
# A normal .xlsx is a zip ("PK"); an encrypted Office file is an OLE2 container.
|
||||
if file_bytes[:2] == b"PK":
|
||||
return openpyxl.load_workbook(io.BytesIO(file_bytes), data_only=True)
|
||||
try:
|
||||
office = msoffcrypto.OfficeFile(io.BytesIO(file_bytes))
|
||||
office.load_key(password=password or "VelvetSweatshop")
|
||||
out = io.BytesIO()
|
||||
office.decrypt(out)
|
||||
out.seek(0)
|
||||
return openpyxl.load_workbook(out, data_only=True)
|
||||
except Exception:
|
||||
if not password:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="This spreadsheet is password-protected. Enter the open password and try again.",
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Could not open the spreadsheet. The password may be incorrect.",
|
||||
)
|
||||
|
||||
|
||||
# --- Schedule of investments import (XLSX) ---
|
||||
def _enav_as_of(wb) -> date | None:
|
||||
"""Find the report date, e.g. a 'DECEMBER 31, 2025' / 'AS OF ...' cell near the top."""
|
||||
for sheet in (["MENU", "HLD"] if "MENU" in wb.sheetnames else wb.sheetnames):
|
||||
ws = wb[sheet]
|
||||
for row in ws.iter_rows(min_row=1, max_row=6, max_col=2, values_only=True):
|
||||
for cell in row:
|
||||
if isinstance(cell, datetime):
|
||||
return cell.date()
|
||||
if isinstance(cell, date):
|
||||
return cell
|
||||
if isinstance(cell, str):
|
||||
text = cell.replace("AS OF", "").strip()
|
||||
for fmt in ("%B %d, %Y", "%b %d, %Y", "%m/%d/%Y"):
|
||||
try:
|
||||
return datetime.strptime(text, fmt).date()
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
def _parse_schedule_xlsx(file_bytes: bytes) -> tuple[str | None, list[dict], list[dict], list[dict]]:
|
||||
|
||||
def _parse_schedule_xlsx(file_bytes: bytes, password: str | None = None) -> tuple[str | None, date | None, list[dict], list[dict], list[dict]]:
|
||||
"""
|
||||
Parse a Carta Schedule of Investments XLSX export.
|
||||
Parse the HLD (Holdings Report) sheet of a fund-administrator eNAV workbook.
|
||||
|
||||
Returns: (entity_name, holdings_preview, positions_preview, errors)
|
||||
Each security row becomes a holding (issuer) + position (security), with cost basis
|
||||
and market value (book). Returns: (entity_name, as_of, holdings, positions, errors).
|
||||
"""
|
||||
wb = openpyxl.load_workbook(io.BytesIO(file_bytes), data_only=True)
|
||||
ws = wb.active
|
||||
wb = _open_workbook(file_bytes, password)
|
||||
|
||||
entity_name: str | None = None
|
||||
holdings_preview: list[dict] = []
|
||||
positions_preview: list[dict] = []
|
||||
errors: list[dict] = []
|
||||
|
||||
# Row 1: entity name
|
||||
row1_val = ws.cell(row=1, column=1).value
|
||||
if row1_val:
|
||||
entity_name = str(row1_val).strip()
|
||||
# Fund name from the MENU cover sheet (row 2) when present.
|
||||
if "MENU" in wb.sheetnames:
|
||||
v = wb["MENU"].cell(row=2, column=1).value
|
||||
if v:
|
||||
entity_name = str(v).strip()
|
||||
|
||||
# Row 4: headers — validate
|
||||
expected_headers = {0: "Investment", 1: "Asset", 4: "Cost", 5: "Value"}
|
||||
for col_idx, expected in expected_headers.items():
|
||||
actual = ws.cell(row=4, column=col_idx + 1).value
|
||||
if actual and str(actual).strip() != expected:
|
||||
errors.append({
|
||||
"row": 4,
|
||||
"errors": [f"Expected header '{expected}' in column {chr(65 + col_idx)}, got '{actual}'"],
|
||||
})
|
||||
as_of = _enav_as_of(wb)
|
||||
|
||||
if "HLD" not in wb.sheetnames:
|
||||
errors.append({"row": 0, "errors": [
|
||||
"No 'HLD' (Holdings Report) sheet found. This does not look like an eNAV workbook."
|
||||
]})
|
||||
return entity_name, as_of, holdings_preview, positions_preview, errors
|
||||
|
||||
ws = wb["HLD"]
|
||||
rows = list(ws.iter_rows(min_row=1, max_row=ws.max_row, max_col=20, values_only=True))
|
||||
|
||||
# Locate the header row (the one whose first cell is "SECURITY NAME").
|
||||
header_idx = None
|
||||
for i, row in enumerate(rows):
|
||||
if row and isinstance(row[0], str) and row[0].strip().upper() == "SECURITY NAME":
|
||||
header_idx = i
|
||||
break
|
||||
if header_idx is None:
|
||||
errors.append({"row": 0, "errors": ["Could not find the holdings header row in the HLD sheet."]})
|
||||
return entity_name, as_of, holdings_preview, positions_preview, errors
|
||||
|
||||
header = [str(c).strip().upper() if isinstance(c, str) else "" for c in rows[header_idx]]
|
||||
|
||||
def col(*names: str) -> int | None:
|
||||
for n in names:
|
||||
if n in header:
|
||||
return header.index(n)
|
||||
return None
|
||||
|
||||
c_name = col("SECURITY NAME")
|
||||
c_qty = col("QUANTITY")
|
||||
c_cost = col("COST BASIS - BOOK", "COST BASIS - LOCAL")
|
||||
c_value = col("MARKET VALUE (BOOK)", "MARKET VALUE (LOCAL)", "MARKET VALUE - BOOK")
|
||||
|
||||
if c_value is None or c_name is None:
|
||||
errors.append({"row": header_idx + 1, "errors": [
|
||||
"HLD sheet is missing a SECURITY NAME or MARKET VALUE column."
|
||||
]})
|
||||
return entity_name, as_of, holdings_preview, positions_preview, errors
|
||||
|
||||
# Parse data rows (starting at row 6)
|
||||
current_company: str | None = None
|
||||
seen_companies: set[str] = set()
|
||||
# Track (company, security) occurrences to disambiguate duplicate tranches
|
||||
security_counts: dict[tuple[str, str], int] = {}
|
||||
|
||||
for row_idx in range(6, ws.max_row + 1):
|
||||
col_a = ws.cell(row=row_idx, column=1).value # Investment (company)
|
||||
col_b = ws.cell(row=row_idx, column=2).value # Asset (security)
|
||||
col_c = ws.cell(row=row_idx, column=3).value # Investment date
|
||||
col_d = ws.cell(row=row_idx, column=4).value # Shares
|
||||
col_e = ws.cell(row=row_idx, column=5).value # Cost
|
||||
col_f = ws.cell(row=row_idx, column=6).value # Value
|
||||
col_g = ws.cell(row=row_idx, column=7).value # Last valuation date
|
||||
|
||||
# Skip empty rows
|
||||
if col_a is None and col_b is None:
|
||||
for r in range(header_idx + 1, len(rows)):
|
||||
row = rows[r]
|
||||
raw_name = row[c_name] if c_name < len(row) else None
|
||||
if raw_name is None or not str(raw_name).strip():
|
||||
continue
|
||||
name = str(raw_name).strip()
|
||||
# Stop at total / report-total summary rows.
|
||||
if name.upper().startswith("TOTAL") or name.upper().startswith("REPORT TOTAL"):
|
||||
continue
|
||||
|
||||
# Skip total row
|
||||
if col_a and str(col_a).strip().lower() == "total":
|
||||
# Group tranches of the same issuer: holding = issuer (before " - "); position = full name.
|
||||
company = name.split(" - ")[0].strip() or name
|
||||
security = name
|
||||
|
||||
if company not in seen_companies:
|
||||
holdings_preview.append({"row": r + 1, "company_name": company})
|
||||
seen_companies.add(company)
|
||||
|
||||
# Quantity -> shares (skip non-numeric like "N/A").
|
||||
shares: str | None = None
|
||||
if c_qty is not None and c_qty < len(row) and row[c_qty] is not None:
|
||||
qv = row[c_qty]
|
||||
if isinstance(qv, (int, float)) and qv != 0:
|
||||
shares = f"{qv:g}"
|
||||
|
||||
cost_cents = None
|
||||
if c_cost is not None and c_cost < len(row) and isinstance(row[c_cost], (int, float)):
|
||||
cost_cents = _dollars_to_cents(row[c_cost])
|
||||
|
||||
value_cents = None
|
||||
if isinstance(row[c_value], (int, float)):
|
||||
value_cents = _dollars_to_cents(row[c_value])
|
||||
|
||||
positions_preview.append({
|
||||
"row": r + 1,
|
||||
"company_name": company,
|
||||
"security_name": security,
|
||||
"investment_date": None, # eNAV holdings report has no acquisition date
|
||||
"shares": shares,
|
||||
"cost_cents": cost_cents,
|
||||
"value_cents": value_cents,
|
||||
"valuation_date": str(as_of) if as_of else None,
|
||||
})
|
||||
|
||||
return entity_name, as_of, holdings_preview, positions_preview, errors
|
||||
|
||||
|
||||
|
||||
|
||||
def reset_entity_holdings(entity_id: int, session: Session) -> dict[str, int]:
|
||||
"""Delete all holdings, positions, valuations, and rounds for one entity.
|
||||
|
||||
For a clean restart — e.g. after switching the source workbook (Carta → eNAV) renamed every
|
||||
position, so the old and new rows can't be matched and both get counted. Capital-account
|
||||
statements (investor data) are left untouched. The caller should re-import afterwards.
|
||||
"""
|
||||
rounds = session.exec(
|
||||
select(ValuationRound).where(ValuationRound.entity_id == entity_id)
|
||||
).all()
|
||||
round_ids = [r.id for r in rounds]
|
||||
valuations = 0
|
||||
if round_ids:
|
||||
for v in session.exec(
|
||||
select(Valuation).where(col(Valuation.round_id).in_(round_ids))
|
||||
).all():
|
||||
session.delete(v)
|
||||
valuations += 1
|
||||
|
||||
holdings = session.exec(select(Holding).where(Holding.entity_id == entity_id)).all()
|
||||
holding_ids = [h.id for h in holdings]
|
||||
positions = 0
|
||||
if holding_ids:
|
||||
for p in session.exec(
|
||||
select(Position).where(col(Position.holding_id).in_(holding_ids))
|
||||
).all():
|
||||
session.delete(p)
|
||||
positions += 1
|
||||
|
||||
for r in rounds:
|
||||
session.delete(r)
|
||||
for h in holdings:
|
||||
session.delete(h)
|
||||
session.flush()
|
||||
return {
|
||||
"rounds": len(rounds),
|
||||
"holdings": len(holdings),
|
||||
"positions": positions,
|
||||
"valuations": valuations,
|
||||
}
|
||||
|
||||
|
||||
def dedupe_entity(entity_id: int, session: Session) -> dict[str, int]:
|
||||
"""Collapse exact-duplicate holdings and positions for one entity.
|
||||
|
||||
Repeated imports on older builds created a second copy of each holding/position, which
|
||||
inflated the entity's "Invested" total (the rollup sums cost across every position). This
|
||||
keeps the lowest-id copy, re-points its positions/valuations, removes duplicate valuations
|
||||
in the same round, and deletes the leftovers. Safe to run repeatedly (a no-op once clean).
|
||||
"""
|
||||
removed_holdings = 0
|
||||
removed_positions = 0
|
||||
|
||||
# 1) Merge holdings with the same name (case-insensitive) into the lowest-id one.
|
||||
holdings = session.exec(
|
||||
select(Holding).where(Holding.entity_id == entity_id).order_by(Holding.id) # type: ignore[arg-type]
|
||||
).all()
|
||||
keep_by_name: dict[str, Holding] = {}
|
||||
for h in holdings:
|
||||
key = h.company_name.strip().lower()
|
||||
keeper = keep_by_name.get(key)
|
||||
if keeper is None:
|
||||
keep_by_name[key] = h
|
||||
continue
|
||||
for p in session.exec(select(Position).where(Position.holding_id == h.id)).all():
|
||||
p.holding_id = keeper.id
|
||||
session.add(p)
|
||||
session.flush()
|
||||
session.delete(h)
|
||||
removed_holdings += 1
|
||||
session.flush()
|
||||
|
||||
# Company subtotal row: col A has name, col B is empty
|
||||
if col_a and not col_b:
|
||||
company_name = str(col_a).strip()
|
||||
current_company = company_name
|
||||
if company_name not in seen_companies:
|
||||
holdings_preview.append({
|
||||
"row": row_idx,
|
||||
"company_name": company_name,
|
||||
})
|
||||
seen_companies.add(company_name)
|
||||
continue
|
||||
# 2) Within each surviving holding, merge positions with the same security name.
|
||||
for keeper_holding in keep_by_name.values():
|
||||
positions = session.exec(
|
||||
select(Position)
|
||||
.where(Position.holding_id == keeper_holding.id)
|
||||
.order_by(Position.id) # type: ignore[arg-type]
|
||||
).all()
|
||||
keep_by_sec: dict[str, Position] = {}
|
||||
for p in positions:
|
||||
key = p.security_name.strip().lower()
|
||||
keeper = keep_by_sec.get(key)
|
||||
if keeper is None:
|
||||
keep_by_sec[key] = p
|
||||
continue
|
||||
# Move this duplicate's valuations onto the keeper, dropping any that would collide
|
||||
# with an existing valuation for the same round (that collision IS the double-count).
|
||||
for v in session.exec(select(Valuation).where(Valuation.position_id == p.id)).all():
|
||||
clash = session.exec(
|
||||
select(Valuation).where(
|
||||
Valuation.round_id == v.round_id,
|
||||
Valuation.position_id == keeper.id,
|
||||
)
|
||||
).first()
|
||||
if clash is not None:
|
||||
session.delete(v)
|
||||
else:
|
||||
v.position_id = keeper.id
|
||||
session.add(v)
|
||||
session.flush()
|
||||
session.delete(p)
|
||||
removed_positions += 1
|
||||
session.flush()
|
||||
|
||||
# Position row: col B has security name
|
||||
if col_b:
|
||||
raw_security_name = str(col_b).strip()
|
||||
company = current_company or "(unknown)"
|
||||
|
||||
# Disambiguate duplicate (company, security) pairs
|
||||
# e.g. two "Warrants" under BIP21 become "Warrants" and "Warrants (2)"
|
||||
pair_key = (company, raw_security_name)
|
||||
security_counts[pair_key] = security_counts.get(pair_key, 0) + 1
|
||||
if security_counts[pair_key] == 1:
|
||||
security_name = raw_security_name
|
||||
else:
|
||||
security_name = f"{raw_security_name} ({security_counts[pair_key]})"
|
||||
|
||||
# Parse investment date
|
||||
inv_date: date | None = None
|
||||
if isinstance(col_c, datetime):
|
||||
inv_date = col_c.date()
|
||||
elif col_c:
|
||||
inv_date = _parse_date(str(col_c))
|
||||
|
||||
# Parse shares (could be 0 for SAFEs)
|
||||
shares: str | None = None
|
||||
if col_d is not None:
|
||||
shares_val = float(col_d)
|
||||
if shares_val != 0:
|
||||
# Preserve precision: use string repr
|
||||
shares = str(col_d) if not isinstance(col_d, float) else f"{col_d:g}"
|
||||
# shares stays None for zero (SAFEs)
|
||||
|
||||
# Parse cost (dollars)
|
||||
cost_cents: int | None = None
|
||||
if col_e is not None:
|
||||
cost_cents = _dollars_to_cents(col_e)
|
||||
|
||||
# Parse value (dollars)
|
||||
value_cents: int | None = None
|
||||
if col_f is not None:
|
||||
value_cents = _dollars_to_cents(col_f)
|
||||
|
||||
# Parse valuation date
|
||||
val_date: date | None = None
|
||||
if isinstance(col_g, datetime):
|
||||
val_date = col_g.date()
|
||||
elif col_g:
|
||||
val_date = _parse_date(str(col_g))
|
||||
|
||||
positions_preview.append({
|
||||
"row": row_idx,
|
||||
"company_name": company,
|
||||
"security_name": security_name,
|
||||
"investment_date": str(inv_date) if inv_date else None,
|
||||
"shares": shares,
|
||||
"cost_cents": cost_cents,
|
||||
"value_cents": value_cents,
|
||||
"valuation_date": str(val_date) if val_date else None,
|
||||
})
|
||||
|
||||
return entity_name, holdings_preview, positions_preview, errors
|
||||
return {"removed_holdings": removed_holdings, "removed_positions": removed_positions}
|
||||
|
||||
|
||||
@router.post("/schedule")
|
||||
def import_schedule(
|
||||
file: UploadFile = File(...),
|
||||
as_of: date = Query(...),
|
||||
as_of: date | None = Query(default=None),
|
||||
entity_id: int | None = Query(default=None),
|
||||
create_entity_type: EntityType = Query(default=EntityType.fund),
|
||||
create_vintage_year: int | None = Query(default=None),
|
||||
commit: bool = Query(default=True),
|
||||
user: User = Depends(require_role(UserRole.approver, UserRole.cfo)),
|
||||
replace_existing: bool = Query(default=False),
|
||||
password: str | None = Form(default=None),
|
||||
user: User = Depends(require_role(UserRole.approver, UserRole.cfo, UserRole.operations)),
|
||||
session: Session = Depends(get_session),
|
||||
) -> dict[str, Any]:
|
||||
file_bytes = file.file.read()
|
||||
filename = file.filename or ""
|
||||
|
||||
# Parse file first to get source_entity_name
|
||||
if filename.endswith(".xlsx") or filename.endswith(".xls"):
|
||||
source_entity_name, holdings_preview, positions_preview, errors = _parse_schedule_xlsx(file_bytes)
|
||||
# Parse the file into holdings/positions previews.
|
||||
if filename.lower().endswith((".xlsx", ".xls")):
|
||||
source_entity_name, parsed_as_of, holdings_preview, positions_preview, errors = (
|
||||
_parse_schedule_xlsx(file_bytes, password)
|
||||
)
|
||||
else:
|
||||
source_entity_name = None
|
||||
parsed_as_of = None
|
||||
holdings_preview, positions_preview, errors = _parse_schedule_csv(file_bytes)
|
||||
|
||||
# As-of date: use the explicit value, else the date read from the sheet.
|
||||
as_of = as_of or parsed_as_of
|
||||
if as_of is None:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Could not determine the quarter-end date. Set the as-of date and try again.",
|
||||
)
|
||||
|
||||
# Resolve entity
|
||||
entity: Entity | None = None
|
||||
entity_resolution: str = "existing" # "existing" | "matched" | "will_create"
|
||||
@@ -405,7 +385,7 @@ def import_schedule(
|
||||
else:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="No entity_id provided and the file has no entity name in row 1. Provide entity_id or upload a Carta XLSX with the fund name.",
|
||||
detail="No entity selected and no fund name found in the file. Choose the fund to import into.",
|
||||
)
|
||||
|
||||
# Dry-run: report resolution without writing
|
||||
@@ -447,19 +427,36 @@ def import_schedule(
|
||||
assert entity is not None
|
||||
resolved_entity_id = entity.id
|
||||
|
||||
# Check for any existing round at this quarter
|
||||
# Replace mode: wipe the fund's existing holdings/positions/rounds first so the file becomes
|
||||
# the single source of truth. Use this after a source change (e.g. Carta → eNAV) renamed the
|
||||
# positions, leaving un-matchable rows that inflate the totals.
|
||||
if replace_existing:
|
||||
reset_entity_holdings(resolved_entity_id, session)
|
||||
|
||||
# Self-heal any duplicate holdings/positions left by imports on older builds, so this
|
||||
# import's upsert lands on a single clean copy and "Invested" stops double-counting.
|
||||
dedupe_entity(resolved_entity_id, session)
|
||||
|
||||
# An existing round at this quarter: a NAV re-import should UPDATE it in place (refresh
|
||||
# to the latest file) instead of stacking a second round and doubling the totals. Only
|
||||
# import-created seed rounds are refreshable; a manually-signed valuation round is left
|
||||
# protected.
|
||||
existing_round = session.exec(
|
||||
select(ValuationRound).where(
|
||||
ValuationRound.entity_id == resolved_entity_id,
|
||||
ValuationRound.quarter_end == as_of,
|
||||
)
|
||||
).first()
|
||||
if existing_round:
|
||||
kind = "seed" if existing_round.is_seed else "valuation"
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail=f"A {kind} round already exists for this entity and quarter ({as_of}). Cannot overwrite signed history.",
|
||||
)
|
||||
reused_round = False
|
||||
seed_round: ValuationRound | None = None
|
||||
if existing_round is not None:
|
||||
if not existing_round.is_seed:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail=f"A signed valuation round already exists for this entity and quarter ({as_of}). Cannot overwrite signed history.",
|
||||
)
|
||||
seed_round = existing_round
|
||||
reused_round = True
|
||||
|
||||
# Commit: create holdings, positions, seed round
|
||||
holding_map: dict[str, Holding] = {}
|
||||
@@ -480,20 +477,22 @@ def import_schedule(
|
||||
session.flush()
|
||||
holding_map[name] = h
|
||||
|
||||
# Create seed round
|
||||
seed_round = ValuationRound(
|
||||
entity_id=resolved_entity_id,
|
||||
quarter_end=as_of,
|
||||
status=RoundStatus.approved,
|
||||
is_seed=True,
|
||||
approved_by=user.id,
|
||||
approved_at=datetime.utcnow(),
|
||||
)
|
||||
session.add(seed_round)
|
||||
session.flush()
|
||||
# Create the seed round, or reuse the existing one for this quarter (re-import).
|
||||
if seed_round is None:
|
||||
seed_round = ValuationRound(
|
||||
entity_id=resolved_entity_id,
|
||||
quarter_end=as_of,
|
||||
status=RoundStatus.approved,
|
||||
is_seed=True,
|
||||
approved_by=user.id,
|
||||
approved_at=datetime.utcnow(),
|
||||
)
|
||||
session.add(seed_round)
|
||||
session.flush()
|
||||
|
||||
positions_created = 0
|
||||
positions_updated = 0
|
||||
seen_position_ids: set[int] = set()
|
||||
for pp in positions_preview:
|
||||
company = pp["company_name"]
|
||||
holding = holding_map.get(company)
|
||||
@@ -541,17 +540,39 @@ def import_schedule(
|
||||
session.flush()
|
||||
positions_updated += 1
|
||||
|
||||
# Attach valuation to seed round
|
||||
val = Valuation(
|
||||
round_id=seed_round.id,
|
||||
position_id=pos.id,
|
||||
value_cents=pp["value_cents"] or 0,
|
||||
)
|
||||
session.add(val)
|
||||
# Upsert this position's valuation in the round, so a re-import refreshes the value
|
||||
# in place instead of adding a second one (which would double the quarter's NAV).
|
||||
val = session.exec(
|
||||
select(Valuation).where(
|
||||
Valuation.round_id == seed_round.id,
|
||||
Valuation.position_id == pos.id,
|
||||
)
|
||||
).first()
|
||||
if val is None:
|
||||
session.add(Valuation(
|
||||
round_id=seed_round.id,
|
||||
position_id=pos.id,
|
||||
value_cents=pp["value_cents"] or 0,
|
||||
))
|
||||
else:
|
||||
val.value_cents = pp["value_cents"] or 0
|
||||
session.add(val)
|
||||
seen_position_ids.add(pos.id)
|
||||
|
||||
# On re-import, drop valuations for holdings no longer in the file so the quarter's NAV
|
||||
# equals the new file's total (no leftovers from the prior import).
|
||||
if reused_round:
|
||||
stale_q = select(Valuation).where(Valuation.round_id == seed_round.id)
|
||||
if seen_position_ids:
|
||||
stale_q = stale_q.where(col(Valuation.position_id).not_in(seen_position_ids))
|
||||
for stale_val in session.exec(stale_q).all():
|
||||
session.delete(stale_val)
|
||||
|
||||
record_audit(session, user.id, "import_schedule", "entity", resolved_entity_id, {
|
||||
"source_entity_name": source_entity_name,
|
||||
"entity_resolution": entity_resolution,
|
||||
"replaced_existing": replace_existing,
|
||||
"round_updated": reused_round,
|
||||
"holdings": len(holding_map),
|
||||
"positions_created": positions_created,
|
||||
"positions_updated": positions_updated,
|
||||
@@ -573,25 +594,35 @@ def import_schedule(
|
||||
"positions_created": positions_created,
|
||||
"positions_updated": positions_updated,
|
||||
"seed_round_id": seed_round.id,
|
||||
"round_updated": reused_round,
|
||||
"replaced_existing": replace_existing,
|
||||
"errors": errors,
|
||||
}
|
||||
|
||||
|
||||
def _parse_schedule_csv(file_bytes: bytes) -> tuple[list[dict], list[dict], list[dict]]:
|
||||
"""Legacy CSV parser fallback."""
|
||||
"""Parse a plain holdings CSV (generic columns or an eNAV HLD export)."""
|
||||
content = file_bytes.decode("utf-8-sig")
|
||||
reader = csv.DictReader(io.StringIO(content))
|
||||
|
||||
CSV_COLUMN_MAP = {
|
||||
"Company": "company_name",
|
||||
"Investment": "company_name",
|
||||
"Issuer": "company_name",
|
||||
"Security": "security_name",
|
||||
"Asset": "security_name",
|
||||
"Security Name": "security_name",
|
||||
"Investment Date": "investment_date",
|
||||
"Investment date": "investment_date",
|
||||
"Shares": "shares",
|
||||
"Quantity": "shares",
|
||||
"Cost": "cost_cents",
|
||||
"Cost Basis - Book": "cost_cents",
|
||||
"Cost Basis - Local": "cost_cents",
|
||||
"Value": "value_cents",
|
||||
"Market Value (Book)": "value_cents",
|
||||
"Market Value (Local)": "value_cents",
|
||||
"Market Value - Book": "value_cents",
|
||||
}
|
||||
|
||||
holdings_preview: list[dict] = []
|
||||
@@ -608,11 +639,18 @@ def _parse_schedule_csv(file_bytes: bytes) -> tuple[list[dict], list[dict], list
|
||||
if k.strip().lower() == csv_col.lower():
|
||||
val = v
|
||||
break
|
||||
if val is not None:
|
||||
mapped[model_field] = val.strip()
|
||||
if val is not None and model_field not in mapped:
|
||||
mapped[model_field] = (val or "").strip()
|
||||
|
||||
company = mapped.get("company_name", "").strip()
|
||||
security = mapped.get("security_name", "").strip()
|
||||
# Skip total / summary rows.
|
||||
if security.upper().startswith("TOTAL") or company.upper().startswith("TOTAL"):
|
||||
continue
|
||||
# eNAV-style rows have only a security name; derive the issuer from its prefix.
|
||||
if security and not company:
|
||||
company = security.split(" - ")[0].strip()
|
||||
mapped["company_name"] = company
|
||||
|
||||
if company and not security:
|
||||
current_company = company
|
||||
|
||||
Reference in New Issue
Block a user