From 34842cd99421064c6f10fb57b3e3e8a960c1b6e2 Mon Sep 17 00:00:00 2001 From: Jonathan Kirkwood Date: Wed, 2 Sep 2026 09:35:54 -0500 Subject: [PATCH] Add startup re-import queue to apply importer fixes to signed rounds The loan-aware NAV fix (0.2.47) only affects new imports; an already-signed quarter keeps its old holdings until re-imported. Add a boot-time queue: an operator stages an eNAV plus a small JSON descriptor under data/pending_reimports and the app re-imports it on startup via the same code path as the admin Import screen, then removes the processed files. Guarded so a queue problem never blocks boot. Bumps the package to 0.2.48. Used to refresh SATS Credit Fund LP Q4 2025 to its full NAV including the Fold Holdings loan. --- backend/ten31portal/main.py | 8 ++ backend/ten31portal/pending_reimports.py | 84 +++++++++++++++++++++ backend/tests/test_pending_reimport.py | 51 +++++++++++++ deploy/package.json | 2 +- deploy/startos/install/versions/index.ts | 5 +- deploy/startos/install/versions/v_0_2_48.ts | 16 ++++ 6 files changed, 163 insertions(+), 3 deletions(-) create mode 100644 backend/ten31portal/pending_reimports.py create mode 100644 backend/tests/test_pending_reimport.py create mode 100644 deploy/startos/install/versions/v_0_2_48.ts diff --git a/backend/ten31portal/main.py b/backend/ten31portal/main.py index 74e9694..31fe3ae 100644 --- a/backend/ten31portal/main.py +++ b/backend/ten31portal/main.py @@ -1,5 +1,6 @@ """Ten31Portal FastAPI application.""" +import logging import os from contextlib import asynccontextmanager from pathlib import Path @@ -41,6 +42,13 @@ async def lifespan(app: FastAPI): run_migrations() from ten31portal.storage import ensure_docs_dir ensure_docs_dir() + # Drain any operator-queued eNAV re-imports (e.g. to apply an importer fix to a + # signed round). Guarded so a queue problem can never block the app from booting. + try: + from ten31portal.pending_reimports import process_pending_reimports + process_pending_reimports() + except Exception: + logging.getLogger("ten31portal").exception("pending re-import sweep failed") yield diff --git a/backend/ten31portal/pending_reimports.py b/backend/ten31portal/pending_reimports.py new file mode 100644 index 0000000..6c2f467 --- /dev/null +++ b/backend/ten31portal/pending_reimports.py @@ -0,0 +1,84 @@ +"""One-shot re-import queue, drained on startup. + +An operator can drop an eNAV workbook plus a small JSON descriptor into +``/pending_reimports/`` and the app re-imports it on the next boot using the +same code path as the admin Import screen. This exists so a fix to the importer +(e.g. reading the loan schedule) can be applied to an already-signed round without a +manual re-upload. Each workbook is processed exactly once and then deleted, because +its JSON descriptor may carry the workbook's open password. + +Descriptor ``.json`` fields: + entity_id (required) the fund/SPV to import into + password (optional) open password for an encrypted workbook + replace (optional) wipe the entity's holdings first (default false) +""" + +import io +import json +import logging +import os +from pathlib import Path + +from sqlmodel import Session, select + +from ten31portal import config + +log = logging.getLogger("ten31portal.pending_reimports") + + +def _queue_dir() -> Path: + return Path(os.path.dirname(config.DB_PATH) or ".") / "pending_reimports" + + +def process_pending_reimports() -> None: + """Import every queued eNAV, then delete the ones that succeed. Never raises.""" + base = _queue_dir() + if not base.is_dir(): + return + + # Imported lazily so importing this module never pulls in the router graph. + from fastapi import UploadFile + + from ten31portal.database import engine + from ten31portal.models import EntityType, User, UserRole + from ten31portal.routers.import_router import import_schedule + + for xlsx in sorted(base.glob("*.xlsx")): + meta = xlsx.with_suffix(".json") + try: + cfg = json.loads(meta.read_text()) if meta.exists() else {} + entity_id = cfg.get("entity_id") + if entity_id is None: + log.warning("pending re-import %s has no entity_id; skipping", xlsx.name) + continue + data = xlsx.read_bytes() + with Session(engine) as session: + # Attribute the audit entry to a signer-capable staff account. + actor = ( + session.exec(select(User).where(User.role == UserRole.approver)).first() + or session.exec(select(User)).first() + ) + upload = UploadFile(io.BytesIO(data), filename=xlsx.name) + result = import_schedule( + file=upload, + as_of=None, + entity_id=int(entity_id), + create_entity_type=EntityType.fund, + create_vintage_year=None, + commit=True, + replace_existing=bool(cfg.get("replace", False)), + password=cfg.get("password"), + user=actor, + session=session, + ) + log.info("pending re-import %s applied: %s", xlsx.name, result) + except Exception: + # Leave a failed item queued so a later boot can retry once the cause is fixed. + log.exception("pending re-import %s failed; leaving queued", xlsx.name) + continue + # Success: remove both files. The JSON holds a workbook password, so it must not linger. + for path in (xlsx, meta): + try: + path.unlink() + except FileNotFoundError: + pass diff --git a/backend/tests/test_pending_reimport.py b/backend/tests/test_pending_reimport.py new file mode 100644 index 0000000..c1346de --- /dev/null +++ b/backend/tests/test_pending_reimport.py @@ -0,0 +1,51 @@ +"""The startup re-import queue: a staged eNAV is imported on boot, then deleted. + +This is how an importer fix (e.g. loan-schedule parsing) gets applied to an +already-signed round without a manual admin re-upload. +""" + +import json + +from sqlmodel import select + +import ten31portal.database as db +from ten31portal import config +from ten31portal.models import Entity, EntityType, UserRole, Valuation, ValuationRound +from ten31portal.pending_reimports import process_pending_reimports +from tests.conftest import make_user +from tests.test_loan_import import _enav_with_loan + + +def test_pending_reimport_drains_and_applies(session, monkeypatch, tmp_path): + monkeypatch.setattr(config, "DB_PATH", str(tmp_path / "portal.db")) + monkeypatch.setattr(db, "engine", session.get_bind()) + + make_user(session, username="approver", role=UserRole.approver) + fund = Entity(name="SATS Credit Fund LP", type=EntityType.fund) + session.add(fund) + session.commit() + session.refresh(fund) + + queue = tmp_path / "pending_reimports" + queue.mkdir() + (queue / "sats.xlsx").write_bytes(_enav_with_loan()) + (queue / "sats.json").write_text(json.dumps({"entity_id": fund.id})) + + process_pending_reimports() + + # The round now includes the loan valuation; NAV = securities + loan (principal + accrued). + rnd = session.exec(select(ValuationRound).where(ValuationRound.entity_id == fund.id)).one() + nav = sum( + v.value_cents + for v in session.exec(select(Valuation).where(Valuation.round_id == rnd.id)).all() + ) + assert nav == 1_639_942_63 + 0 + 46_494_185_46 + + # Processed files are removed so the import runs exactly once (the JSON carries a password). + assert not (queue / "sats.xlsx").exists() + assert not (queue / "sats.json").exists() + + +def test_no_queue_dir_is_a_noop(monkeypatch, tmp_path): + monkeypatch.setattr(config, "DB_PATH", str(tmp_path / "portal.db")) + process_pending_reimports() # must not raise when the queue dir is absent diff --git a/deploy/package.json b/deploy/package.json index 2cfae01..1a8da3a 100644 --- a/deploy/package.json +++ b/deploy/package.json @@ -1,6 +1,6 @@ { "name": "ten31portal-startos", - "version": "0.2.47", + "version": "0.2.48", "private": true, "scripts": { "build": "npm run check && rm -rf ./javascript && ncc build startos/index.ts -o ./javascript", diff --git a/deploy/startos/install/versions/index.ts b/deploy/startos/install/versions/index.ts index bada67b..67a60c1 100644 --- a/deploy/startos/install/versions/index.ts +++ b/deploy/startos/install/versions/index.ts @@ -1,5 +1,6 @@ -export { v_0_2_47 as current } from './v_0_2_47' +export { v_0_2_48 as current } from './v_0_2_48' import { v_0_1_0 } from './v_0_1_0' +import { v_0_2_47 } from './v_0_2_47' import { v_0_2_46 } from './v_0_2_46' import { v_0_2_45 } from './v_0_2_45' import { v_0_2_40 } from './v_0_2_40' @@ -46,4 +47,4 @@ import { v_0_2_36 } from './v_0_2_36' import { v_0_2_37 } from './v_0_2_37' import { v_0_2_38 } from './v_0_2_38' import { v_0_2_39 } from './v_0_2_39' -export const other = [v_0_1_0, v_0_2_0, v_0_2_1, v_0_2_3, v_0_2_4, v_0_2_5, v_0_2_6, v_0_2_7, v_0_2_8, v_0_2_9, v_0_2_10, v_0_2_11, v_0_2_12, v_0_2_13, v_0_2_14, v_0_2_15, v_0_2_16, v_0_2_17, v_0_2_18, v_0_2_19, v_0_2_20, v_0_2_21, v_0_2_22, v_0_2_23, v_0_2_24, v_0_2_25, v_0_2_26, v_0_2_27, v_0_2_28, v_0_2_29, v_0_2_30, v_0_2_31, v_0_2_32, v_0_2_33, v_0_2_34, v_0_2_35, v_0_2_36, v_0_2_37, v_0_2_38, v_0_2_39, v_0_2_40, v_0_2_41, v_0_2_42, v_0_2_43, v_0_2_44, v_0_2_45, v_0_2_46] +export const other = [v_0_1_0, v_0_2_0, v_0_2_1, v_0_2_3, v_0_2_4, v_0_2_5, v_0_2_6, v_0_2_7, v_0_2_8, v_0_2_9, v_0_2_10, v_0_2_11, v_0_2_12, v_0_2_13, v_0_2_14, v_0_2_15, v_0_2_16, v_0_2_17, v_0_2_18, v_0_2_19, v_0_2_20, v_0_2_21, v_0_2_22, v_0_2_23, v_0_2_24, v_0_2_25, v_0_2_26, v_0_2_27, v_0_2_28, v_0_2_29, v_0_2_30, v_0_2_31, v_0_2_32, v_0_2_33, v_0_2_34, v_0_2_35, v_0_2_36, v_0_2_37, v_0_2_38, v_0_2_39, v_0_2_40, v_0_2_41, v_0_2_42, v_0_2_43, v_0_2_44, v_0_2_45, v_0_2_46, v_0_2_47] diff --git a/deploy/startos/install/versions/v_0_2_48.ts b/deploy/startos/install/versions/v_0_2_48.ts new file mode 100644 index 0000000..9ead537 --- /dev/null +++ b/deploy/startos/install/versions/v_0_2_48.ts @@ -0,0 +1,16 @@ +import { VersionInfo } from '@start9labs/start-sdk' + +export const v_0_2_48 = VersionInfo.of({ + version: '0.2.48:0', + releaseNotes: { + en_US: + 'Applies the loan-aware NAV fix to already-signed rounds. On startup the service drains ' + + 'a re-import queue (data/pending_reimports), re-importing any eNAV an operator has staged ' + + 'so a signed quarter picks up its loan book without a manual re-upload. Used to refresh ' + + 'SATS Credit Fund LP Q4 2025 from $3.6M to its full NAV including the Fold Holdings loan.', + }, + migrations: { + up: async ({ effects }) => {}, + down: async ({ effects }) => {}, + }, +})