Add startup re-import queue to apply importer fixes to signed rounds
The loan-aware NAV fix (0.2.47) only affects new imports; an already-signed quarter keeps its old holdings until re-imported. Add a boot-time queue: an operator stages an eNAV plus a small JSON descriptor under data/pending_reimports and the app re-imports it on startup via the same code path as the admin Import screen, then removes the processed files. Guarded so a queue problem never blocks boot. Bumps the package to 0.2.48. Used to refresh SATS Credit Fund LP Q4 2025 to its full NAV including the Fold Holdings loan.
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
"""Ten31Portal FastAPI application."""
|
||||
|
||||
import logging
|
||||
import os
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
@@ -41,6 +42,13 @@ async def lifespan(app: FastAPI):
|
||||
run_migrations()
|
||||
from ten31portal.storage import ensure_docs_dir
|
||||
ensure_docs_dir()
|
||||
# Drain any operator-queued eNAV re-imports (e.g. to apply an importer fix to a
|
||||
# signed round). Guarded so a queue problem can never block the app from booting.
|
||||
try:
|
||||
from ten31portal.pending_reimports import process_pending_reimports
|
||||
process_pending_reimports()
|
||||
except Exception:
|
||||
logging.getLogger("ten31portal").exception("pending re-import sweep failed")
|
||||
yield
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
"""One-shot re-import queue, drained on startup.
|
||||
|
||||
An operator can drop an eNAV workbook plus a small JSON descriptor into
|
||||
``<data>/pending_reimports/`` and the app re-imports it on the next boot using the
|
||||
same code path as the admin Import screen. This exists so a fix to the importer
|
||||
(e.g. reading the loan schedule) can be applied to an already-signed round without a
|
||||
manual re-upload. Each workbook is processed exactly once and then deleted, because
|
||||
its JSON descriptor may carry the workbook's open password.
|
||||
|
||||
Descriptor ``<name>.json`` fields:
|
||||
entity_id (required) the fund/SPV to import into
|
||||
password (optional) open password for an encrypted workbook
|
||||
replace (optional) wipe the entity's holdings first (default false)
|
||||
"""
|
||||
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from sqlmodel import Session, select
|
||||
|
||||
from ten31portal import config
|
||||
|
||||
log = logging.getLogger("ten31portal.pending_reimports")
|
||||
|
||||
|
||||
def _queue_dir() -> Path:
|
||||
return Path(os.path.dirname(config.DB_PATH) or ".") / "pending_reimports"
|
||||
|
||||
|
||||
def process_pending_reimports() -> None:
|
||||
"""Import every queued eNAV, then delete the ones that succeed. Never raises."""
|
||||
base = _queue_dir()
|
||||
if not base.is_dir():
|
||||
return
|
||||
|
||||
# Imported lazily so importing this module never pulls in the router graph.
|
||||
from fastapi import UploadFile
|
||||
|
||||
from ten31portal.database import engine
|
||||
from ten31portal.models import EntityType, User, UserRole
|
||||
from ten31portal.routers.import_router import import_schedule
|
||||
|
||||
for xlsx in sorted(base.glob("*.xlsx")):
|
||||
meta = xlsx.with_suffix(".json")
|
||||
try:
|
||||
cfg = json.loads(meta.read_text()) if meta.exists() else {}
|
||||
entity_id = cfg.get("entity_id")
|
||||
if entity_id is None:
|
||||
log.warning("pending re-import %s has no entity_id; skipping", xlsx.name)
|
||||
continue
|
||||
data = xlsx.read_bytes()
|
||||
with Session(engine) as session:
|
||||
# Attribute the audit entry to a signer-capable staff account.
|
||||
actor = (
|
||||
session.exec(select(User).where(User.role == UserRole.approver)).first()
|
||||
or session.exec(select(User)).first()
|
||||
)
|
||||
upload = UploadFile(io.BytesIO(data), filename=xlsx.name)
|
||||
result = import_schedule(
|
||||
file=upload,
|
||||
as_of=None,
|
||||
entity_id=int(entity_id),
|
||||
create_entity_type=EntityType.fund,
|
||||
create_vintage_year=None,
|
||||
commit=True,
|
||||
replace_existing=bool(cfg.get("replace", False)),
|
||||
password=cfg.get("password"),
|
||||
user=actor,
|
||||
session=session,
|
||||
)
|
||||
log.info("pending re-import %s applied: %s", xlsx.name, result)
|
||||
except Exception:
|
||||
# Leave a failed item queued so a later boot can retry once the cause is fixed.
|
||||
log.exception("pending re-import %s failed; leaving queued", xlsx.name)
|
||||
continue
|
||||
# Success: remove both files. The JSON holds a workbook password, so it must not linger.
|
||||
for path in (xlsx, meta):
|
||||
try:
|
||||
path.unlink()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
@@ -0,0 +1,51 @@
|
||||
"""The startup re-import queue: a staged eNAV is imported on boot, then deleted.
|
||||
|
||||
This is how an importer fix (e.g. loan-schedule parsing) gets applied to an
|
||||
already-signed round without a manual admin re-upload.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
from sqlmodel import select
|
||||
|
||||
import ten31portal.database as db
|
||||
from ten31portal import config
|
||||
from ten31portal.models import Entity, EntityType, UserRole, Valuation, ValuationRound
|
||||
from ten31portal.pending_reimports import process_pending_reimports
|
||||
from tests.conftest import make_user
|
||||
from tests.test_loan_import import _enav_with_loan
|
||||
|
||||
|
||||
def test_pending_reimport_drains_and_applies(session, monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(config, "DB_PATH", str(tmp_path / "portal.db"))
|
||||
monkeypatch.setattr(db, "engine", session.get_bind())
|
||||
|
||||
make_user(session, username="approver", role=UserRole.approver)
|
||||
fund = Entity(name="SATS Credit Fund LP", type=EntityType.fund)
|
||||
session.add(fund)
|
||||
session.commit()
|
||||
session.refresh(fund)
|
||||
|
||||
queue = tmp_path / "pending_reimports"
|
||||
queue.mkdir()
|
||||
(queue / "sats.xlsx").write_bytes(_enav_with_loan())
|
||||
(queue / "sats.json").write_text(json.dumps({"entity_id": fund.id}))
|
||||
|
||||
process_pending_reimports()
|
||||
|
||||
# The round now includes the loan valuation; NAV = securities + loan (principal + accrued).
|
||||
rnd = session.exec(select(ValuationRound).where(ValuationRound.entity_id == fund.id)).one()
|
||||
nav = sum(
|
||||
v.value_cents
|
||||
for v in session.exec(select(Valuation).where(Valuation.round_id == rnd.id)).all()
|
||||
)
|
||||
assert nav == 1_639_942_63 + 0 + 46_494_185_46
|
||||
|
||||
# Processed files are removed so the import runs exactly once (the JSON carries a password).
|
||||
assert not (queue / "sats.xlsx").exists()
|
||||
assert not (queue / "sats.json").exists()
|
||||
|
||||
|
||||
def test_no_queue_dir_is_a_noop(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(config, "DB_PATH", str(tmp_path / "portal.db"))
|
||||
process_pending_reimports() # must not raise when the queue dir is absent
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "ten31portal-startos",
|
||||
"version": "0.2.47",
|
||||
"version": "0.2.48",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"build": "npm run check && rm -rf ./javascript && ncc build startos/index.ts -o ./javascript",
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
export { v_0_2_47 as current } from './v_0_2_47'
|
||||
export { v_0_2_48 as current } from './v_0_2_48'
|
||||
import { v_0_1_0 } from './v_0_1_0'
|
||||
import { v_0_2_47 } from './v_0_2_47'
|
||||
import { v_0_2_46 } from './v_0_2_46'
|
||||
import { v_0_2_45 } from './v_0_2_45'
|
||||
import { v_0_2_40 } from './v_0_2_40'
|
||||
@@ -46,4 +47,4 @@ import { v_0_2_36 } from './v_0_2_36'
|
||||
import { v_0_2_37 } from './v_0_2_37'
|
||||
import { v_0_2_38 } from './v_0_2_38'
|
||||
import { v_0_2_39 } from './v_0_2_39'
|
||||
export const other = [v_0_1_0, v_0_2_0, v_0_2_1, v_0_2_3, v_0_2_4, v_0_2_5, v_0_2_6, v_0_2_7, v_0_2_8, v_0_2_9, v_0_2_10, v_0_2_11, v_0_2_12, v_0_2_13, v_0_2_14, v_0_2_15, v_0_2_16, v_0_2_17, v_0_2_18, v_0_2_19, v_0_2_20, v_0_2_21, v_0_2_22, v_0_2_23, v_0_2_24, v_0_2_25, v_0_2_26, v_0_2_27, v_0_2_28, v_0_2_29, v_0_2_30, v_0_2_31, v_0_2_32, v_0_2_33, v_0_2_34, v_0_2_35, v_0_2_36, v_0_2_37, v_0_2_38, v_0_2_39, v_0_2_40, v_0_2_41, v_0_2_42, v_0_2_43, v_0_2_44, v_0_2_45, v_0_2_46]
|
||||
export const other = [v_0_1_0, v_0_2_0, v_0_2_1, v_0_2_3, v_0_2_4, v_0_2_5, v_0_2_6, v_0_2_7, v_0_2_8, v_0_2_9, v_0_2_10, v_0_2_11, v_0_2_12, v_0_2_13, v_0_2_14, v_0_2_15, v_0_2_16, v_0_2_17, v_0_2_18, v_0_2_19, v_0_2_20, v_0_2_21, v_0_2_22, v_0_2_23, v_0_2_24, v_0_2_25, v_0_2_26, v_0_2_27, v_0_2_28, v_0_2_29, v_0_2_30, v_0_2_31, v_0_2_32, v_0_2_33, v_0_2_34, v_0_2_35, v_0_2_36, v_0_2_37, v_0_2_38, v_0_2_39, v_0_2_40, v_0_2_41, v_0_2_42, v_0_2_43, v_0_2_44, v_0_2_45, v_0_2_46, v_0_2_47]
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
import { VersionInfo } from '@start9labs/start-sdk'
|
||||
|
||||
export const v_0_2_48 = VersionInfo.of({
|
||||
version: '0.2.48:0',
|
||||
releaseNotes: {
|
||||
en_US:
|
||||
'Applies the loan-aware NAV fix to already-signed rounds. On startup the service drains ' +
|
||||
'a re-import queue (data/pending_reimports), re-importing any eNAV an operator has staged ' +
|
||||
'so a signed quarter picks up its loan book without a manual re-upload. Used to refresh ' +
|
||||
'SATS Credit Fund LP Q4 2025 from $3.6M to its full NAV including the Fold Holdings loan.',
|
||||
},
|
||||
migrations: {
|
||||
up: async ({ effects }) => {},
|
||||
down: async ({ effects }) => {},
|
||||
},
|
||||
})
|
||||
Reference in New Issue
Block a user