Add daily-digest Phase A: per-package SMTP send + admin test endpoint (v0.1.0:75)

Groundwork for the daily activity digest: give the CRM an outbound mail path.
Today nothing leaves the box (Gmail capture + drafts only), so this adds a
dedicated, per-package SMTP account independent of any StartOS system-wide SMTP.

- configureDigestSmtp Start9 action: writes host/port/from/username/password/
  security to /data/secrets/smtp/* (password piped over stdin, never argv/env;
  per-field files, owner-only) — mirrors the setAnthropicApiKey pattern.
- docker_entrypoint.sh reads those at boot and exports SMTP_* (operator env wins).
- backend/smtp_send.py: stdlib smtplib wrapper reading SMTP_* (one code path for
  dev .env and the box); starttls/tls/none modes.
- POST /api/admin/digest/test-email (admin-only): proves the pipe. Recipients are
  restricted to the active-admin set — an arbitrary `to` is rejected, so the
  endpoint is not an open relay; send failures are logged, not echoed (an SMTP
  auth error can carry the credential).
- Tests: test_smtp_send.py (sender), test_smtp_endpoint.py (gating + relay
  restriction + no-leak). 18/18 backend green; s9pk typechecks.

Analysis/summarization for the digest body (Phase B) will run on Spark, never
Claude — the digest is deliberately un-anonymized. Decisions + Phase B plan in
ROADMAP.md.
This commit is contained in:
Keysat
2026-06-15 18:33:06 -05:00
parent ecfc5d968a
commit 2758ac81d3
13 changed files with 765 additions and 14 deletions
+63
View File
@@ -1919,6 +1919,8 @@ class CRMHandler(BaseHTTPRequestHandler):
return self.handle_admin_create_user(user, body)
if path == '/api/admin/reset-all-data':
return self.handle_admin_reset_all_data(user, body)
if path == '/api/admin/digest/test-email':
return self.handle_admin_send_test_email(user, body)
if path == '/api/fundraising/backup':
return self.handle_backup_fundraising_state(user)
if path == '/api/fundraising/restore-preview':
@@ -4234,6 +4236,67 @@ class CRMHandler(BaseHTTPRequestHandler):
}
})
def handle_admin_send_test_email(self, user, body):
"""Send a test email through the configured digest SMTP account, to prove
the outbound pipe before the daily digest itself is built (Phase A)."""
if not require_admin(user):
return self.send_error_json("Admin only", 403)
from smtp_send import send_email, smtp_configured, SMTPNotConfigured
if not smtp_configured():
return self.send_error_json(
"SMTP is not configured. Set it via the 'Configure Digest SMTP' "
"Start9 action (or SMTP_* in .env for dev), then restart.", 400)
# Recipients are restricted to the active-admin set (the real digest
# audience). An explicit `to` may NARROW to specific admins but can never
# introduce an outside address — this endpoint is not an open mail relay.
conn = get_db()
try:
rows = conn.execute(
"SELECT email FROM users WHERE role = 'admin' AND is_active = 1 "
"AND email IS NOT NULL AND TRIM(email) != ''"
).fetchall()
finally:
conn.close()
admin_emails = [str(r['email']).strip() for r in rows if str(r['email']).strip()]
admin_lower = {e.lower() for e in admin_emails}
to = body.get('to')
if to:
requested = [to] if isinstance(to, str) else list(to)
requested = [str(e).strip() for e in requested if str(e).strip()]
outside = [e for e in requested if e.lower() not in admin_lower]
if outside:
return self.send_error_json(
"Test email may only go to an active admin address; "
f"not allowed: {', '.join(outside)}", 400)
recipients = requested
else:
recipients = admin_emails
if not recipients:
return self.send_error_json(
"No recipient: give an active admin an email address first.", 400)
subject = "Ten31 CRM — test digest email"
email_body = (
"This is a test message from the Ten31 CRM daily-digest mailer.\n\n"
f"Triggered by {user.get('full_name') or user.get('user_id')} at {now()}.\n\n"
"If you received this, outbound SMTP works and the daily digest can be "
"delivered to this address."
)
try:
result = send_email(recipients, subject, email_body)
except SMTPNotConfigured as exc:
return self.send_error_json(f"SMTP not configured: {exc}", 400)
except Exception as exc:
# Never echo the exception to the client — an SMTP auth error can carry
# the server's reply (and potentially the credential). Log it instead.
print(f"[smtp] test send failed: {type(exc).__name__}: {exc}", file=sys.stderr)
return self.send_error_json("Send failed — see server logs for details.", 502)
return self.send_json({"data": {"status": "sent", **result}})
def handle_list_audit_log(self, user, params):
if not require_admin(user):
return self.send_error_json("Admin access required", 403)
+99
View File
@@ -0,0 +1,99 @@
"""Minimal outbound SMTP sender for the CRM (daily digest, test sends).
Config comes ONLY from SMTP_* environment variables. Two ways those get set:
* dev / bare run: a local .env (see .env.example).
* Start9 box: docker_entrypoint.sh reads the files the "Configure Digest SMTP"
StartOS action writes under /data/secrets/smtp/ and exports them as env.
The backend never reads those files directly, so dev and prod share one path.
This is the package's OWN dedicated mailbox (per-package custom SMTP) — it is
independent of any StartOS system-wide SMTP account; nothing here calls into the
platform. Stdlib only (smtplib/ssl/email), consistent with the rest of runtime.
"""
import os
import smtplib
import ssl
from email.message import EmailMessage
class SMTPNotConfigured(Exception):
"""Raised when SMTP_* env is absent — callers turn this into a clear 'not
configured' response rather than a 500."""
def smtp_configured():
return bool(os.environ.get("SMTP_HOST", "").strip())
def load_smtp_config():
host = os.environ.get("SMTP_HOST", "").strip()
if not host:
raise SMTPNotConfigured("SMTP_HOST is not set")
# Port/security come from a free-text action field; normalize defensively.
try:
port = int(str(os.environ.get("SMTP_PORT", "") or "587").strip())
except ValueError:
port = 587
security = (os.environ.get("SMTP_SECURITY", "") or "starttls").strip().lower()
if security not in ("starttls", "tls", "none"):
security = "starttls"
return {
"host": host,
"port": port,
"from_addr": os.environ.get("SMTP_FROM", "").strip(),
"username": os.environ.get("SMTP_USERNAME", "").strip(),
"password": os.environ.get("SMTP_PASSWORD", ""),
"security": security,
}
def _connect(cfg, timeout):
"""Open an authenticated SMTP connection per the configured security mode.
'tls' = implicit TLS (SMTPS, usually 465); 'starttls' = upgrade on 587;
'none' = plaintext (for a LAN relay that does its own transport security)."""
if cfg["security"] == "tls":
ctx = ssl.create_default_context()
server = smtplib.SMTP_SSL(cfg["host"], cfg["port"], timeout=timeout, context=ctx)
else:
server = smtplib.SMTP(cfg["host"], cfg["port"], timeout=timeout)
server.ehlo()
if cfg["security"] == "starttls":
server.starttls(context=ssl.create_default_context())
server.ehlo()
if cfg["username"]:
server.login(cfg["username"], cfg["password"])
return server
def send_email(to_addrs, subject, body, *, html=None, cfg=None, timeout=30):
"""Send one message. `to_addrs` is a str or list; `body` is plain text and
`html` an optional HTML alternative. Returns {'sent_to', 'from'} on success;
raises SMTPNotConfigured / ValueError / smtplib.SMTPException otherwise."""
cfg = cfg or load_smtp_config()
if isinstance(to_addrs, str):
to_addrs = [to_addrs]
to_addrs = [a for a in (str(x).strip() for x in to_addrs) if a]
if not to_addrs:
raise ValueError("no recipients")
from_addr = cfg["from_addr"] or cfg["username"]
if not from_addr:
raise SMTPNotConfigured("no SMTP_FROM or SMTP_USERNAME to use as sender")
msg = EmailMessage()
msg["From"] = from_addr
msg["To"] = ", ".join(to_addrs)
msg["Subject"] = subject
msg.set_content(body)
if html:
msg.add_alternative(html, subtype="html")
server = _connect(cfg, timeout)
try:
server.send_message(msg, from_addr=from_addr, to_addrs=to_addrs)
finally:
try:
server.quit()
except Exception:
pass
return {"sent_to": to_addrs, "from": from_addr}
+171
View File
@@ -0,0 +1,171 @@
#!/usr/bin/env python3
"""Endpoint tests for POST /api/admin/digest/test-email (handle_admin_send_test_email).
Boots the REAL server in-process against a throwaway DB, monkeypatches the SMTP
sender (no network), and proves the security-relevant contract:
* admin-gated: 401 without a token, 403 for a non-admin.
* recipients are restricted to the active-admin set — an arbitrary `to` is
rejected (the endpoint is NOT an open relay), an admin `to` is allowed,
and the default audience is every active admin.
* a send failure does not echo the exception (which could carry the SMTP
credential) back to the caller.
Run: cd backend && python3 test_smtp_endpoint.py
"""
import http.client
import json
import os
import sys
import tempfile
import threading
from http.server import ThreadingHTTPServer
_BASE = tempfile.mkdtemp()
_FRONTEND = os.path.join(_BASE, "frontend")
os.makedirs(os.path.join(_FRONTEND, "assets"))
_DATA = os.path.join(_BASE, "data")
os.makedirs(_DATA)
with open(os.path.join(_FRONTEND, "index.html"), "w") as f:
f.write("<!doctype html><title>crm</title>")
os.environ["CRM_FRONTEND_DIR"] = _FRONTEND
os.environ["CRM_DATA_DIR"] = _DATA
os.environ["CRM_DB_PATH"] = os.path.join(_DATA, "crm.db")
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import server # noqa: E402
import smtp_send # noqa: E402
FAILS = []
# ── fake sender: record calls, optionally raise (no network) ──
SENT = []
RAISE = {"exc": None}
def fake_send(to_addrs, subject, body, **kw):
if RAISE["exc"] is not None:
raise RAISE["exc"]
SENT.append({"to": list(to_addrs), "subject": subject})
return {"sent_to": list(to_addrs), "from": "digest@ten31.test"}
smtp_send.send_email = fake_send
smtp_send.smtp_configured = lambda: True
def check(cond, msg):
print((" PASS " if cond else " FAIL ") + msg)
if not cond:
FAILS.append(msg)
class _Quiet(server.CRMHandler):
def log_message(self, *a):
pass
def _post(port, path, token=None, body=None):
conn = http.client.HTTPConnection("127.0.0.1", port, timeout=10)
headers = {"Content-Type": "application/json"}
if token:
headers["Authorization"] = "Bearer " + token
conn.request("POST", path, body=json.dumps(body or {}), headers=headers)
resp = conn.getresponse()
raw = resp.read().decode("utf-8", "replace")
conn.close()
try:
parsed = json.loads(raw)
except Exception:
parsed = None
return resp.status, raw, parsed
def main():
server.init_db()
# Admin = first registered user. A member is inserted directly (self-register
# is disabled after the first user) so get_user resolves a real active row.
httpd = ThreadingHTTPServer(("127.0.0.1", 0), _Quiet)
port = httpd.server_address[1]
threading.Thread(target=httpd.serve_forever, daemon=True).start()
try:
st, raw, j = _post(port, "/api/auth/register", body={
"username": "admin", "email": "admin@ten31.test",
"password": "password123", "full_name": "Admin User"})
check(st == 201 and j and j.get("token"), f"register first user as admin (got {st})")
admin_token = j["token"]
admin_email = j["user"]["email"]
conn = server.get_db()
conn.execute(
"INSERT INTO users (id, username, email, password_hash, full_name, role, is_active) "
"VALUES (?,?,?,?,?,?,1)",
("member-1", "member1", "member1@ten31.test",
server.hash_password("password123"), "Member One", "member"))
conn.commit()
conn.close()
member_token = server.create_token("member-1", "member1", "member")
path = "/api/admin/digest/test-email"
# 1. unauthenticated -> 401, sender untouched
SENT.clear()
st, raw, j = _post(port, path)
check(st == 401, f"no token -> 401 (got {st})")
check(not SENT, "no token: sender not called")
# 2. non-admin -> 403
SENT.clear()
st, raw, j = _post(port, path, token=member_token)
check(st == 403, f"member -> 403 (got {st})")
check(not SENT, "member: sender not called")
# 3. admin, no `to` -> 200, default audience = the admin set
SENT.clear()
st, raw, j = _post(port, path, token=admin_token)
check(st == 200, f"admin default -> 200 (got {st})")
check(len(SENT) == 1 and SENT[0]["to"] == [admin_email],
f"default recipients = active admins ({admin_email}); got {SENT}")
# 4. admin, arbitrary outside `to` -> 400, NOT an open relay
SENT.clear()
st, raw, j = _post(port, path, token=admin_token, body={"to": "attacker@evil.com"})
check(st == 400, f"outside `to` -> 400 (got {st})")
check(not SENT, "outside `to`: sender not called (no relay)")
# 5. admin, `to` an admin address (case-insensitive) -> 200
SENT.clear()
st, raw, j = _post(port, path, token=admin_token, body={"to": admin_email.upper()})
check(st == 200, f"admin `to` -> 200 (got {st})")
check(len(SENT) == 1 and SENT[0]["to"] == [admin_email.upper()],
"admin `to` is delivered as given")
# 6. mixed list with one outside address -> 400, all rejected
SENT.clear()
st, raw, j = _post(port, path, token=admin_token,
body={"to": [admin_email, "outsider@evil.com"]})
check(st == 400, f"mixed list with outsider -> 400 (got {st})")
check(not SENT, "mixed list: sender not called")
# 7. send failure must NOT leak the exception text (could carry the credential)
SENT.clear()
secret = "PWLEAK_SsHh99"
RAISE["exc"] = Exception(f"535 auth failed for user with password {secret}")
st, raw, j = _post(port, path, token=admin_token)
RAISE["exc"] = None
check(st == 502, f"send failure -> 502 (got {st})")
check(secret not in raw, "send-failure response does not leak the exception/credential text")
finally:
httpd.shutdown()
print()
if FAILS:
print(f"FAILED ({len(FAILS)}):")
for f in FAILS:
print(f" - {f}")
sys.exit(1)
print("ALL PASS (digest test-email endpoint)")
if __name__ == "__main__":
main()
+151
View File
@@ -0,0 +1,151 @@
#!/usr/bin/env python3
"""Standalone tests for backend/smtp_send.py — config parsing + send paths.
No network: smtplib.SMTP / SMTP_SSL are monkeypatched with a fake that records
the calls. Run directly or via backend/run_tests.py.
"""
import os
import smtplib
import smtp_send
FAILS = []
def check(cond, label):
print(f" {'ok' if cond else 'XX'} {label}")
if not cond:
FAILS.append(label)
class FakeServer:
def __init__(self, kind, log):
self.kind, self.log = kind, log
self.logged_in = None
self.sent = []
def ehlo(self, *a):
self.log.append(("ehlo", self.kind))
def starttls(self, *a, **k):
self.log.append(("starttls", self.kind))
def login(self, user, pw):
self.logged_in = (user, pw)
self.log.append(("login", user))
def send_message(self, msg, from_addr=None, to_addrs=None):
self.sent.append({"from_addr": from_addr, "to_addrs": to_addrs})
self.log.append(("send", from_addr))
def quit(self):
self.log.append(("quit", self.kind))
def install_fakes():
"""Patch smtplib; return (log, holder) where holder['last'] is the server."""
log, holder = [], {}
def mk(kind):
def factory(host, port, timeout=None, context=None):
log.append((kind, host, port))
srv = FakeServer(kind, log)
holder["last"] = srv
return srv
return factory
smtplib.SMTP = mk("SMTP")
smtplib.SMTP_SSL = mk("SMTP_SSL")
return log, holder
def set_env(**kw):
for k in ("SMTP_HOST", "SMTP_PORT", "SMTP_FROM", "SMTP_USERNAME",
"SMTP_PASSWORD", "SMTP_SECURITY"):
os.environ.pop(k, None)
for k, v in kw.items():
os.environ["SMTP_" + k.upper()] = str(v)
def main():
# 1. unconfigured
set_env()
check(not smtp_send.smtp_configured(), "smtp_configured() false when unset")
try:
smtp_send.load_smtp_config(); ok = False
except smtp_send.SMTPNotConfigured:
ok = True
check(ok, "load_smtp_config raises SMTPNotConfigured without host")
# 2. parse + normalize garbage
set_env(host="smtp.x.com", port="nope", security="WEIRD", username="u@x", password="p")
os.environ["SMTP_FROM"] = "f@x"
cfg = smtp_send.load_smtp_config()
check(cfg["host"] == "smtp.x.com", "host parsed")
check(cfg["port"] == 587, "non-numeric port falls back to 587")
check(cfg["security"] == "starttls", "unknown security falls back to starttls")
# 3. STARTTLS path: plain SMTP, upgrade, auth, multi-recipient
log, holder = install_fakes()
set_env(host="h", port="587", security="starttls", username="u@x", password="pw")
os.environ["SMTP_FROM"] = "f@x"
res = smtp_send.send_email(["a@x.com", "b@x.com"], "s", "body")
srv = holder["last"]
check(log[0] == ("SMTP", "h", 587), "starttls connects via plain SMTP")
check(("starttls", "SMTP") in log, "starttls negotiated")
check(srv.logged_in == ("u@x", "pw"), "login with username/password")
check(srv.sent[0]["from_addr"] == "f@x", "from = SMTP_FROM")
check(srv.sent[0]["to_addrs"] == ["a@x.com", "b@x.com"], "recipients passed through")
check(res["sent_to"] == ["a@x.com", "b@x.com"] and res["from"] == "f@x", "result reports send")
# 4. implicit TLS path: SMTP_SSL, no STARTTLS
log, holder = install_fakes()
set_env(host="h", port="465", security="tls", username="u@x", password="pw")
os.environ["SMTP_FROM"] = "f@x"
smtp_send.send_email("a@x.com", "s", "b")
check(log[0][0] == "SMTP_SSL", "tls uses SMTP_SSL")
check(all(e[0] != "starttls" for e in log), "no STARTTLS on implicit TLS")
# 5. security=none, from falls back to username
log, holder = install_fakes()
set_env(host="h", port="25", security="none", username="relay@x", password="")
smtp_send.send_email("a@x.com", "s", "b")
srv = holder["last"]
check(all(e[0] != "starttls" for e in log), "no STARTTLS when security=none")
check(srv.sent[0]["from_addr"] == "relay@x", "from falls back to username")
# 6. no username -> no login attempted
log, holder = install_fakes()
set_env(host="h", port="25", security="none")
os.environ["SMTP_FROM"] = "f@x"
smtp_send.send_email("a@x.com", "s", "b")
check(holder["last"].logged_in is None, "no login when username blank")
# 7. no sender at all -> SMTPNotConfigured
set_env(host="h", port="25", security="none")
try:
smtp_send.send_email("a@x.com", "s", "b"); ok = False
except smtp_send.SMTPNotConfigured:
ok = True
check(ok, "no from/username raises SMTPNotConfigured")
# 8. empty recipients -> ValueError
set_env(host="h", port="587", security="starttls", username="u", password="p")
os.environ["SMTP_FROM"] = "f@x"
try:
smtp_send.send_email([], "s", "b"); ok = False
except ValueError:
ok = True
check(ok, "empty recipients raises ValueError")
print()
if FAILS:
print(f"FAILED ({len(FAILS)}):")
for f in FAILS:
print(f" - {f}")
raise SystemExit(1)
print("ALL PASS (smtp_send)")
if __name__ == "__main__":
main()