"""SEC EDGAR ingestion (§4.1).

Hits the official data.sec.gov / www.sec.gov APIs directly (free, keyless, full history).
Two hard requirements:
  - a descriptive User-Agent (SEC 403s requests without one) — from config.edgar_user_agent.
  - ≤10 requests/sec aggregate — enforced by a min-interval throttle here.

Supports an explicit date range AND historical shards (filings.files[]), so the §7.1 backtest can
reach 2022–2023 filings, not just the most-recent ~1000.
"""
from __future__ import annotations

import hashlib
import sqlite3
import time
from typing import Iterator

import requests

_FILING_COLS = ("accessionNumber", "form", "filingDate", "primaryDocument", "primaryDocDescription")


class EdgarClient:
    BASE_DATA = "https://data.sec.gov"
    BASE_WWW = "https://www.sec.gov"

    def __init__(self, user_agent: str, *, min_interval: float = 0.12) -> None:
        if not user_agent or "@" not in user_agent:
            raise ValueError("EDGAR requires a descriptive User-Agent with contact email (config.edgar_user_agent)")
        self.s = requests.Session()
        self.s.headers.update({"User-Agent": user_agent, "Accept-Encoding": "gzip, deflate"})
        self.min_interval = min_interval
        self._last = 0.0
        self._tickers: dict[str, int] | None = None

    def _throttle(self) -> None:
        dt = time.monotonic() - self._last
        if dt < self.min_interval:
            time.sleep(self.min_interval - dt)
        self._last = time.monotonic()

    def _get(self, url: str) -> requests.Response:
        self._throttle()
        r = self.s.get(url, timeout=30)
        r.raise_for_status()
        return r

    # ---- ticker → CIK ----
    def ticker_map(self) -> dict[str, int]:
        if self._tickers is None:
            data = self._get(f"{self.BASE_WWW}/files/company_tickers.json").json()
            self._tickers = {row["ticker"].upper(): int(row["cik_str"]) for row in data.values()}
        return self._tickers

    def cik_for(self, ticker: str) -> int | None:
        return self.ticker_map().get(ticker.upper())

    # ---- filings ----
    def _iter_array(self, block: dict, forms, since, until) -> Iterator[dict]:
        arrays = [block.get(c, []) for c in _FILING_COLS]
        for acc, form, fdate, pdoc, pdesc in zip(*arrays):
            if forms and form not in forms:
                continue
            if since and fdate < since:
                continue
            if until and fdate > until:
                continue
            yield {"accession": acc, "form": form, "filing_date": fdate,
                   "primary_document": pdoc, "description": pdesc}

    def iter_filings(
        self,
        cik: int,
        *,
        forms: tuple[str, ...] = ("10-K", "10-Q", "8-K"),
        since: str | None = None,
        until: str | None = None,
    ) -> Iterator[dict]:
        """Yield filing descriptors. Pulls the inline 'recent' block AND any historical shards whose
        date window overlaps [since, until] — required to reach the backtest era for active filers."""
        sub = self._get(f"{self.BASE_DATA}/submissions/CIK{cik:010d}.json").json()
        recent = sub.get("filings", {}).get("recent", {})
        for f in self._iter_array(recent, forms, since, until):
            yield self._with_url(cik, f)
        for shard in sub.get("filings", {}).get("files", []):
            # shard has filingFrom / filingTo; skip shards entirely outside the window.
            if until and shard.get("filingFrom", "") > until:
                continue
            if since and shard.get("filingTo", "9999") < since:
                continue
            block = self._get(f"{self.BASE_DATA}/submissions/{shard['name']}").json()
            for f in self._iter_array(block, forms, since, until):
                yield self._with_url(cik, f)

    def _with_url(self, cik: int, f: dict) -> dict:
        acc_nodash = f["accession"].replace("-", "")
        f["cik"] = cik
        f["url"] = f"{self.BASE_WWW}/Archives/edgar/data/{cik}/{acc_nodash}/{f['primary_document']}"
        return f

    def fetch_html(self, filing: dict) -> str:
        return self._get(filing["url"]).text


# Domestic annual/quarterly + foreign-private-issuer equivalents. 20-F (foreign annual, e.g. TSM/IREN),
# 40-F (Canadian annual, e.g. CCJ). 8-K/6-K (current reports) excluded by default — low claim yield.
HIGH_YIELD_FORMS = ("10-K", "10-Q", "20-F", "40-F")


def ingest_filings(
    conn: sqlite3.Connection,
    client: EdgarClient,
    *,
    source_id: str,
    ticker: str,
    since: str | None = None,
    until: str | None = None,
    forms: tuple[str, ...] = HIGH_YIELD_FORMS,
    prompt_version: str = "extract-v0",
) -> tuple[int, int]:
    """Insert filing documents and enqueue 'extract' jobs. Filings are text → no transcription;
    they go straight to extraction (the extract worker fetches + strips the HTML later). Default
    forms cover both domestic (10-K/10-Q) and foreign-private-issuer (20-F/40-F) filers.
    Returns (new_documents, new_jobs). Idempotent on (source_id, accession)."""
    from ..backfill import queue

    cik = client.cik_for(ticker)
    if cik is None:
        raise ValueError(f"No CIK found for ticker {ticker!r}")
    n_docs = n_jobs = 0
    for f in client.iter_filings(cik, forms=forms, since=since, until=until):
        doc_id = f"edgar:{f['accession']}"
        cur = conn.execute(
            """INSERT OR IGNORE INTO documents (doc_id, source_id, kind, external_id, url, title, date)
               VALUES (?,?,?,?,?,?,?)""",
            (doc_id, source_id, "filing", f["accession"], f["url"],
             f"{ticker} {f['form']} {f['filing_date']}", f["filing_date"]),
        )
        conn.commit()
        if not cur.rowcount:
            continue
        n_docs += 1
        h = hashlib.sha256(f"{doc_id}|{prompt_version}".encode()).hexdigest()
        # priority 50: filings are high-info-density (§4.1) → ahead of podcasts (100)
        if queue.enqueue(conn, job_type="extract", target_id=doc_id, input_hash=h,
                         parent_doc_id=doc_id, priority=50) is not None:
            n_jobs += 1
    return n_docs, n_jobs