fithia2/apps/pipeline/filing_poller/main.py

"""Filing Poller: discover new 8-K/6-K filings via Stock Oracle."""
from __future__ import annotations

import argparse
import asyncio
import datetime as dt
import uuid
from pathlib import Path

import yaml
from sqlalchemy import select

from libs.common.config import get_settings
from libs.common.ids import document_id as make_document_id
from libs.common.ids import new_job_run_id
from libs.common.logging import bind_job_run_id, configure_logging, get_logger
from libs.db.models import Document, IssuerMaster, JobRun, SymbolMaster
from libs.db.session import get_session
from libs.oracle_client import FilingsService, make_oracle_client

logger = get_logger(__name__)


def _load_symbols_from_yaml(path: str) -> list[str]:
    """Load ticker list from a symbols YAML (supports list or dict with 'symbols' key)."""
    raw = yaml.safe_load(Path(path).read_text())
    if isinstance(raw, list):
        items = raw
    elif isinstance(raw, dict):
        items = raw.get("symbols", [])
    else:
        items = []
    return sorted({str(s).upper() for s in items if s})


async def poll_filings(
    run_id: str,
    start_date: str | None = None,
    end_date: str | None = None,
    symbols: list[str] | None = None,
) -> dict[str, int]:
    settings = get_settings()
    if symbols is None:
        symbols = settings.get_symbols()
    app_config = settings.get_app_config()
    form_types = ",".join(app_config.get("pipeline", {}).get("form_types", ["8-K", "6-K"]))

    effective_start = start_date or (dt.date.today() - dt.timedelta(days=7)).isoformat()

    stats = {"seen": 0, "written": 0, "skipped": 0, "errors": 0}

    async with make_oracle_client() as client:
        svc = FilingsService(client)

        async with get_session() as session:
            # Record job start
            job = JobRun(
                job_run_id=uuid.UUID(run_id),
                job_name="filing_poller",
                source_name="oracle",
                run_date=dt.date.today(),
                status="running",
            )
            session.add(job)
            await session.flush()

            # Load issuer/symbol lookup maps
            issuer_result = await session.execute(select(IssuerMaster))
            ticker_to_issuer = {i.ticker: i.issuer_id for i in issuer_result.scalars().all()}

            symbol_result = await session.execute(
                select(SymbolMaster).where(SymbolMaster.is_primary == True)  # noqa: E712
            )
            ticker_to_symbol = {s.ticker: s.symbol_id for s in symbol_result.scalars().all()}

            for ticker in symbols:
                last_exc: Exception | None = None
                response = None
                for attempt in range(3):
                    try:
                        response = await svc.search_filings(
                            ticker,
                            form_type=form_types,
                            start_date=effective_start,
                            end_date=end_date,
                        )
                        last_exc = None
                        break
                    except Exception as exc:
                        last_exc = exc
                        wait = 2 ** attempt  # 1s, 2s, 4s
                        logger.warning(
                            "poll_retry",
                            ticker=ticker,
                            attempt=attempt + 1,
                            wait=wait,
                            error=str(exc) or repr(exc),
                            exc_type=type(exc).__name__,
                        )
                        await asyncio.sleep(wait)

                if last_exc is not None:
                    logger.error("poll_error", ticker=ticker, error=str(last_exc) or repr(last_exc), exc_type=type(last_exc).__name__)
                    stats["errors"] += 1
                    continue

                stats["seen"] += len(response.filings)

                for filing in response.filings:
                    # Check for duplicate
                    existing = await session.execute(
                        select(Document).where(
                            Document.accession_no == filing.accession_no,
                            Document.form_type == filing.form_type,
                        )
                    )
                    if existing.scalar_one_or_none() is not None:
                        stats["skipped"] += 1
                        continue

                    doc_id = make_document_id(
                        "sec",
                        f"TICKER::{ticker}",
                        filing.filing_date,
                        filing.accession_no,
                    )

                    doc = Document(
                        document_id=doc_id,
                        source_name="sec",
                        issuer_id=ticker_to_issuer.get(ticker),
                        symbol_id=ticker_to_symbol.get(ticker),
                        accession_no=filing.accession_no,
                        form_type=filing.form_type,
                        filing_date=dt.date.fromisoformat(filing.filing_date),
                        accepted_at_utc=(
                            dt.datetime.fromisoformat(
                                filing.accepted_at.replace("Z", "+00:00")
                            )
                            if filing.accepted_at
                            else None
                        ),
                        primary_document_name=filing.primary_document,
                        item_numbers=filing.items if filing.items else None,
                        parsed_status="pending",
                    )
                    session.add(doc)
                    stats["written"] += 1
                    logger.info(
                        "new_filing_discovered",
                        ticker=ticker,
                        accession_no=filing.accession_no,
                        form_type=filing.form_type,
                        filing_date=filing.filing_date,
                    )

            # Update job record
            job.status = "succeeded" if stats["errors"] == 0 else "partial"
            job.finished_at_utc = dt.datetime.now(tz=dt.UTC)
            job.records_seen = stats["seen"]
            job.records_written = stats["written"]
            job.records_skipped = stats["skipped"]
            job.error_count = stats["errors"]

    logger.info("filing_poller_done", **stats)
    return stats


def main() -> None:
    parser = argparse.ArgumentParser(description="Filing Poller")
    parser.add_argument("--run-id", default=new_job_run_id())
    parser.add_argument(
        "--start-date",
        default=None,
        metavar="YYYY-MM-DD",
        help="Start date for filing search (default: 7 days ago)",
    )
    parser.add_argument(
        "--end-date",
        default=None,
        metavar="YYYY-MM-DD",
        help="End date for filing search (default: today)",
    )
    parser.add_argument(
        "--symbols-file",
        default=None,
        help="Override symbols YAML (default: settings.symbols_file). "
             "Use for one-off backfills against a wider universe (e.g. broad snapshot).",
    )
    args = parser.parse_args()

    settings = get_settings()
    configure_logging(settings.log_level)
    bind_job_run_id(args.run_id)

    override_symbols = _load_symbols_from_yaml(args.symbols_file) if args.symbols_file else None

    asyncio.run(poll_filings(
        args.run_id,
        start_date=args.start_date,
        end_date=args.end_date,
        symbols=override_symbols,
    ))


if __name__ == "__main__":
    main()