fithia2/apps/pipeline/filing_poller/main.py

"""Filing Poller: discover new 8-K/6-K filings via Stock Oracle."""
from __future__ import annotations

import argparse
import asyncio
import datetime as dt
import uuid

from sqlalchemy import select

from libs.common.config import get_settings
from libs.common.ids import document_id as make_document_id
from libs.common.ids import new_job_run_id
from libs.common.logging import bind_job_run_id, configure_logging, get_logger
from libs.db.models import Document, IssuerMaster, JobRun, SymbolMaster
from libs.db.session import get_session
from libs.oracle_client import FilingsService, make_oracle_client

logger = get_logger(__name__)


async def poll_filings(
    run_id: str,
    start_date: str | None = None,
    end_date: str | None = None,
) -> dict[str, int]:
    settings = get_settings()
    symbols = settings.get_symbols()
    app_config = settings.get_app_config()
    form_types = ",".join(app_config.get("pipeline", {}).get("form_types", ["8-K", "6-K"]))

    effective_start = start_date or (dt.date.today() - dt.timedelta(days=7)).isoformat()

    stats = {"seen": 0, "written": 0, "skipped": 0, "errors": 0}

    async with make_oracle_client() as client:
        svc = FilingsService(client)

        async with get_session() as session:
            # Record job start
            job = JobRun(
                job_run_id=uuid.UUID(run_id),
                job_name="filing_poller",
                source_name="oracle",
                run_date=dt.date.today(),
                status="running",
            )
            session.add(job)
            await session.flush()

            # Load issuer/symbol lookup maps
            issuer_result = await session.execute(select(IssuerMaster))
            ticker_to_issuer = {i.ticker: i.issuer_id for i in issuer_result.scalars().all()}

            symbol_result = await session.execute(
                select(SymbolMaster).where(SymbolMaster.is_primary == True)  # noqa: E712
            )
            ticker_to_symbol = {s.ticker: s.symbol_id for s in symbol_result.scalars().all()}

            for ticker in symbols:
                try:
                    response = await svc.search_filings(
                        ticker,
                        form_type=form_types,
                        start_date=effective_start,
                        end_date=end_date,
                    )
                    stats["seen"] += len(response.filings)

                    for filing in response.filings:
                        # Check for duplicate
                        existing = await session.execute(
                            select(Document).where(
                                Document.accession_no == filing.accession_no,
                                Document.form_type == filing.form_type,
                            )
                        )
                        if existing.scalar_one_or_none() is not None:
                            stats["skipped"] += 1
                            continue

                        doc_id = make_document_id(
                            "sec",
                            f"TICKER::{ticker}",
                            filing.filing_date,
                            filing.accession_no,
                        )

                        doc = Document(
                            document_id=doc_id,
                            source_name="sec",
                            issuer_id=ticker_to_issuer.get(ticker),
                            symbol_id=ticker_to_symbol.get(ticker),
                            accession_no=filing.accession_no,
                            form_type=filing.form_type,
                            filing_date=dt.date.fromisoformat(filing.filing_date),
                            accepted_at_utc=(
                                dt.datetime.fromisoformat(
                                    filing.accepted_at.replace("Z", "+00:00")
                                )
                                if filing.accepted_at
                                else None
                            ),
                            primary_document_name=filing.primary_document,
                            item_numbers=filing.items if filing.items else None,
                            parsed_status="pending",
                        )
                        session.add(doc)
                        stats["written"] += 1
                        logger.info(
                            "new_filing_discovered",
                            ticker=ticker,
                            accession_no=filing.accession_no,
                            form_type=filing.form_type,
                            filing_date=filing.filing_date,
                        )

                except Exception as exc:
                    logger.error("poll_error", ticker=ticker, error=str(exc) or repr(exc), exc_type=type(exc).__name__)
                    stats["errors"] += 1

            # Update job record
            job.status = "succeeded" if stats["errors"] == 0 else "partial"
            job.finished_at_utc = dt.datetime.now(tz=dt.UTC)
            job.records_seen = stats["seen"]
            job.records_written = stats["written"]
            job.records_skipped = stats["skipped"]
            job.error_count = stats["errors"]

    logger.info("filing_poller_done", **stats)
    return stats


def main() -> None:
    parser = argparse.ArgumentParser(description="Filing Poller")
    parser.add_argument("--run-id", default=new_job_run_id())
    parser.add_argument(
        "--start-date",
        default=None,
        metavar="YYYY-MM-DD",
        help="Start date for filing search (default: 7 days ago)",
    )
    parser.add_argument(
        "--end-date",
        default=None,
        metavar="YYYY-MM-DD",
        help="End date for filing search (default: today)",
    )
    args = parser.parse_args()

    settings = get_settings()
    configure_logging(settings.log_level)
    bind_job_run_id(args.run_id)

    asyncio.run(poll_filings(args.run_id, start_date=args.start_date, end_date=args.end_date))


if __name__ == "__main__":
    main()