"""Filing Poller: discover new 8-K/6-K filings via Stock Oracle.""" from __future__ import annotations import argparse import asyncio import datetime as dt import uuid from pathlib import Path import yaml from sqlalchemy import select from libs.common.config import get_settings from libs.common.ids import document_id as make_document_id from libs.common.ids import new_job_run_id from libs.common.logging import bind_job_run_id, configure_logging, get_logger from libs.db.models import Document, IssuerMaster, JobRun, SymbolMaster from libs.db.session import get_session from libs.oracle_client import FilingsService, make_oracle_client logger = get_logger(__name__) def _load_symbols_from_yaml(path: str) -> list[str]: """Load ticker list from a symbols YAML (supports list or dict with 'symbols' key).""" raw = yaml.safe_load(Path(path).read_text()) if isinstance(raw, list): items = raw elif isinstance(raw, dict): items = raw.get("symbols", []) else: items = [] return sorted({str(s).upper() for s in items if s}) async def poll_filings( run_id: str, start_date: str | None = None, end_date: str | None = None, symbols: list[str] | None = None, ) -> dict[str, int]: settings = get_settings() if symbols is None: symbols = settings.get_symbols() app_config = settings.get_app_config() form_types = ",".join(app_config.get("pipeline", {}).get("form_types", ["8-K", "6-K"])) effective_start = start_date or (dt.date.today() - dt.timedelta(days=7)).isoformat() stats = {"seen": 0, "written": 0, "skipped": 0, "errors": 0} async with make_oracle_client() as client: svc = FilingsService(client) async with get_session() as session: # Record job start job = JobRun( job_run_id=uuid.UUID(run_id), job_name="filing_poller", source_name="oracle", run_date=dt.date.today(), status="running", ) session.add(job) await session.flush() # Load issuer/symbol lookup maps issuer_result = await session.execute(select(IssuerMaster)) ticker_to_issuer = {i.ticker: i.issuer_id for i in issuer_result.scalars().all()} symbol_result = await session.execute( select(SymbolMaster).where(SymbolMaster.is_primary == True) # noqa: E712 ) ticker_to_symbol = {s.ticker: s.symbol_id for s in symbol_result.scalars().all()} for ticker in symbols: last_exc: Exception | None = None response = None for attempt in range(3): try: response = await svc.search_filings( ticker, form_type=form_types, start_date=effective_start, end_date=end_date, ) last_exc = None break except Exception as exc: last_exc = exc wait = 2 ** attempt # 1s, 2s, 4s logger.warning( "poll_retry", ticker=ticker, attempt=attempt + 1, wait=wait, error=str(exc) or repr(exc), exc_type=type(exc).__name__, ) await asyncio.sleep(wait) if last_exc is not None: logger.error("poll_error", ticker=ticker, error=str(last_exc) or repr(last_exc), exc_type=type(last_exc).__name__) stats["errors"] += 1 continue stats["seen"] += len(response.filings) for filing in response.filings: # Check for duplicate existing = await session.execute( select(Document).where( Document.accession_no == filing.accession_no, Document.form_type == filing.form_type, ) ) if existing.scalar_one_or_none() is not None: stats["skipped"] += 1 continue doc_id = make_document_id( "sec", f"TICKER::{ticker}", filing.filing_date, filing.accession_no, ) doc = Document( document_id=doc_id, source_name="sec", issuer_id=ticker_to_issuer.get(ticker), symbol_id=ticker_to_symbol.get(ticker), accession_no=filing.accession_no, form_type=filing.form_type, filing_date=dt.date.fromisoformat(filing.filing_date), accepted_at_utc=( dt.datetime.fromisoformat( filing.accepted_at.replace("Z", "+00:00") ) if filing.accepted_at else None ), primary_document_name=filing.primary_document, item_numbers=filing.items if filing.items else None, parsed_status="pending", ) session.add(doc) stats["written"] += 1 logger.info( "new_filing_discovered", ticker=ticker, accession_no=filing.accession_no, form_type=filing.form_type, filing_date=filing.filing_date, ) # Update job record job.status = "succeeded" if stats["errors"] == 0 else "partial" job.finished_at_utc = dt.datetime.now(tz=dt.UTC) job.records_seen = stats["seen"] job.records_written = stats["written"] job.records_skipped = stats["skipped"] job.error_count = stats["errors"] logger.info("filing_poller_done", **stats) return stats def main() -> None: parser = argparse.ArgumentParser(description="Filing Poller") parser.add_argument("--run-id", default=new_job_run_id()) parser.add_argument( "--start-date", default=None, metavar="YYYY-MM-DD", help="Start date for filing search (default: 7 days ago)", ) parser.add_argument( "--end-date", default=None, metavar="YYYY-MM-DD", help="End date for filing search (default: today)", ) parser.add_argument( "--symbols-file", default=None, help="Override symbols YAML (default: settings.symbols_file). " "Use for one-off backfills against a wider universe (e.g. broad snapshot).", ) args = parser.parse_args() settings = get_settings() configure_logging(settings.log_level) bind_job_run_id(args.run_id) override_symbols = _load_symbols_from_yaml(args.symbols_file) if args.symbols_file else None asyncio.run(poll_filings( args.run_id, start_date=args.start_date, end_date=args.end_date, symbols=override_symbols, )) if __name__ == "__main__": main()