"""Filing Fetcher: download exhibit text and cache locally.""" from __future__ import annotations import argparse import asyncio import datetime as dt import uuid from dataclasses import dataclass from sqlalchemy import select from libs.common.config import get_settings from libs.common.file_store import exists_exhibit, write_exhibit from libs.common.ids import new_job_run_id from libs.common.logging import bind_job_run_id, configure_logging, get_logger from libs.db.models import Document, ExhibitCache, JobRun from libs.db.session import get_session from libs.oracle_client import FilingsService, make_oracle_client from libs.oracle_client.exceptions import OracleNotFoundError logger = get_logger(__name__) @dataclass(slots=True) class _FetchedExhibit: exhibit_type: str checksum: str cache_path: str @dataclass(slots=True) class _FetchResult: doc_id: str accession_no: str | None fetched: list[_FetchedExhibit] item_numbers: list[str] | None errors: int async def _fetch_doc( svc: FilingsService, doc: Document, exhibit_types: list[str], ) -> _FetchResult: if not doc.accession_no: return _FetchResult( doc_id=str(doc.document_id), accession_no=None, fetched=[], item_numbers=None, errors=0, ) fetched: list[_FetchedExhibit] = [] errors = 0 for exhibit_type in exhibit_types: if exists_exhibit(doc.accession_no, exhibit_type): logger.info( "exhibit_already_cached", accession_no=doc.accession_no, exhibit_type=exhibit_type, ) continue try: response = await asyncio.wait_for( svc.get_exhibit(doc.accession_no, exhibit_type), timeout=30.0, ) checksum = write_exhibit(doc.accession_no, exhibit_type, response.content) from libs.common.file_store import exhibit_path fetched.append( _FetchedExhibit( exhibit_type=exhibit_type, checksum=checksum, cache_path=str(exhibit_path(doc.accession_no, exhibit_type)), ) ) logger.info( "exhibit_fetched", accession_no=doc.accession_no, exhibit_type=exhibit_type, ) except OracleNotFoundError: logger.warning( "exhibit_not_found", accession_no=doc.accession_no, exhibit_type=exhibit_type, ) except Exception as exc: logger.error( "exhibit_fetch_error", accession_no=doc.accession_no, exhibit_type=exhibit_type, error=str(exc), ) errors += 1 item_numbers: list[str] | None = None if not doc.item_numbers: try: item_numbers = await asyncio.wait_for( svc.get_filing_items(doc.accession_no), timeout=15.0, ) if item_numbers: logger.info( "item_numbers_extracted", accession_no=doc.accession_no, items=item_numbers, ) except Exception as exc: logger.debug( "item_numbers_extraction_failed", accession_no=doc.accession_no, error=str(exc), ) return _FetchResult( doc_id=str(doc.document_id), accession_no=doc.accession_no, fetched=fetched, item_numbers=item_numbers, errors=errors, ) async def fetch_exhibits( run_id: str, start_date: str | None = None, end_date: str | None = None, ) -> dict[str, int]: settings = get_settings() app_config = settings.get_app_config() exhibit_types = app_config.get("pipeline", {}).get("exhibit_types", ["EX-99.1"]) concurrency = max(1, int(app_config.get("pipeline", {}).get("fetcher_concurrency", 20))) stats = {"seen": 0, "written": 0, "skipped": 0, "errors": 0} batch_size = max(10, int(app_config.get("pipeline", {}).get("fetcher_batch_size", 25))) async with make_oracle_client() as client: svc = FilingsService(client) async with get_session() as session: job = JobRun( job_run_id=uuid.UUID(run_id), job_name="filing_fetcher", source_name="oracle", run_date=dt.date.today(), status="running", ) session.add(job) await session.flush() await session.commit() stmt = select(Document).where(Document.parsed_status == "pending") if start_date: stmt = stmt.where(Document.filing_date >= dt.date.fromisoformat(start_date)) if end_date: stmt = stmt.where(Document.filing_date <= dt.date.fromisoformat(end_date)) result = await session.execute(stmt.order_by(Document.filing_date, Document.document_id)) docs = result.scalars().all() stats["seen"] = len(docs) doc_map = {str(doc.document_id): doc for doc in docs} for batch_start in range(0, len(docs), batch_size): batch_docs = docs[batch_start: batch_start + batch_size] for task_start in range(0, len(batch_docs), concurrency): task_docs = batch_docs[task_start: task_start + concurrency] results = await asyncio.gather( *[_fetch_doc(svc, doc, exhibit_types) for doc in task_docs] ) for result_row in results: doc = doc_map[result_row.doc_id] if not result_row.accession_no: stats["skipped"] += 1 continue for fetched in result_row.fetched: existing_cache = await session.execute( select(ExhibitCache).where( ExhibitCache.accession_no == result_row.accession_no, ExhibitCache.exhibit_type == fetched.exhibit_type, ) ) if existing_cache.scalar_one_or_none() is None: session.add( ExhibitCache( accession_no=result_row.accession_no, exhibit_type=fetched.exhibit_type, content_hash=fetched.checksum, cache_path=fetched.cache_path, ) ) stats["written"] += 1 if result_row.item_numbers: doc.item_numbers = result_row.item_numbers doc.parsed_status = "ready_for_parse" doc.updated_at_utc = dt.datetime.now(tz=dt.UTC) stats["errors"] += result_row.errors processed = min(batch_start + len(batch_docs), len(docs)) await session.commit() logger.info( "batch_committed", processed=processed, total=stats["seen"], written=stats["written"], errors=stats["errors"], ) job.status = "succeeded" if stats["errors"] == 0 else "partial" job.finished_at_utc = dt.datetime.now(tz=dt.UTC) job.records_seen = stats["seen"] job.records_written = stats["written"] job.records_skipped = stats["skipped"] job.error_count = stats["errors"] logger.info("filing_fetcher_done", **stats) return stats def main() -> None: parser = argparse.ArgumentParser(description="Filing Fetcher") parser.add_argument("--run-id", default=new_job_run_id()) parser.add_argument("--start-date", default=None, metavar="YYYY-MM-DD") parser.add_argument("--end-date", default=None, metavar="YYYY-MM-DD") args = parser.parse_args() settings = get_settings() configure_logging(settings.log_level) bind_job_run_id(args.run_id) asyncio.run(fetch_exhibits(args.run_id, start_date=args.start_date, end_date=args.end_date)) if __name__ == "__main__": main()