"""Event Parser: parse exhibit text → events + event_parses. Supports two modes: - Normal: parse documents with parsed_status="ready_for_parse" - Reparse (--reparse): re-classify existing events using item_numbers from the Document table, updating event_type in Event rows. """ from __future__ import annotations import argparse import asyncio import datetime as dt import uuid from sqlalchemy import select, update from libs.common.config import get_settings from libs.common.file_store import read_exhibit from libs.common.ids import event_id as make_event_id from libs.common.ids import new_job_run_id from libs.common.logging import bind_job_run_id, configure_logging, get_logger from libs.db.models import Document, Event, EventParse, JobRun, SymbolMaster from libs.db.session import get_session from libs.oracle_client import FilingsService, make_oracle_client from libs.oracle_client.models import FilingEventEntry from libs.parser.event_type_normalizer import normalize_oracle_event, normalize_oracle_event_type from libs.parser.rule_parser import PARSER_VERSION, SCHEMA_VERSION, RuleBasedParser, _classify_event_type from libs.parser.schema_validator import validate_parser_output from libs.parser.text_normalizer import looks_like_html, normalize_text logger = get_logger(__name__) _parser = RuleBasedParser() _ORACLE_EVENT_DIRECTION_MAP: dict[str, str] = { "other_material_event": "unknown", "material_contract": "unknown", "earnings_release": "mixed", "guidance_update": "mixed", "management_change": "unknown", } async def _try_oracle_events_fallback( svc: FilingsService, doc: Document, ticker: str, ) -> FilingEventEntry | None: """Try Oracle events API as fallback when no exhibit text is available.""" if not doc.accession_no or not ticker: return None try: resp = await asyncio.wait_for( svc.get_filing_events( ticker=ticker, start_date=doc.filing_date.isoformat(), end_date=doc.filing_date.isoformat(), accession_no=doc.accession_no, ), timeout=15.0, ) if resp.events: return resp.events[0] except Exception as exc: logger.debug( "oracle_events_fallback_failed", accession_no=doc.accession_no, error=str(exc), ) return None async def run_event_parser( run_id: str, start_date: str | None = None, end_date: str | None = None, ) -> dict[str, int]: settings = get_settings() app_config = settings.get_app_config() exhibit_types = app_config.get("pipeline", {}).get("exhibit_types", ["EX-99.1"]) stats = {"seen": 0, "valid": 0, "invalid": 0, "errors": 0, "oracle_fallback": 0} async with make_oracle_client() as client, get_session() as session: svc = FilingsService(client) job = JobRun( job_run_id=uuid.UUID(run_id), job_name="event_parser", source_name="oracle", run_date=dt.date.today(), status="running", ) session.add(job) await session.flush() stmt = select(Document).where(Document.parsed_status == "ready_for_parse") if start_date: stmt = stmt.where(Document.filing_date >= dt.date.fromisoformat(start_date)) if end_date: stmt = stmt.where(Document.filing_date <= dt.date.fromisoformat(end_date)) result = await session.execute(stmt.order_by(Document.filing_date, Document.document_id)) docs = result.scalars().all() stats["seen"] = len(docs) # Prefetch ticker mapping for Oracle events fallback symbol_ids = {doc.symbol_id for doc in docs if doc.symbol_id} ticker_map: dict[str, str] = {} if symbol_ids: sm_result = await session.execute( select(SymbolMaster).where(SymbolMaster.symbol_id.in_(symbol_ids)) ) for sm in sm_result.scalars(): ticker_map[sm.symbol_id] = sm.ticker for _batch_i, doc in enumerate(docs): if _batch_i > 0 and _batch_i % 500 == 0: await session.commit() logger.info("event_parser_batch_commit", processed=_batch_i, total=stats["seen"]) if not doc.accession_no: stats["invalid"] += 1 doc.parsed_status = "failed" continue # Try each exhibit type until we find one text: str | None = None for exhibit_type in exhibit_types: try: text = read_exhibit(doc.accession_no, exhibit_type) break except FileNotFoundError: continue if text is None: # Fallback: try Oracle events API for pre-parsed event ticker = ticker_map.get(doc.symbol_id, "") if doc.symbol_id else "" oracle_event = await _try_oracle_events_fallback(svc, doc, ticker) if oracle_event: try: async with session.begin_nested(): raw_oracle_event_type = oracle_event.event_type event_type = normalize_oracle_event( raw_oracle_event_type, item_number=oracle_event.item_number, ) if event_type != raw_oracle_event_type: logger.info( "oracle_event_type_normalized", accession_no=doc.accession_no, raw=raw_oracle_event_type, normalized=event_type, item_number=oracle_event.item_number, ) event_direction = _ORACLE_EVENT_DIRECTION_MAP.get(event_type, "unknown") event_id_str = make_event_id(doc.document_id, event_type) existing_evt = await session.execute( select(Event).where(Event.event_id == event_id_str) ) if existing_evt.scalar_one_or_none() is not None: doc.parsed_status = "succeeded" stats["valid"] += 1 stats["oracle_fallback"] += 1 continue event = Event( event_id=event_id_str, primary_document_id=doc.document_id, issuer_id=doc.issuer_id, symbol_id=doc.symbol_id, event_type=event_type, event_direction=event_direction, event_date=doc.filing_date, filed_at_utc=doc.accepted_at_utc, parser_version=f"oracle-fallback-{PARSER_VERSION}", parse_confidence=0.5, status="pending", ) session.add(event) await session.flush() parse_row = EventParse( event_id=event_id_str, parser_kind="oracle_fallback", parser_version=f"oracle-fallback-{PARSER_VERSION}", schema_version=SCHEMA_VERSION, output_json={ "oracle_event_id": oracle_event.id, "event_type": event_type, "event_direction": event_direction, "title": oracle_event.title, "summary": oracle_event.summary or "", "content_source": oracle_event.content_source, "item_number": oracle_event.item_number, "guidance": { "status": "not_provided", "scope": "unknown", "notes": "", }, "signals": { "demand_strength": "unknown", "pricing_power": "unknown", "backlog_or_bookings": "unknown", "customer_expansion": "unknown", "margin_quality": "unknown", }, "risk_flags": { "oneoff_item": False, "tax_benefit": False, "valuation_gain": False, "non_gaap_heavy": False, "financing_related": False, "legal_or_regulatory_overhang": False, }, "confidence": { "overall": 0.5, "event_type": 0.6, "event_direction": 0.4, "guidance": 0.0, "risk_flags": 0.5, }, "evidence": [], }, validation_status="valid", validation_errors=None, ) session.add(parse_row) doc.parsed_status = "succeeded" stats["valid"] += 1 stats["oracle_fallback"] += 1 logger.info( "event_created_oracle_fallback", event_id=event_id_str, event_type=event_type, ticker=ticker, ) except Exception as exc: logger.error( "oracle_fallback_insert_error", document_id=doc.document_id, error=str(exc), ) doc.parsed_status = "failed" stats["errors"] += 1 continue logger.warning("no_exhibit_text", accession_no=doc.accession_no) doc.parsed_status = "failed" stats["errors"] += 1 continue normalized = normalize_text(text, is_html=looks_like_html(text)) metadata = { "filing_date": doc.filing_date.isoformat(), "accepted_at_utc": ( doc.accepted_at_utc.isoformat() if doc.accepted_at_utc else None ), "form_type": doc.form_type, "item_numbers": doc.item_numbers or [], } try: # Use savepoint so one failure doesn't break the session async with session.begin_nested(): output = _parser.parse( document_id=doc.document_id, form_type=doc.form_type, text=normalized, metadata=metadata, ) output_dict = output.model_dump() # Sanitize null bytes that PostgreSQL JSONB rejects import json as _json _raw = _json.dumps(output_dict) if "\x00" in _raw or "\\u0000" in _raw: _raw = _raw.replace("\x00", "").replace("\\u0000", "") output_dict = _json.loads(_raw) errors = validate_parser_output(output_dict) if errors: logger.warning( "parse_validation_failed", document_id=doc.document_id, errors=errors[:3], ) event_id_str = make_event_id(doc.document_id, output.event_type) event = Event( event_id=event_id_str, primary_document_id=doc.document_id, issuer_id=doc.issuer_id, symbol_id=doc.symbol_id, event_type=output.event_type, event_direction=output.event_direction, event_date=doc.filing_date, filed_at_utc=doc.accepted_at_utc, parser_version=PARSER_VERSION, parse_confidence=output.confidence.overall, status="rejected", ) session.add(event) await session.flush() parse_row = EventParse( event_id=event_id_str, parser_kind="rule", parser_version=PARSER_VERSION, schema_version=SCHEMA_VERSION, output_json=output_dict, validation_status="invalid", validation_errors={"errors": errors}, ) session.add(parse_row) doc.parsed_status = "failed" stats["invalid"] += 1 else: event_id_str = make_event_id(doc.document_id, output.event_type) existing_evt = await session.execute( select(Event).where(Event.event_id == event_id_str) ) if existing_evt.scalar_one_or_none() is not None: logger.info("event_already_exists", event_id=event_id_str) doc.parsed_status = "succeeded" stats["valid"] += 1 continue event = Event( event_id=event_id_str, primary_document_id=doc.document_id, issuer_id=doc.issuer_id, symbol_id=doc.symbol_id, event_type=output.event_type, event_direction=output.event_direction, event_date=doc.filing_date, filed_at_utc=doc.accepted_at_utc, parser_version=PARSER_VERSION, parse_confidence=output.confidence.overall, status="pending", ) session.add(event) await session.flush() parse_row = EventParse( event_id=event_id_str, parser_kind="rule", parser_version=PARSER_VERSION, schema_version=SCHEMA_VERSION, output_json=output_dict, validation_status="valid", validation_errors=None, ) session.add(parse_row) doc.parsed_status = "succeeded" stats["valid"] += 1 logger.info( "event_created", event_id=event_id_str, event_type=output.event_type, confidence=output.confidence.overall, ) except Exception as exc: logger.error( "parse_error", document_id=doc.document_id, error=str(exc), ) doc.parsed_status = "failed" stats["errors"] += 1 doc.updated_at_utc = dt.datetime.now(tz=dt.UTC) job.status = "succeeded" if stats["errors"] == 0 else "partial" job.finished_at_utc = dt.datetime.now(tz=dt.UTC) job.records_seen = stats["seen"] job.records_written = stats["valid"] job.error_count = stats["errors"] + stats["invalid"] logger.info("event_parser_done", **stats) return stats async def reparse_events(run_id: str) -> dict[str, int]: """Re-classify existing events by fetching item_numbers from Oracle API. For each Document with a linked Event where event_type='unknown': 1. Look up filing in Oracle API to get item_numbers 2. Store item_numbers in Document row 3. Re-classify event_type based on items 4. Update Event row with new event_type and re-parse with exhibit text """ settings = get_settings() app_config = settings.get_app_config() exhibit_types = app_config.get("pipeline", {}).get("exhibit_types", ["EX-99.1"]) stats = {"seen": 0, "updated": 0, "skipped": 0, "errors": 0} async with make_oracle_client() as client: svc = FilingsService(client) async with get_session() as session: job = JobRun( job_run_id=uuid.UUID(run_id), job_name="event_parser_reparse", source_name="oracle", run_date=dt.date.today(), status="running", ) session.add(job) await session.flush() # Find all events with event_type='unknown' result = await session.execute( select(Event, Document) .join(Document, Event.primary_document_id == Document.document_id) .where(Event.event_type == "unknown") ) pairs = result.all() stats["seen"] = len(pairs) if not pairs: logger.info("reparse_nothing_to_do") job.status = "succeeded" job.finished_at_utc = dt.datetime.now(tz=dt.UTC) job.records_seen = 0 return stats # Step 1: Fetch item_numbers from Oracle for all documents missing them docs_needing_items = { doc.document_id: doc for _, doc in pairs if not doc.item_numbers } if docs_needing_items: logger.info("reparse_fetching_items", count=len(docs_needing_items)) for doc in docs_needing_items.values(): if not doc.accession_no: continue try: items = await svc.get_filing_items(doc.accession_no) if items: doc.item_numbers = items logger.info( "reparse_items_found", accession_no=doc.accession_no, items=items, ) except Exception as exc: logger.warning( "reparse_oracle_error", accession_no=doc.accession_no, error=str(exc), ) await session.flush() # Step 2: Re-parse each event with updated item_numbers for event, doc in pairs: try: items = doc.item_numbers or [] new_event_type = _classify_event_type(items) if new_event_type == event.event_type: stats["skipped"] += 1 continue # Also re-parse the full exhibit text with items text: str | None = None for exhibit_type in exhibit_types: try: text = read_exhibit(doc.accession_no, exhibit_type) break except FileNotFoundError: continue if text: normalized = normalize_text(text, is_html=looks_like_html(text)) metadata = { "filing_date": doc.filing_date.isoformat(), "accepted_at_utc": ( doc.accepted_at_utc.isoformat() if doc.accepted_at_utc else None ), "form_type": doc.form_type, "item_numbers": items, } output = _parser.parse( document_id=doc.document_id, form_type=doc.form_type, text=normalized, metadata=metadata, ) event.event_type = output.event_type event.event_direction = output.event_direction event.parse_confidence = output.confidence.overall else: # No exhibit text available, just update event_type from items event.event_type = new_event_type stats["updated"] += 1 logger.info( "reparse_event_updated", event_id=event.event_id, old_type="unknown", new_type=event.event_type, ) except Exception as exc: logger.error( "reparse_error", event_id=event.event_id, error=str(exc), ) stats["errors"] += 1 job.status = "succeeded" if stats["errors"] == 0 else "partial" job.finished_at_utc = dt.datetime.now(tz=dt.UTC) job.records_seen = stats["seen"] job.records_written = stats["updated"] job.error_count = stats["errors"] logger.info("reparse_done", **stats) return stats async def _apply_oracle_renormalization(session) -> dict[str, int | dict[str, int]]: """Inner helper: scan oracle-fallback rows, apply the string normalizer, update. Extracted so tests can drive it against an injected session without forking a separate ``get_session()`` context. Caller owns commit/rollback. """ stats: dict[str, int | dict[str, int]] = { "seen": 0, "updated": 0, "skipped": 0, "errors": 0, "transitions": {}, # "raw -> normalized" -> count } result = await session.execute( select(Event).where(Event.parser_version.like("oracle-fallback%")) ) events = result.scalars().all() stats["seen"] = len(events) if not events: logger.info("renormalize_nothing_to_do") return stats transitions: dict[str, int] = stats["transitions"] # type: ignore[assignment] for event in events: try: raw = event.event_type normalized = normalize_oracle_event_type(raw) if normalized == raw: stats["skipped"] = int(stats["skipped"]) + 1 # type: ignore[arg-type] continue event.event_type = normalized event.updated_at_utc = dt.datetime.now(tz=dt.UTC) stats["updated"] = int(stats["updated"]) + 1 # type: ignore[arg-type] key = f"{raw} -> {normalized}" transitions[key] = transitions.get(key, 0) + 1 logger.info( "renormalize_event_updated", event_id=event.event_id, old_type=raw, new_type=normalized, ) except Exception as exc: logger.error( "renormalize_error", event_id=event.event_id, error=str(exc), ) stats["errors"] = int(stats["errors"]) + 1 # type: ignore[arg-type] return stats async def renormalize_oracle_fallback_events(run_id: str) -> dict[str, int | dict[str, int]]: """Apply the Oracle vocabulary normalizer to historical oracle-fallback rows. Scope: rows where ``parser_version LIKE 'oracle-fallback%'``. The string-only ``normalize_oracle_event_type`` is applied to each ``event_type`` and the row is updated only when the value changes. Already-aligned values (``earnings_release``, ``other_material_event``, etc.) are skipped naturally because the normalizer returns them unchanged. Out of scope (intentional, mirrors ``reparse_events()`` behavior): - ``event_id`` rewrite — the PK embeds the original ``event_type`` string but changing it would cascade across ``event_parses``, ``feature_snapshots``, and ``event_labels``. Left stale; harmless to downstream consumers. - ``EventParse.output_json['event_type']`` — also left stale. ``Event.event_type`` is the column the strategy reads. """ stats: dict[str, int | dict[str, int]] = { "seen": 0, "updated": 0, "skipped": 0, "errors": 0, "transitions": {}, } async with get_session() as session: job = JobRun( job_run_id=uuid.UUID(run_id), job_name="event_parser_renormalize_oracle", source_name="oracle", run_date=dt.date.today(), status="running", ) session.add(job) await session.flush() stats = await _apply_oracle_renormalization(session) errors = int(stats["errors"]) # type: ignore[arg-type] job.status = "succeeded" if errors == 0 else "partial" job.finished_at_utc = dt.datetime.now(tz=dt.UTC) job.records_seen = int(stats["seen"]) # type: ignore[arg-type] job.records_written = int(stats["updated"]) # type: ignore[arg-type] job.error_count = errors logger.info("renormalize_done", **{k: v for k, v in stats.items() if k != "transitions"}) logger.info("renormalize_transitions", transitions=stats["transitions"]) return stats def main() -> None: parser = argparse.ArgumentParser(description="Event Parser") parser.add_argument("--run-id", default=new_job_run_id()) parser.add_argument("--start-date", default=None, metavar="YYYY-MM-DD") parser.add_argument("--end-date", default=None, metavar="YYYY-MM-DD") parser.add_argument( "--reparse", action="store_true", help="Re-classify existing events with event_type='unknown' using Oracle API item numbers", ) parser.add_argument( "--renormalize-oracle-fallback", action="store_true", help=( "One-shot: apply libs.parser.event_type_normalizer to historical " "oracle-fallback rows. Pure DB UPDATEs (no Oracle API calls)." ), ) args = parser.parse_args() settings = get_settings() configure_logging(settings.log_level) bind_job_run_id(args.run_id) if args.renormalize_oracle_fallback: asyncio.run(renormalize_oracle_fallback_events(args.run_id)) if args.reparse: # Allow chaining: renormalize first, then run the unknown reparse. asyncio.run(reparse_events(new_job_run_id())) elif args.reparse: asyncio.run(reparse_events(args.run_id)) else: asyncio.run(run_event_parser(args.run_id, start_date=args.start_date, end_date=args.end_date)) if __name__ == "__main__": main()