You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
389 lines
15 KiB
Python
389 lines
15 KiB
Python
"""Event Parser: parse exhibit text → events + event_parses.
|
|
|
|
Supports two modes:
|
|
- Normal: parse documents with parsed_status="ready_for_parse"
|
|
- Reparse (--reparse): re-classify existing events using item_numbers
|
|
from the Document table, updating event_type in Event rows.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
import datetime as dt
|
|
import uuid
|
|
|
|
from sqlalchemy import select, update
|
|
|
|
from libs.common.config import get_settings
|
|
from libs.common.file_store import read_exhibit
|
|
from libs.common.ids import event_id as make_event_id
|
|
from libs.common.ids import new_job_run_id
|
|
from libs.common.logging import bind_job_run_id, configure_logging, get_logger
|
|
from libs.db.models import Document, Event, EventParse, JobRun
|
|
from libs.db.session import get_session
|
|
from libs.oracle_client import FilingsService, make_oracle_client
|
|
from libs.parser.rule_parser import PARSER_VERSION, SCHEMA_VERSION, RuleBasedParser, _classify_event_type
|
|
from libs.parser.schema_validator import validate_parser_output
|
|
from libs.parser.text_normalizer import looks_like_html, normalize_text
|
|
|
|
logger = get_logger(__name__)
|
|
_parser = RuleBasedParser()
|
|
|
|
|
|
async def run_event_parser(
|
|
run_id: str,
|
|
start_date: str | None = None,
|
|
end_date: str | None = None,
|
|
) -> dict[str, int]:
|
|
settings = get_settings()
|
|
app_config = settings.get_app_config()
|
|
exhibit_types = app_config.get("pipeline", {}).get("exhibit_types", ["EX-99.1"])
|
|
|
|
stats = {"seen": 0, "valid": 0, "invalid": 0, "errors": 0}
|
|
|
|
async with get_session() as session:
|
|
job = JobRun(
|
|
job_run_id=uuid.UUID(run_id),
|
|
job_name="event_parser",
|
|
source_name="oracle",
|
|
run_date=dt.date.today(),
|
|
status="running",
|
|
)
|
|
session.add(job)
|
|
await session.flush()
|
|
|
|
stmt = select(Document).where(Document.parsed_status == "ready_for_parse")
|
|
if start_date:
|
|
stmt = stmt.where(Document.filing_date >= dt.date.fromisoformat(start_date))
|
|
if end_date:
|
|
stmt = stmt.where(Document.filing_date <= dt.date.fromisoformat(end_date))
|
|
|
|
result = await session.execute(stmt.order_by(Document.filing_date, Document.document_id))
|
|
docs = result.scalars().all()
|
|
stats["seen"] = len(docs)
|
|
|
|
for doc in docs:
|
|
if not doc.accession_no:
|
|
stats["invalid"] += 1
|
|
doc.parsed_status = "failed"
|
|
continue
|
|
|
|
# Try each exhibit type until we find one
|
|
text: str | None = None
|
|
for exhibit_type in exhibit_types:
|
|
try:
|
|
text = read_exhibit(doc.accession_no, exhibit_type)
|
|
break
|
|
except FileNotFoundError:
|
|
continue
|
|
|
|
if text is None:
|
|
logger.warning("no_exhibit_text", accession_no=doc.accession_no)
|
|
doc.parsed_status = "failed"
|
|
stats["errors"] += 1
|
|
continue
|
|
|
|
normalized = normalize_text(text, is_html=looks_like_html(text))
|
|
|
|
metadata = {
|
|
"filing_date": doc.filing_date.isoformat(),
|
|
"accepted_at_utc": (
|
|
doc.accepted_at_utc.isoformat() if doc.accepted_at_utc else None
|
|
),
|
|
"form_type": doc.form_type,
|
|
"item_numbers": doc.item_numbers or [],
|
|
}
|
|
|
|
try:
|
|
# Use savepoint so one failure doesn't break the session
|
|
async with session.begin_nested():
|
|
output = _parser.parse(
|
|
document_id=doc.document_id,
|
|
form_type=doc.form_type,
|
|
text=normalized,
|
|
metadata=metadata,
|
|
)
|
|
output_dict = output.model_dump()
|
|
# Sanitize null bytes that PostgreSQL JSONB rejects
|
|
import json as _json
|
|
_raw = _json.dumps(output_dict)
|
|
if "\x00" in _raw or "\\u0000" in _raw:
|
|
_raw = _raw.replace("\x00", "").replace("\\u0000", "")
|
|
output_dict = _json.loads(_raw)
|
|
errors = validate_parser_output(output_dict)
|
|
|
|
if errors:
|
|
logger.warning(
|
|
"parse_validation_failed",
|
|
document_id=doc.document_id,
|
|
errors=errors[:3],
|
|
)
|
|
event_id_str = make_event_id(doc.document_id, output.event_type)
|
|
event = Event(
|
|
event_id=event_id_str,
|
|
primary_document_id=doc.document_id,
|
|
issuer_id=doc.issuer_id,
|
|
symbol_id=doc.symbol_id,
|
|
event_type=output.event_type,
|
|
event_direction=output.event_direction,
|
|
event_date=doc.filing_date,
|
|
filed_at_utc=doc.accepted_at_utc,
|
|
parser_version=PARSER_VERSION,
|
|
parse_confidence=output.confidence.overall,
|
|
status="rejected",
|
|
)
|
|
session.add(event)
|
|
await session.flush()
|
|
|
|
parse_row = EventParse(
|
|
event_id=event_id_str,
|
|
parser_kind="rule",
|
|
parser_version=PARSER_VERSION,
|
|
schema_version=SCHEMA_VERSION,
|
|
output_json=output_dict,
|
|
validation_status="invalid",
|
|
validation_errors={"errors": errors},
|
|
)
|
|
session.add(parse_row)
|
|
doc.parsed_status = "failed"
|
|
stats["invalid"] += 1
|
|
else:
|
|
event_id_str = make_event_id(doc.document_id, output.event_type)
|
|
|
|
existing_evt = await session.execute(
|
|
select(Event).where(Event.event_id == event_id_str)
|
|
)
|
|
if existing_evt.scalar_one_or_none() is not None:
|
|
logger.info("event_already_exists", event_id=event_id_str)
|
|
doc.parsed_status = "succeeded"
|
|
stats["valid"] += 1
|
|
continue
|
|
|
|
event = Event(
|
|
event_id=event_id_str,
|
|
primary_document_id=doc.document_id,
|
|
issuer_id=doc.issuer_id,
|
|
symbol_id=doc.symbol_id,
|
|
event_type=output.event_type,
|
|
event_direction=output.event_direction,
|
|
event_date=doc.filing_date,
|
|
filed_at_utc=doc.accepted_at_utc,
|
|
parser_version=PARSER_VERSION,
|
|
parse_confidence=output.confidence.overall,
|
|
status="pending",
|
|
)
|
|
session.add(event)
|
|
await session.flush()
|
|
|
|
parse_row = EventParse(
|
|
event_id=event_id_str,
|
|
parser_kind="rule",
|
|
parser_version=PARSER_VERSION,
|
|
schema_version=SCHEMA_VERSION,
|
|
output_json=output_dict,
|
|
validation_status="valid",
|
|
validation_errors=None,
|
|
)
|
|
session.add(parse_row)
|
|
doc.parsed_status = "succeeded"
|
|
stats["valid"] += 1
|
|
|
|
logger.info(
|
|
"event_created",
|
|
event_id=event_id_str,
|
|
event_type=output.event_type,
|
|
confidence=output.confidence.overall,
|
|
)
|
|
|
|
except Exception as exc:
|
|
logger.error(
|
|
"parse_error",
|
|
document_id=doc.document_id,
|
|
error=str(exc),
|
|
)
|
|
doc.parsed_status = "failed"
|
|
stats["errors"] += 1
|
|
|
|
doc.updated_at_utc = dt.datetime.now(tz=dt.UTC)
|
|
|
|
job.status = "succeeded" if stats["errors"] == 0 else "partial"
|
|
job.finished_at_utc = dt.datetime.now(tz=dt.UTC)
|
|
job.records_seen = stats["seen"]
|
|
job.records_written = stats["valid"]
|
|
job.error_count = stats["errors"] + stats["invalid"]
|
|
|
|
logger.info("event_parser_done", **stats)
|
|
return stats
|
|
|
|
|
|
async def reparse_events(run_id: str) -> dict[str, int]:
|
|
"""Re-classify existing events by fetching item_numbers from Oracle API.
|
|
|
|
For each Document with a linked Event where event_type='unknown':
|
|
1. Look up filing in Oracle API to get item_numbers
|
|
2. Store item_numbers in Document row
|
|
3. Re-classify event_type based on items
|
|
4. Update Event row with new event_type and re-parse with exhibit text
|
|
"""
|
|
settings = get_settings()
|
|
app_config = settings.get_app_config()
|
|
exhibit_types = app_config.get("pipeline", {}).get("exhibit_types", ["EX-99.1"])
|
|
|
|
stats = {"seen": 0, "updated": 0, "skipped": 0, "errors": 0}
|
|
|
|
async with make_oracle_client() as client:
|
|
svc = FilingsService(client)
|
|
|
|
async with get_session() as session:
|
|
job = JobRun(
|
|
job_run_id=uuid.UUID(run_id),
|
|
job_name="event_parser_reparse",
|
|
source_name="oracle",
|
|
run_date=dt.date.today(),
|
|
status="running",
|
|
)
|
|
session.add(job)
|
|
await session.flush()
|
|
|
|
# Find all events with event_type='unknown'
|
|
result = await session.execute(
|
|
select(Event, Document)
|
|
.join(Document, Event.primary_document_id == Document.document_id)
|
|
.where(Event.event_type == "unknown")
|
|
)
|
|
pairs = result.all()
|
|
stats["seen"] = len(pairs)
|
|
|
|
if not pairs:
|
|
logger.info("reparse_nothing_to_do")
|
|
job.status = "succeeded"
|
|
job.finished_at_utc = dt.datetime.now(tz=dt.UTC)
|
|
job.records_seen = 0
|
|
return stats
|
|
|
|
# Step 1: Fetch item_numbers from Oracle for all documents missing them
|
|
docs_needing_items = {
|
|
doc.document_id: doc
|
|
for _, doc in pairs
|
|
if not doc.item_numbers
|
|
}
|
|
|
|
if docs_needing_items:
|
|
logger.info("reparse_fetching_items", count=len(docs_needing_items))
|
|
|
|
for doc in docs_needing_items.values():
|
|
if not doc.accession_no:
|
|
continue
|
|
try:
|
|
items = await svc.get_filing_items(doc.accession_no)
|
|
if items:
|
|
doc.item_numbers = items
|
|
logger.info(
|
|
"reparse_items_found",
|
|
accession_no=doc.accession_no,
|
|
items=items,
|
|
)
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"reparse_oracle_error",
|
|
accession_no=doc.accession_no,
|
|
error=str(exc),
|
|
)
|
|
|
|
await session.flush()
|
|
|
|
# Step 2: Re-parse each event with updated item_numbers
|
|
for event, doc in pairs:
|
|
try:
|
|
items = doc.item_numbers or []
|
|
new_event_type = _classify_event_type(items)
|
|
|
|
if new_event_type == event.event_type:
|
|
stats["skipped"] += 1
|
|
continue
|
|
|
|
# Also re-parse the full exhibit text with items
|
|
text: str | None = None
|
|
for exhibit_type in exhibit_types:
|
|
try:
|
|
text = read_exhibit(doc.accession_no, exhibit_type)
|
|
break
|
|
except FileNotFoundError:
|
|
continue
|
|
|
|
if text:
|
|
normalized = normalize_text(text, is_html=looks_like_html(text))
|
|
metadata = {
|
|
"filing_date": doc.filing_date.isoformat(),
|
|
"accepted_at_utc": (
|
|
doc.accepted_at_utc.isoformat()
|
|
if doc.accepted_at_utc
|
|
else None
|
|
),
|
|
"form_type": doc.form_type,
|
|
"item_numbers": items,
|
|
}
|
|
output = _parser.parse(
|
|
document_id=doc.document_id,
|
|
form_type=doc.form_type,
|
|
text=normalized,
|
|
metadata=metadata,
|
|
)
|
|
event.event_type = output.event_type
|
|
event.event_direction = output.event_direction
|
|
event.parse_confidence = output.confidence.overall
|
|
else:
|
|
# No exhibit text available, just update event_type from items
|
|
event.event_type = new_event_type
|
|
|
|
stats["updated"] += 1
|
|
logger.info(
|
|
"reparse_event_updated",
|
|
event_id=event.event_id,
|
|
old_type="unknown",
|
|
new_type=event.event_type,
|
|
)
|
|
|
|
except Exception as exc:
|
|
logger.error(
|
|
"reparse_error",
|
|
event_id=event.event_id,
|
|
error=str(exc),
|
|
)
|
|
stats["errors"] += 1
|
|
|
|
job.status = "succeeded" if stats["errors"] == 0 else "partial"
|
|
job.finished_at_utc = dt.datetime.now(tz=dt.UTC)
|
|
job.records_seen = stats["seen"]
|
|
job.records_written = stats["updated"]
|
|
job.error_count = stats["errors"]
|
|
|
|
logger.info("reparse_done", **stats)
|
|
return stats
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description="Event Parser")
|
|
parser.add_argument("--run-id", default=new_job_run_id())
|
|
parser.add_argument("--start-date", default=None, metavar="YYYY-MM-DD")
|
|
parser.add_argument("--end-date", default=None, metavar="YYYY-MM-DD")
|
|
parser.add_argument(
|
|
"--reparse",
|
|
action="store_true",
|
|
help="Re-classify existing events with event_type='unknown' using Oracle API item numbers",
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
settings = get_settings()
|
|
configure_logging(settings.log_level)
|
|
bind_job_run_id(args.run_id)
|
|
|
|
if args.reparse:
|
|
asyncio.run(reparse_events(args.run_id))
|
|
else:
|
|
asyncio.run(run_event_parser(args.run_id, start_date=args.start_date, end_date=args.end_date))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|