feat: 8-K filing parser — Item 추출 + event 생성
AVGO 8-K (accession 0001193125-26-144028, Item 8.01, Google TPU 공급계약)이
DB에 수집은 되었으나 파싱/event 생성이 불가했던 문제 해결.
구현:
- sec_8k_parser.py: 8-K primary document HTML 파싱
- extract_items(): regex 기반 Item 헤더 추출, 목차 중복 제거 (last-wins)
- _strip_ixbrl_viewer(): documents_json의 /ix?doc=... URL → 직접 URL 변환
- _find_primary_doc_url(): primary_document_url 우선 사용 (iXBRL viewer 회피)
- Item 8.01 단독 filing: exhibit(9.01) 없이 본문에서 직접 content 추출
- Item 9.01 skip, 나머지는 ITEM_EVENT_MAP으로 event_type 분류
- Exhibit enrichment: 2.02/7.01/8.01 + EX-99.1 있을 때 exhibit content 우선
- sec_filing_events 테이블 신설 (UniqueConstraint: accession_number + item_number)
- sec_filings 테이블에 parsed_status / items_json 컬럼 추가
- index_filings() 완료 후 신규 8-K auto-parse 트리거
- GET /filings/events/{ticker}: lazy parse + 조회
- POST /filings/events/parse/bulk: backfill용 일괄 파싱
- FilingSummary에 parsed_status / items 필드 포함
- alembic migration: g8a9b0c1d2e3
- 테스트 22개 추가 (extract_items, strip_ixbrl, find_primary_doc, parse_filing)
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
main
I Luk Kim
4 months ago
parent
3eb736445b
commit
25603c390c
@ -0,0 +1,81 @@
|
|||||||
|
"""add 8-K parsing: parsed_status/items_json + sec_filing_events
|
||||||
|
|
||||||
|
Revision ID: g8a9b0c1d2e3
|
||||||
|
Revises: f7a8b9c0d1e2
|
||||||
|
Create Date: 2026-04-08
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from sqlalchemy.dialects import postgresql
|
||||||
|
|
||||||
|
revision: str = "g8a9b0c1d2e3"
|
||||||
|
down_revision: Union[str, Sequence[str], None] = "f7a8b9c0d1e2"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
conn = op.get_bind()
|
||||||
|
|
||||||
|
# Add parsed_status + items_json to sec_filings
|
||||||
|
col_names = [row[0] for row in conn.execute(
|
||||||
|
sa.text("SELECT column_name FROM information_schema.columns WHERE table_name='sec_filings'")
|
||||||
|
)]
|
||||||
|
if "parsed_status" not in col_names:
|
||||||
|
op.add_column(
|
||||||
|
"sec_filings",
|
||||||
|
sa.Column("parsed_status", sa.String(20), server_default="pending", nullable=True),
|
||||||
|
)
|
||||||
|
if "items_json" not in col_names:
|
||||||
|
op.add_column(
|
||||||
|
"sec_filings",
|
||||||
|
sa.Column("items_json", postgresql.JSON(), nullable=True),
|
||||||
|
)
|
||||||
|
|
||||||
|
# Mark all existing 8-K filings as 'pending' so the parser can backfill them
|
||||||
|
conn.execute(
|
||||||
|
sa.text(
|
||||||
|
"UPDATE sec_filings SET parsed_status = 'pending' "
|
||||||
|
"WHERE form_type IN ('8-K', '8-K/A') AND parsed_status IS NULL"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
# Create sec_filing_events table
|
||||||
|
if not conn.dialect.has_table(conn, "sec_filing_events"):
|
||||||
|
op.create_table(
|
||||||
|
"sec_filing_events",
|
||||||
|
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||||
|
sa.Column("ticker", sa.String(10), nullable=False),
|
||||||
|
sa.Column("accession_number", sa.String(30), nullable=False),
|
||||||
|
sa.Column("form_type", sa.String(20), nullable=False),
|
||||||
|
sa.Column("filing_date", postgresql.TIMESTAMP(timezone=True), nullable=False),
|
||||||
|
sa.Column("item_number", sa.String(10), nullable=False),
|
||||||
|
sa.Column("event_type", sa.String(50), nullable=False),
|
||||||
|
sa.Column("title", sa.String(512), nullable=True),
|
||||||
|
sa.Column("summary", sa.Text(), nullable=True),
|
||||||
|
sa.Column("content_source", sa.String(20), nullable=True),
|
||||||
|
sa.Column("created_at", postgresql.TIMESTAMP(timezone=True), nullable=True),
|
||||||
|
sa.Column("updated_at", postgresql.TIMESTAMP(timezone=True), nullable=True),
|
||||||
|
sa.UniqueConstraint("accession_number", "item_number", name="uq_filing_event"),
|
||||||
|
)
|
||||||
|
op.create_index("ix_filing_events_ticker", "sec_filing_events", ["ticker"])
|
||||||
|
op.create_index("ix_filing_events_accession", "sec_filing_events", ["accession_number"])
|
||||||
|
op.create_index(
|
||||||
|
"idx_filing_events_ticker_date",
|
||||||
|
"sec_filing_events",
|
||||||
|
["ticker", "filing_date"],
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"idx_filing_events_ticker_type",
|
||||||
|
"sec_filing_events",
|
||||||
|
["ticker", "event_type"],
|
||||||
|
)
|
||||||
|
op.create_index("idx_filing_events_date", "sec_filing_events", ["filing_date"])
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_table("sec_filing_events")
|
||||||
|
op.drop_column("sec_filings", "items_json")
|
||||||
|
op.drop_column("sec_filings", "parsed_status")
|
||||||
@ -0,0 +1,39 @@
|
|||||||
|
"""
|
||||||
|
Database model for parsed SEC 8-K filing events.
|
||||||
|
Each row represents one Item extracted from an 8-K primary document.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
from sqlalchemy import Column, String, Text, Index
|
||||||
|
from sqlalchemy.dialects.postgresql import UUID, TIMESTAMP, JSON
|
||||||
|
|
||||||
|
from app.core.database import Base
|
||||||
|
|
||||||
|
|
||||||
|
class SECFilingEvent(Base):
|
||||||
|
__tablename__ = "sec_filing_events"
|
||||||
|
|
||||||
|
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||||
|
ticker = Column(String(10), nullable=False, index=True)
|
||||||
|
accession_number = Column(String(30), nullable=False, index=True)
|
||||||
|
form_type = Column(String(20), nullable=False)
|
||||||
|
filing_date = Column(TIMESTAMP(timezone=True), nullable=False)
|
||||||
|
item_number = Column(String(10), nullable=False) # e.g. "8.01"
|
||||||
|
event_type = Column(String(50), nullable=False) # e.g. "other_material_event"
|
||||||
|
title = Column(String(512)) # Item title from document
|
||||||
|
summary = Column(Text) # Extracted text content (truncated)
|
||||||
|
content_source = Column(String(20)) # "primary_doc" or "exhibit"
|
||||||
|
created_at = Column(TIMESTAMP(timezone=True), default=lambda: datetime.now(timezone.utc))
|
||||||
|
updated_at = Column(
|
||||||
|
TIMESTAMP(timezone=True),
|
||||||
|
default=lambda: datetime.now(timezone.utc),
|
||||||
|
onupdate=lambda: datetime.now(timezone.utc),
|
||||||
|
)
|
||||||
|
|
||||||
|
__table_args__ = (
|
||||||
|
Index("idx_filing_events_ticker_date", "ticker", "filing_date"),
|
||||||
|
Index("idx_filing_events_ticker_type", "ticker", "event_type"),
|
||||||
|
Index("idx_filing_events_date", "filing_date"),
|
||||||
|
)
|
||||||
@ -0,0 +1,431 @@
|
|||||||
|
"""
|
||||||
|
SEC 8-K Parser Service
|
||||||
|
|
||||||
|
Parses 8-K primary document HTML to extract Items and create structured events.
|
||||||
|
Supports Item 8.01 standalone (no exhibit required) by reading primary doc body.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
import time as _time
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from typing import Dict, List, Optional, Tuple
|
||||||
|
|
||||||
|
import uuid
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
from sqlalchemy import select, and_, func
|
||||||
|
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
|
from app.models.filing import SECFiling
|
||||||
|
from app.models.filing_event import SECFilingEvent
|
||||||
|
from app.services.sec_http_client import SECHttpClient
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# Maximum text size for summary storage
|
||||||
|
MAX_SUMMARY_CHARS = 10_000
|
||||||
|
# Maximum size for primary document fetch
|
||||||
|
MAX_PRIMARY_DOC_SIZE = 5 * 1024 * 1024 # 5 MB
|
||||||
|
|
||||||
|
# Item regex: matches "Item X.XX" with optional punctuation and title
|
||||||
|
ITEM_RE = re.compile(
|
||||||
|
r"Item\s+(\d+\.\d+)\s*[.\:\u2014\u2013\u2012\-]?\s*(.*)",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
# End-of-items marker
|
||||||
|
END_MARKER_RE = re.compile(r"\bSIGNATURE[S]?\b", re.IGNORECASE)
|
||||||
|
|
||||||
|
# Batch size for bulk parse queries
|
||||||
|
_BULK_CHUNK = 50
|
||||||
|
|
||||||
|
|
||||||
|
class SEC8KParser:
|
||||||
|
"""Parse 8-K primary documents and persist structured events to DB."""
|
||||||
|
|
||||||
|
# Maps 8-K Item numbers to semantic event types
|
||||||
|
ITEM_EVENT_MAP: Dict[str, Optional[str]] = {
|
||||||
|
"1.01": "material_contract",
|
||||||
|
"1.02": "contract_termination",
|
||||||
|
"1.03": "bankruptcy",
|
||||||
|
"1.04": "mine_safety",
|
||||||
|
"2.01": "acquisition_disposition",
|
||||||
|
"2.02": "earnings_result",
|
||||||
|
"2.03": "financial_obligation",
|
||||||
|
"2.04": "triggering_event",
|
||||||
|
"2.05": "exit_activity",
|
||||||
|
"2.06": "material_impairment",
|
||||||
|
"3.01": "delisting_notice",
|
||||||
|
"3.02": "unregistered_equity_sale",
|
||||||
|
"3.03": "rights_modification",
|
||||||
|
"4.01": "accountant_change",
|
||||||
|
"4.02": "financial_restatement",
|
||||||
|
"5.01": "control_change",
|
||||||
|
"5.02": "management_change",
|
||||||
|
"5.03": "articles_amendment",
|
||||||
|
"5.05": "bylaws_amendment",
|
||||||
|
"5.06": "shell_status_change",
|
||||||
|
"5.07": "shareholder_vote",
|
||||||
|
"5.08": "shareholder_nomination",
|
||||||
|
"7.01": "regulation_fd",
|
||||||
|
"8.01": "other_material_event",
|
||||||
|
"9.01": None, # Exhibits listing — not a separate event
|
||||||
|
}
|
||||||
|
|
||||||
|
# Items that may have richer content in EX-99.1 (press release)
|
||||||
|
EXHIBIT_ENRICHABLE = {"2.02", "7.01", "8.01"}
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self._http = SECHttpClient("Stock Oracle 8K Parser")
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Public: parse one filing by accession number
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
async def parse_filing(self, db: AsyncSession, accession_number: str) -> int:
|
||||||
|
"""Parse a single filing. Returns number of events created/updated."""
|
||||||
|
result = await db.execute(
|
||||||
|
select(SECFiling).where(SECFiling.accession_number == accession_number)
|
||||||
|
)
|
||||||
|
filing = result.scalar_one_or_none()
|
||||||
|
if not filing:
|
||||||
|
raise ValueError(f"Filing {accession_number} not found in DB")
|
||||||
|
return await self._parse_one(db, filing)
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Public: parse multiple filings by accession numbers
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
async def parse_filings_by_accessions(
|
||||||
|
self, db: AsyncSession, accession_numbers: List[str]
|
||||||
|
) -> Dict[str, int]:
|
||||||
|
"""Parse multiple filings. Returns {accession: events_count}."""
|
||||||
|
result = await db.execute(
|
||||||
|
select(SECFiling).where(SECFiling.accession_number.in_(accession_numbers))
|
||||||
|
)
|
||||||
|
filings = result.scalars().all()
|
||||||
|
results: Dict[str, int] = {}
|
||||||
|
for filing in filings:
|
||||||
|
try:
|
||||||
|
n = await self._parse_one(db, filing)
|
||||||
|
results[filing.accession_number] = n
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"Parse failed for {filing.accession_number}: {e}")
|
||||||
|
results[filing.accession_number] = 0
|
||||||
|
return results
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Public: bulk parse pending 8-K filings
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
async def parse_bulk(
|
||||||
|
self,
|
||||||
|
db: AsyncSession,
|
||||||
|
tickers: Optional[List[str]] = None,
|
||||||
|
limit: int = 100,
|
||||||
|
) -> Dict:
|
||||||
|
"""Parse pending 8-K filings, optionally filtered by ticker.
|
||||||
|
|
||||||
|
Returns summary dict with succeeded/failed/skipped counts.
|
||||||
|
"""
|
||||||
|
t0 = _time.monotonic()
|
||||||
|
|
||||||
|
conditions = [
|
||||||
|
SECFiling.parsed_status == "pending",
|
||||||
|
SECFiling.form_type.in_(["8-K", "8-K/A"]),
|
||||||
|
]
|
||||||
|
if tickers:
|
||||||
|
conditions.append(SECFiling.ticker.in_([t.upper() for t in tickers]))
|
||||||
|
|
||||||
|
result = await db.execute(
|
||||||
|
select(SECFiling)
|
||||||
|
.where(and_(*conditions))
|
||||||
|
.order_by(SECFiling.filing_date.desc())
|
||||||
|
.limit(limit)
|
||||||
|
)
|
||||||
|
filings = result.scalars().all()
|
||||||
|
|
||||||
|
if not filings:
|
||||||
|
return {
|
||||||
|
"succeeded": 0,
|
||||||
|
"failed": 0,
|
||||||
|
"skipped": 0,
|
||||||
|
"total": 0,
|
||||||
|
"elapsed": 0.0,
|
||||||
|
}
|
||||||
|
|
||||||
|
sem = asyncio.Semaphore(4)
|
||||||
|
succeeded = failed = skipped = 0
|
||||||
|
|
||||||
|
async def _parse_limited(f: SECFiling) -> str:
|
||||||
|
async with sem:
|
||||||
|
from app.core.database import AsyncSessionLocal
|
||||||
|
async with AsyncSessionLocal() as session:
|
||||||
|
try:
|
||||||
|
n = await self._parse_one(session, f)
|
||||||
|
return "succeeded" if n > 0 else "skipped"
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"Bulk parse failed {f.accession_number}: {e}")
|
||||||
|
return "failed"
|
||||||
|
|
||||||
|
outcomes = await asyncio.gather(*[_parse_limited(f) for f in filings])
|
||||||
|
for outcome in outcomes:
|
||||||
|
if outcome == "succeeded":
|
||||||
|
succeeded += 1
|
||||||
|
elif outcome == "failed":
|
||||||
|
failed += 1
|
||||||
|
else:
|
||||||
|
skipped += 1
|
||||||
|
|
||||||
|
return {
|
||||||
|
"succeeded": succeeded,
|
||||||
|
"failed": failed,
|
||||||
|
"skipped": skipped,
|
||||||
|
"total": len(filings),
|
||||||
|
"elapsed": round(_time.monotonic() - t0, 3),
|
||||||
|
}
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Internal: parse a single SECFiling object
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
async def _parse_one(self, db: AsyncSession, filing: SECFiling) -> int:
|
||||||
|
"""Fetch and parse the primary document of a filing. Returns event count."""
|
||||||
|
self._http.set_deadline(60.0)
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
accession_number = filing.accession_number
|
||||||
|
|
||||||
|
# Re-fetch filing in the current session to ensure it is tracked for updates
|
||||||
|
result = await db.execute(
|
||||||
|
select(SECFiling).where(SECFiling.accession_number == accession_number)
|
||||||
|
)
|
||||||
|
filing = result.scalar_one_or_none()
|
||||||
|
if not filing:
|
||||||
|
raise ValueError(f"Filing {accession_number} not found in DB")
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Get document list (cached in documents_json)
|
||||||
|
from app.services.sec_filings_service import sec_filings_service
|
||||||
|
docs = await sec_filings_service.get_filing_documents(db, filing.accession_number)
|
||||||
|
|
||||||
|
# Find primary document URL
|
||||||
|
primary_url = _find_primary_doc_url(docs, filing)
|
||||||
|
if not primary_url:
|
||||||
|
logger.warning(
|
||||||
|
f"No primary document found for {filing.accession_number}, skipping"
|
||||||
|
)
|
||||||
|
filing.parsed_status = "skipped"
|
||||||
|
filing.updated_at = now
|
||||||
|
await db.commit()
|
||||||
|
return 0
|
||||||
|
|
||||||
|
# Fetch and parse HTML
|
||||||
|
html = await self._http.fetch_text(primary_url, max_bytes=MAX_PRIMARY_DOC_SIZE)
|
||||||
|
items = extract_items(html)
|
||||||
|
|
||||||
|
if not items:
|
||||||
|
logger.info(f"No items found in {filing.accession_number}, marking skipped")
|
||||||
|
filing.parsed_status = "skipped"
|
||||||
|
filing.items_json = []
|
||||||
|
filing.updated_at = now
|
||||||
|
await db.commit()
|
||||||
|
return 0
|
||||||
|
|
||||||
|
# Build event rows
|
||||||
|
item_numbers = [item["number"] for item in items]
|
||||||
|
events_to_upsert = []
|
||||||
|
for item in items:
|
||||||
|
event_type = self.ITEM_EVENT_MAP.get(item["number"], "other")
|
||||||
|
if event_type is None:
|
||||||
|
continue # skip 9.01 (exhibits listing)
|
||||||
|
|
||||||
|
content = item["content"]
|
||||||
|
content_source = "primary_doc"
|
||||||
|
|
||||||
|
# Optionally enrich with exhibit content for press-release items
|
||||||
|
if item["number"] in self.EXHIBIT_ENRICHABLE:
|
||||||
|
exhibit_content = await _try_exhibit_content(
|
||||||
|
db, filing.accession_number, docs
|
||||||
|
)
|
||||||
|
if exhibit_content and len(exhibit_content) > len(content):
|
||||||
|
content = exhibit_content
|
||||||
|
content_source = "exhibit"
|
||||||
|
|
||||||
|
summary = content[:MAX_SUMMARY_CHARS] if content else None
|
||||||
|
title = item.get("title") or filing.filing_description or ""
|
||||||
|
|
||||||
|
events_to_upsert.append({
|
||||||
|
"id": uuid.uuid4(),
|
||||||
|
"ticker": filing.ticker,
|
||||||
|
"accession_number": filing.accession_number,
|
||||||
|
"form_type": filing.form_type,
|
||||||
|
"filing_date": filing.filing_date,
|
||||||
|
"item_number": item["number"],
|
||||||
|
"event_type": event_type,
|
||||||
|
"title": title[:512] if title else None,
|
||||||
|
"summary": summary,
|
||||||
|
"content_source": content_source,
|
||||||
|
"created_at": now,
|
||||||
|
"updated_at": now,
|
||||||
|
})
|
||||||
|
|
||||||
|
if events_to_upsert:
|
||||||
|
stmt = pg_insert(SECFilingEvent).values(events_to_upsert)
|
||||||
|
stmt = stmt.on_conflict_do_update(
|
||||||
|
constraint="uq_filing_event",
|
||||||
|
set_={
|
||||||
|
"event_type": stmt.excluded.event_type,
|
||||||
|
"title": stmt.excluded.title,
|
||||||
|
"summary": stmt.excluded.summary,
|
||||||
|
"content_source": stmt.excluded.content_source,
|
||||||
|
"updated_at": stmt.excluded.updated_at,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
await db.execute(stmt)
|
||||||
|
|
||||||
|
# Update filing parse status
|
||||||
|
filing.parsed_status = "succeeded" if events_to_upsert else "skipped"
|
||||||
|
filing.items_json = item_numbers
|
||||||
|
filing.updated_at = now
|
||||||
|
await db.commit()
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
f"Parsed {filing.accession_number} ({filing.ticker}): "
|
||||||
|
f"{len(events_to_upsert)} events from items {item_numbers}"
|
||||||
|
)
|
||||||
|
return len(events_to_upsert)
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"Parse error for {filing.accession_number}: {e}")
|
||||||
|
try:
|
||||||
|
filing.parsed_status = "failed"
|
||||||
|
filing.updated_at = now
|
||||||
|
await db.commit()
|
||||||
|
except Exception:
|
||||||
|
await db.rollback()
|
||||||
|
raise
|
||||||
|
finally:
|
||||||
|
self._http.clear_deadline()
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
# Module-level helpers
|
||||||
|
# ------------------------------------------------------------------
|
||||||
|
|
||||||
|
def extract_items(html: str) -> List[Dict]:
|
||||||
|
"""Extract 8-K items from primary document HTML.
|
||||||
|
|
||||||
|
Returns list of {"number": "8.01", "title": "Other Events", "content": "..."}.
|
||||||
|
Deduplicates item numbers (table of contents entries overwritten by body entries).
|
||||||
|
"""
|
||||||
|
soup = BeautifulSoup(html, "html.parser")
|
||||||
|
# Remove noise tags
|
||||||
|
for tag in soup.find_all(["script", "style", "ix:header"]):
|
||||||
|
tag.decompose()
|
||||||
|
text = soup.get_text(separator="\n")
|
||||||
|
|
||||||
|
# Find all Item headers
|
||||||
|
all_matches = list(ITEM_RE.finditer(text))
|
||||||
|
if not all_matches:
|
||||||
|
return []
|
||||||
|
|
||||||
|
# Deduplicate: keep the *last* occurrence of each item number
|
||||||
|
# (earlier occurrences are usually the table of contents)
|
||||||
|
seen: Dict[str, re.Match] = {}
|
||||||
|
for m in all_matches:
|
||||||
|
num = m.group(1)
|
||||||
|
seen[num] = m # last match wins
|
||||||
|
|
||||||
|
deduped = sorted(seen.values(), key=lambda m: m.start())
|
||||||
|
|
||||||
|
# Find end-of-body marker (SIGNATURES section)
|
||||||
|
last_item_end = deduped[-1].end() if deduped else 0
|
||||||
|
sig_match = END_MARKER_RE.search(text, last_item_end)
|
||||||
|
text_end = sig_match.start() if sig_match else len(text)
|
||||||
|
|
||||||
|
items: List[Dict] = []
|
||||||
|
for i, m in enumerate(deduped):
|
||||||
|
number = m.group(1)
|
||||||
|
title = m.group(2).strip().rstrip(".").strip()
|
||||||
|
content_start = m.end()
|
||||||
|
content_end = deduped[i + 1].start() if i + 1 < len(deduped) else text_end
|
||||||
|
content = text[content_start:content_end].strip()
|
||||||
|
# Clean up excessive whitespace
|
||||||
|
content = re.sub(r"\n{3,}", "\n\n", content)
|
||||||
|
items.append({"number": number, "title": title, "content": content})
|
||||||
|
|
||||||
|
return items
|
||||||
|
|
||||||
|
|
||||||
|
def _find_primary_doc_url(docs: List[Dict], filing: SECFiling) -> Optional[str]:
|
||||||
|
"""Find the primary 8-K document URL.
|
||||||
|
|
||||||
|
Prefers the stored primary_document_url (always a direct link from SEC EDGAR
|
||||||
|
submissions JSON). Falls back to documents_json with iXBRL viewer URL stripping.
|
||||||
|
"""
|
||||||
|
# Primary URL from the submissions JSON is always a direct link — prefer it
|
||||||
|
if filing.primary_document_url:
|
||||||
|
return filing.primary_document_url
|
||||||
|
|
||||||
|
if not docs:
|
||||||
|
return None
|
||||||
|
|
||||||
|
form_type_upper = filing.form_type.upper()
|
||||||
|
# Exact type match (e.g., "8-K")
|
||||||
|
for doc in docs:
|
||||||
|
doc_type = (doc.get("type") or "").upper()
|
||||||
|
if doc_type == form_type_upper:
|
||||||
|
return _strip_ixbrl_viewer(doc.get("url") or "")
|
||||||
|
|
||||||
|
# Fallback: type contains form type base (strip "/A")
|
||||||
|
for doc in docs:
|
||||||
|
doc_type = (doc.get("type") or "").upper()
|
||||||
|
if form_type_upper.replace("/A", "") in doc_type:
|
||||||
|
return _strip_ixbrl_viewer(doc.get("url") or "")
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _strip_ixbrl_viewer(url: str) -> str:
|
||||||
|
"""Convert '/ix?doc=/Archives/...' URLs to direct document URLs."""
|
||||||
|
# Pattern: https://www.sec.gov/ix?doc=/Archives/edgar/data/...
|
||||||
|
if "/ix?doc=" in url:
|
||||||
|
idx = url.index("/ix?doc=")
|
||||||
|
path = url[idx + len("/ix?doc="):] # e.g. /Archives/edgar/...
|
||||||
|
if path.startswith("/"):
|
||||||
|
return "https://www.sec.gov" + path
|
||||||
|
return url
|
||||||
|
|
||||||
|
|
||||||
|
async def _try_exhibit_content(
|
||||||
|
db: AsyncSession,
|
||||||
|
accession_number: str,
|
||||||
|
docs: List[Dict],
|
||||||
|
) -> Optional[str]:
|
||||||
|
"""Try to fetch EX-99.1 exhibit content. Returns None on failure."""
|
||||||
|
has_exhibit = any(
|
||||||
|
(d.get("type") or "").upper() == "EX-99.1" for d in docs
|
||||||
|
)
|
||||||
|
if not has_exhibit:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
from app.services.sec_filings_service import sec_filings_service
|
||||||
|
result = await asyncio.wait_for(
|
||||||
|
sec_filings_service.get_exhibit_content(db, accession_number, "EX-99.1"),
|
||||||
|
timeout=20,
|
||||||
|
)
|
||||||
|
content = result.get("content", "")
|
||||||
|
if content:
|
||||||
|
# Strip HTML tags from exhibit content
|
||||||
|
soup = BeautifulSoup(content, "html.parser")
|
||||||
|
for tag in soup.find_all(["script", "style"]):
|
||||||
|
tag.decompose()
|
||||||
|
return soup.get_text(separator="\n").strip()
|
||||||
|
return None
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
# Singleton
|
||||||
|
sec_8k_parser = SEC8KParser()
|
||||||
@ -0,0 +1,395 @@
|
|||||||
|
"""
|
||||||
|
Unit tests for SEC 8-K parser service.
|
||||||
|
|
||||||
|
Tests:
|
||||||
|
- extract_items(): HTML parsing, item extraction, deduplication
|
||||||
|
- parse_filing(): end-to-end with mocked HTTP
|
||||||
|
- _find_primary_doc_url(): iXBRL URL stripping
|
||||||
|
- _strip_ixbrl_viewer(): URL conversion
|
||||||
|
"""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import pytest
|
||||||
|
import uuid
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from unittest.mock import AsyncMock, MagicMock, patch
|
||||||
|
|
||||||
|
from app.services.sec_8k_parser import (
|
||||||
|
SEC8KParser,
|
||||||
|
extract_items,
|
||||||
|
_find_primary_doc_url,
|
||||||
|
_strip_ixbrl_viewer,
|
||||||
|
)
|
||||||
|
from app.models.filing import SECFiling
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Fixtures
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _make_filing(**kwargs):
|
||||||
|
defaults = dict(
|
||||||
|
id=uuid.uuid4(),
|
||||||
|
ticker="AVGO",
|
||||||
|
cik="1730168",
|
||||||
|
accession_number="0001193125-26-144028",
|
||||||
|
form_type="8-K",
|
||||||
|
filing_date=datetime(2026, 4, 6, tzinfo=timezone.utc),
|
||||||
|
primary_document="d87999d8k.htm",
|
||||||
|
primary_document_url="https://www.sec.gov/Archives/edgar/data/1730168/000119312526144028/d87999d8k.htm",
|
||||||
|
filing_description=None,
|
||||||
|
documents_json=None,
|
||||||
|
parsed_status="pending",
|
||||||
|
items_json=None,
|
||||||
|
)
|
||||||
|
defaults.update(kwargs)
|
||||||
|
f = MagicMock(spec=SECFiling)
|
||||||
|
for k, v in defaults.items():
|
||||||
|
setattr(f, k, v)
|
||||||
|
return f
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# TestExtractItems: HTML parsing
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestExtractItems:
|
||||||
|
|
||||||
|
def test_single_item_8_01(self):
|
||||||
|
html = """
|
||||||
|
<html><body>
|
||||||
|
<p>Item 8.01 Other Events.</p>
|
||||||
|
<p>Broadcom and Google signed a TPU supply agreement.</p>
|
||||||
|
<p>SIGNATURES</p>
|
||||||
|
</body></html>
|
||||||
|
"""
|
||||||
|
items = extract_items(html)
|
||||||
|
assert len(items) == 1
|
||||||
|
assert items[0]["number"] == "8.01"
|
||||||
|
assert "Broadcom" in items[0]["content"]
|
||||||
|
assert "SIGNATURES" not in items[0]["content"]
|
||||||
|
|
||||||
|
def test_multiple_items(self):
|
||||||
|
html = """
|
||||||
|
<html><body>
|
||||||
|
<p>Item 2.02 Results of Operations.</p>
|
||||||
|
<p>Q1 revenue was $14.9 billion.</p>
|
||||||
|
<p>Item 9.01 Financial Statements and Exhibits.</p>
|
||||||
|
<p>(d) Exhibits.</p>
|
||||||
|
<p>SIGNATURES</p>
|
||||||
|
</body></html>
|
||||||
|
"""
|
||||||
|
items = extract_items(html)
|
||||||
|
assert len(items) == 2
|
||||||
|
assert items[0]["number"] == "2.02"
|
||||||
|
assert "revenue" in items[0]["content"]
|
||||||
|
assert items[1]["number"] == "9.01"
|
||||||
|
|
||||||
|
def test_deduplication_removes_toc_entries(self):
|
||||||
|
"""Table of contents entries appear before body — last occurrence wins."""
|
||||||
|
html = """
|
||||||
|
<html><body>
|
||||||
|
<p>TABLE OF CONTENTS</p>
|
||||||
|
<p>Item 8.01 Other Events...1</p>
|
||||||
|
<p>Item 9.01 Financial Statements...2</p>
|
||||||
|
<p>Item 8.01 Other Events.</p>
|
||||||
|
<p>This is the actual body content.</p>
|
||||||
|
<p>SIGNATURES</p>
|
||||||
|
</body></html>
|
||||||
|
"""
|
||||||
|
items = extract_items(html)
|
||||||
|
# Only body occurrences should survive
|
||||||
|
numbers = [i["number"] for i in items]
|
||||||
|
assert numbers.count("8.01") == 1
|
||||||
|
# The body occurrence should have actual content
|
||||||
|
item_8 = next(i for i in items if i["number"] == "8.01")
|
||||||
|
assert "body content" in item_8["content"]
|
||||||
|
|
||||||
|
def test_content_stops_at_signatures(self):
|
||||||
|
html = """
|
||||||
|
<html><body>
|
||||||
|
<p>Item 8.01 Other Events.</p>
|
||||||
|
<p>Material contract signed.</p>
|
||||||
|
<p>SIGNATURES</p>
|
||||||
|
<p>John Doe, CEO</p>
|
||||||
|
</body></html>
|
||||||
|
"""
|
||||||
|
items = extract_items(html)
|
||||||
|
assert "SIGNATURES" not in items[0]["content"]
|
||||||
|
assert "John Doe" not in items[0]["content"]
|
||||||
|
|
||||||
|
def test_empty_html_returns_empty(self):
|
||||||
|
items = extract_items("<html><body><p>No items here.</p></body></html>")
|
||||||
|
assert items == []
|
||||||
|
|
||||||
|
def test_item_title_extracted(self):
|
||||||
|
html = """
|
||||||
|
<html><body>
|
||||||
|
<p>Item 8.01 Other Events.</p>
|
||||||
|
<p>Some content.</p>
|
||||||
|
<p>SIGNATURES</p>
|
||||||
|
</body></html>
|
||||||
|
"""
|
||||||
|
items = extract_items(html)
|
||||||
|
assert items[0]["title"] == "Other Events"
|
||||||
|
|
||||||
|
def test_script_tags_removed(self):
|
||||||
|
html = """
|
||||||
|
<html><body>
|
||||||
|
<script>var x = 'Item 5.02 Fake';</script>
|
||||||
|
<p>Item 8.01 Other Events.</p>
|
||||||
|
<p>Real content here.</p>
|
||||||
|
<p>SIGNATURES</p>
|
||||||
|
</body></html>
|
||||||
|
"""
|
||||||
|
items = extract_items(html)
|
||||||
|
# Script tag contents should not produce items
|
||||||
|
assert len(items) == 1
|
||||||
|
assert items[0]["number"] == "8.01"
|
||||||
|
|
||||||
|
def test_three_items_boundaries(self):
|
||||||
|
html = """
|
||||||
|
<html><body>
|
||||||
|
<p>Item 2.02 Results of Operations.</p>
|
||||||
|
<p>Revenue $10B.</p>
|
||||||
|
<p>Item 8.01 Other Events.</p>
|
||||||
|
<p>Partnership signed.</p>
|
||||||
|
<p>Item 9.01 Financial Statements.</p>
|
||||||
|
<p>Exhibit list.</p>
|
||||||
|
<p>SIGNATURES</p>
|
||||||
|
</body></html>
|
||||||
|
"""
|
||||||
|
items = extract_items(html)
|
||||||
|
assert len(items) == 3
|
||||||
|
assert "Revenue" in items[0]["content"]
|
||||||
|
assert "Partnership" in items[1]["content"]
|
||||||
|
assert "Exhibit" in items[2]["content"]
|
||||||
|
# No cross-contamination
|
||||||
|
assert "Partnership" not in items[0]["content"]
|
||||||
|
assert "Revenue" not in items[1]["content"]
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# TestStripIxbrlViewer
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestStripIxbrlViewer:
|
||||||
|
|
||||||
|
def test_strips_ix_doc_prefix(self):
|
||||||
|
url = "https://www.sec.gov/ix?doc=/Archives/edgar/data/1730168/000119312526144028/d87999d8k.htm"
|
||||||
|
result = _strip_ixbrl_viewer(url)
|
||||||
|
assert result == "https://www.sec.gov/Archives/edgar/data/1730168/000119312526144028/d87999d8k.htm"
|
||||||
|
|
||||||
|
def test_passthrough_for_direct_url(self):
|
||||||
|
url = "https://www.sec.gov/Archives/edgar/data/320193/000032019324000006/a8k.htm"
|
||||||
|
assert _strip_ixbrl_viewer(url) == url
|
||||||
|
|
||||||
|
def test_empty_string(self):
|
||||||
|
assert _strip_ixbrl_viewer("") == ""
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# TestFindPrimaryDocUrl
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestFindPrimaryDocUrl:
|
||||||
|
|
||||||
|
def test_prefers_primary_document_url(self):
|
||||||
|
filing = _make_filing(
|
||||||
|
primary_document_url="https://www.sec.gov/Archives/edgar/data/123/000123/doc.htm"
|
||||||
|
)
|
||||||
|
docs = [
|
||||||
|
{"type": "8-K", "url": "https://www.sec.gov/ix?doc=/Archives/edgar/data/123/000123/doc.htm"}
|
||||||
|
]
|
||||||
|
result = _find_primary_doc_url(docs, filing)
|
||||||
|
# Should use the stored primary_document_url, not the ix?doc= URL
|
||||||
|
assert result == "https://www.sec.gov/Archives/edgar/data/123/000123/doc.htm"
|
||||||
|
|
||||||
|
def test_falls_back_to_docs_when_no_primary_url(self):
|
||||||
|
filing = _make_filing(primary_document_url=None)
|
||||||
|
docs = [
|
||||||
|
{"type": "8-K", "url": "https://www.sec.gov/Archives/edgar/data/123/000123/doc.htm"}
|
||||||
|
]
|
||||||
|
result = _find_primary_doc_url(docs, filing)
|
||||||
|
assert result == "https://www.sec.gov/Archives/edgar/data/123/000123/doc.htm"
|
||||||
|
|
||||||
|
def test_strips_ixbrl_in_fallback(self):
|
||||||
|
filing = _make_filing(primary_document_url=None)
|
||||||
|
docs = [
|
||||||
|
{"type": "8-K", "url": "https://www.sec.gov/ix?doc=/Archives/edgar/data/123/000123/doc.htm"}
|
||||||
|
]
|
||||||
|
result = _find_primary_doc_url(docs, filing)
|
||||||
|
assert "/ix?doc=" not in result
|
||||||
|
|
||||||
|
def test_returns_none_for_empty_docs_and_no_url(self):
|
||||||
|
filing = _make_filing(primary_document_url=None)
|
||||||
|
result = _find_primary_doc_url([], filing)
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
def test_form_type_8k_a_matching(self):
|
||||||
|
filing = _make_filing(form_type="8-K/A", primary_document_url=None)
|
||||||
|
docs = [
|
||||||
|
{"type": "8-K/A", "url": "https://www.sec.gov/Archives/edgar/data/123/000123/doc.htm"}
|
||||||
|
]
|
||||||
|
result = _find_primary_doc_url(docs, filing)
|
||||||
|
assert result is not None
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# TestSEC8KParserItemEventMap
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestItemEventMap:
|
||||||
|
|
||||||
|
def test_key_items_present(self):
|
||||||
|
parser = SEC8KParser()
|
||||||
|
assert parser.ITEM_EVENT_MAP["8.01"] == "other_material_event"
|
||||||
|
assert parser.ITEM_EVENT_MAP["2.02"] == "earnings_result"
|
||||||
|
assert parser.ITEM_EVENT_MAP["5.02"] == "management_change"
|
||||||
|
assert parser.ITEM_EVENT_MAP["1.01"] == "material_contract"
|
||||||
|
assert parser.ITEM_EVENT_MAP["9.01"] is None # skip
|
||||||
|
|
||||||
|
def test_exhibit_enrichable_set(self):
|
||||||
|
parser = SEC8KParser()
|
||||||
|
assert "8.01" in parser.EXHIBIT_ENRICHABLE
|
||||||
|
assert "2.02" in parser.EXHIBIT_ENRICHABLE
|
||||||
|
assert "9.01" not in parser.EXHIBIT_ENRICHABLE
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# TestParseFiling: end-to-end with mocks
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
AVGO_8K_HTML = """
|
||||||
|
<html><body>
|
||||||
|
<p>FORM 8-K CURRENT REPORT</p>
|
||||||
|
<p>Date of Report: April 6, 2026</p>
|
||||||
|
<p>Broadcom Inc.</p>
|
||||||
|
<p>Item 8.01 Other Events.</p>
|
||||||
|
<p>Broadcom Inc. and Google LLC have entered into a Long Term Agreement
|
||||||
|
for Broadcom to develop and supply custom Tensor Processing Units (TPUs)
|
||||||
|
for Google's future generations of TPUs.</p>
|
||||||
|
<p>Cautionary Note Regarding Forward-Looking Statements</p>
|
||||||
|
<p>SIGNATURES</p>
|
||||||
|
<p>John Doe, CEO</p>
|
||||||
|
</body></html>
|
||||||
|
"""
|
||||||
|
|
||||||
|
MULTI_ITEM_HTML = """
|
||||||
|
<html><body>
|
||||||
|
<p>Item 2.02 Results of Operations.</p>
|
||||||
|
<p>Q1 2026 revenue was $14.9 billion.</p>
|
||||||
|
<p>Item 8.01 Other Events.</p>
|
||||||
|
<p>Google partnership announced.</p>
|
||||||
|
<p>Item 9.01 Financial Statements and Exhibits.</p>
|
||||||
|
<p>(d) Exhibits. See exhibit index.</p>
|
||||||
|
<p>SIGNATURES</p>
|
||||||
|
</body></html>
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
class TestParseFiling:
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_parse_item_8_01_standalone(self):
|
||||||
|
"""Item 8.01 standalone — no exhibit needed, content from primary doc."""
|
||||||
|
parser = SEC8KParser()
|
||||||
|
filing = _make_filing()
|
||||||
|
|
||||||
|
docs = [
|
||||||
|
{"type": "8-K", "url": "https://www.sec.gov/Archives/edgar/data/1730168/000119312526144028/d87999d8k.htm"},
|
||||||
|
]
|
||||||
|
|
||||||
|
# lazy import patched at the source module
|
||||||
|
with patch("app.services.sec_filings_service.sec_filings_service") as mock_svc:
|
||||||
|
mock_svc.get_filing_documents = AsyncMock(return_value=docs)
|
||||||
|
with patch.object(parser._http, "fetch_text", new_callable=AsyncMock) as mock_fetch:
|
||||||
|
mock_fetch.return_value = AVGO_8K_HTML
|
||||||
|
with patch("app.services.sec_8k_parser.pg_insert") as mock_pg_insert:
|
||||||
|
mock_stmt = MagicMock()
|
||||||
|
mock_stmt.on_conflict_do_update.return_value = mock_stmt
|
||||||
|
mock_pg_insert.return_value = mock_stmt
|
||||||
|
db = AsyncMock()
|
||||||
|
db.execute = AsyncMock(side_effect=[
|
||||||
|
_make_db_result(filing), # re-fetch filing
|
||||||
|
MagicMock(), # pg_insert execute
|
||||||
|
])
|
||||||
|
db.commit = AsyncMock()
|
||||||
|
|
||||||
|
n = await parser._parse_one(db, filing)
|
||||||
|
|
||||||
|
assert n == 1
|
||||||
|
assert filing.parsed_status == "succeeded"
|
||||||
|
assert filing.items_json == ["8.01"]
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_parse_multiple_items_skips_9_01(self):
|
||||||
|
"""Multiple items: 2.02+8.01+9.01, 9.01 should be skipped (no event created)."""
|
||||||
|
parser = SEC8KParser()
|
||||||
|
filing = _make_filing(accession_number="0001730168-26-000011")
|
||||||
|
docs = [
|
||||||
|
{"type": "8-K", "url": "https://www.sec.gov/Archives/edgar/data/1730168/000173016826000011/avgo.htm"},
|
||||||
|
]
|
||||||
|
|
||||||
|
with patch("app.services.sec_filings_service.sec_filings_service") as mock_svc:
|
||||||
|
mock_svc.get_filing_documents = AsyncMock(return_value=docs)
|
||||||
|
with patch.object(parser._http, "fetch_text", new_callable=AsyncMock) as mock_fetch:
|
||||||
|
mock_fetch.return_value = MULTI_ITEM_HTML
|
||||||
|
with patch("app.services.sec_8k_parser.pg_insert") as mock_pg_insert:
|
||||||
|
mock_stmt = MagicMock()
|
||||||
|
mock_stmt.on_conflict_do_update.return_value = mock_stmt
|
||||||
|
mock_pg_insert.return_value = mock_stmt
|
||||||
|
db = AsyncMock()
|
||||||
|
db.execute = AsyncMock(side_effect=[
|
||||||
|
_make_db_result(filing),
|
||||||
|
MagicMock(),
|
||||||
|
])
|
||||||
|
db.commit = AsyncMock()
|
||||||
|
|
||||||
|
n = await parser._parse_one(db, filing)
|
||||||
|
|
||||||
|
# 2 events (2.02 and 8.01), 9.01 skipped
|
||||||
|
assert n == 2
|
||||||
|
assert filing.parsed_status == "succeeded"
|
||||||
|
assert "9.01" in filing.items_json # items_json includes 9.01 (for reference)
|
||||||
|
assert "2.02" in filing.items_json
|
||||||
|
assert "8.01" in filing.items_json
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_parse_filing_sets_failed_on_error(self):
|
||||||
|
"""HTTP fetch failure → parsed_status = 'failed'."""
|
||||||
|
parser = SEC8KParser()
|
||||||
|
filing = _make_filing()
|
||||||
|
docs = [{"type": "8-K", "url": "https://www.sec.gov/Archives/doc.htm"}]
|
||||||
|
|
||||||
|
with patch("app.services.sec_filings_service.sec_filings_service") as mock_svc:
|
||||||
|
mock_svc.get_filing_documents = AsyncMock(return_value=docs)
|
||||||
|
with patch.object(parser._http, "fetch_text", new_callable=AsyncMock) as mock_fetch:
|
||||||
|
mock_fetch.side_effect = RuntimeError("network error")
|
||||||
|
db = AsyncMock()
|
||||||
|
db.execute = AsyncMock(return_value=_make_db_result(filing))
|
||||||
|
db.commit = AsyncMock()
|
||||||
|
|
||||||
|
with pytest.raises(RuntimeError, match="network error"):
|
||||||
|
await parser._parse_one(db, filing)
|
||||||
|
|
||||||
|
assert filing.parsed_status == "failed"
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_parse_filing_not_found_raises(self):
|
||||||
|
"""Filing not in DB → ValueError."""
|
||||||
|
parser = SEC8KParser()
|
||||||
|
filing = _make_filing()
|
||||||
|
db = AsyncMock()
|
||||||
|
db.execute = AsyncMock(return_value=_make_db_result(None))
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match="not found in DB"):
|
||||||
|
await parser._parse_one(db, filing)
|
||||||
|
|
||||||
|
|
||||||
|
def _make_db_result(value):
|
||||||
|
# SQLAlchemy result methods (scalar_one_or_none, scalars, etc.) are synchronous,
|
||||||
|
# so use MagicMock (not AsyncMock) to avoid returning unawaited coroutines.
|
||||||
|
result = MagicMock()
|
||||||
|
result.scalar_one_or_none.return_value = value
|
||||||
|
return result
|
||||||
Loading…
Reference in New Issue