feat: SEC Form 4 PIT + SC 13D/G activist ownership API
## 신규 엔드포인트
- GET /insider/form4/{ticker} — PIT-safe Form 4 거래 목록 (as_of 필수)
- GET /insider/form4/by-date/{date} — 특정 공시일 cross-ticker 거래
- GET /insider/form4/aggregate/{ticker} — 내부자 매수 집계 (window_days)
- GET /ownership/13dg/{ticker} — SC 13D/G activist 이벤트 (as_of 필수)
- GET /ownership/13dg/active — 현재 활성 activist 포지션 (window function PIT)
## 데이터 인프라
- activist_ownership_events 테이블 (42,329행, 최근 2년 bootstrap 완료)
- insider_transactions: is_ceo/is_cfo/is_c_suite/purchase_pct_of_holding/owner_relationship 컬럼 추가
- SECFullIndexService: company.idx fixed-width 파서, form345.zip 파서
- ActivistOwnershipService: SGML header issuer 해석 + cover-page XML/HTML enrich
- SEC ingest scheduler: 매 영업일 09:00 ET daily ingest + 30분 enrich job
- scripts: bootstrap_form4_by_ticker.py, bootstrap_form4_2y.py, bootstrap_13dg.py
## 버그 수정 (enrich 품질)
- filing_url double edgar/ prefix 제거
- XML: classPercent + amountBeneficiallyOwned + reportingPerson... 태그 추가
- HTML: 태그 제거 후 regex 적용 (CSS 10pt → shares 오인식 방지)
- HTML: ownership_pct > 100 방어 로직
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
main
I Luk Kim
4 months ago
parent
1f0017a4ea
commit
db9abd7786
@ -0,0 +1,63 @@
|
||||
"""add Form 4 derived columns (is_ceo, is_cfo, is_c_suite, purchase_pct_of_holding, owner_relationship)
|
||||
|
||||
Revision ID: l3d4e5f6g7h8
|
||||
Revises: k2c3d4e5f6g7
|
||||
Create Date: 2026-04-22
|
||||
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision = "l3d4e5f6g7h8"
|
||||
down_revision = "k2c3d4e5f6g7"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
|
||||
# Add new columns
|
||||
op.add_column("insider_transactions", sa.Column("owner_relationship", sa.Text(), nullable=True))
|
||||
op.add_column("insider_transactions", sa.Column("is_ceo", sa.Boolean(), nullable=False, server_default="false"))
|
||||
op.add_column("insider_transactions", sa.Column("is_cfo", sa.Boolean(), nullable=False, server_default="false"))
|
||||
op.add_column("insider_transactions", sa.Column("is_c_suite", sa.Boolean(), nullable=False, server_default="false"))
|
||||
op.add_column("insider_transactions", sa.Column("purchase_pct_of_holding", sa.Float(), nullable=True))
|
||||
|
||||
# Widen transaction_code from varchar(5) to varchar(10)
|
||||
op.alter_column("insider_transactions", "transaction_code",
|
||||
existing_type=sa.String(5), type_=sa.String(10))
|
||||
|
||||
# New indexes for PIT / by-date queries
|
||||
op.create_index("idx_insider_filing_ticker", "insider_transactions", ["filing_date", "ticker"])
|
||||
|
||||
# Backfill c-suite flags from existing officer_title data
|
||||
conn.execute(sa.text("""
|
||||
UPDATE insider_transactions
|
||||
SET
|
||||
is_ceo = (officer_title ~* '\\mCEO\\M|Chief\\s+Executive\\s+Officer'),
|
||||
is_cfo = (officer_title ~* '\\mCFO\\M|Chief\\s+Financial\\s+Officer|Principal\\s+Financial\\s+Officer'),
|
||||
is_c_suite = (officer_title ~* '\\mCEO\\M|Chief\\s+Executive\\s+Officer|\\mCFO\\M|Chief\\s+Financial\\s+Officer|Principal\\s+Financial\\s+Officer|\\mCOO\\M|\\mCTO\\M|\\mCIO\\M|\\mCLO\\M|\\mCMO\\M|\\mPresident\\M|\\mChair(man|person|woman)?\\M|Chief\\s+\\w+\\s+Officer')
|
||||
WHERE officer_title IS NOT NULL
|
||||
"""))
|
||||
|
||||
# Backfill purchase_pct_of_holding for existing buy transactions
|
||||
conn.execute(sa.text("""
|
||||
UPDATE insider_transactions
|
||||
SET purchase_pct_of_holding = ABS(shares) / NULLIF(shares_owned_after, 0)
|
||||
WHERE transaction_code IN ('P', 'A')
|
||||
AND shares > 0
|
||||
AND shares_owned_after IS NOT NULL
|
||||
AND shares_owned_after > 0
|
||||
"""))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("idx_insider_filing_ticker", table_name="insider_transactions")
|
||||
op.drop_column("insider_transactions", "purchase_pct_of_holding")
|
||||
op.drop_column("insider_transactions", "is_c_suite")
|
||||
op.drop_column("insider_transactions", "is_cfo")
|
||||
op.drop_column("insider_transactions", "is_ceo")
|
||||
op.drop_column("insider_transactions", "owner_relationship")
|
||||
op.alter_column("insider_transactions", "transaction_code",
|
||||
existing_type=sa.String(10), type_=sa.String(5))
|
||||
@ -0,0 +1,55 @@
|
||||
"""add activist_ownership_events table for SC 13D/13G filings
|
||||
|
||||
Revision ID: m4e5f6g7h8i9
|
||||
Revises: l3d4e5f6g7h8
|
||||
Create Date: 2026-04-22
|
||||
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
revision = "m4e5f6g7h8i9"
|
||||
down_revision = "l3d4e5f6g7h8"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
if conn.dialect.has_table(conn, "activist_ownership_events"):
|
||||
return
|
||||
|
||||
op.create_table(
|
||||
"activist_ownership_events",
|
||||
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||
sa.Column("symbol", sa.String(10), nullable=False),
|
||||
sa.Column("issuer_cik", sa.String(20), nullable=False),
|
||||
sa.Column("filer_cik", sa.String(20), nullable=False),
|
||||
sa.Column("filer_name", sa.String(500), nullable=False),
|
||||
sa.Column("accession_number", sa.String(30), nullable=False),
|
||||
sa.Column("form_type", sa.String(20), nullable=False),
|
||||
sa.Column("filing_date", sa.TIMESTAMP(timezone=True), nullable=False),
|
||||
sa.Column("is_amendment", sa.Boolean(), nullable=False, server_default="false"),
|
||||
sa.Column("ownership_pct", sa.Float(), nullable=True),
|
||||
sa.Column("shares_owned", sa.Float(), nullable=True),
|
||||
sa.Column("change_pct", sa.Float(), nullable=True),
|
||||
sa.Column("parse_status", sa.String(20), nullable=False, server_default="index_only"),
|
||||
sa.Column("filing_url", sa.Text(), nullable=True),
|
||||
sa.Column("created_at", sa.TIMESTAMP(timezone=True), nullable=False,
|
||||
server_default=sa.text("NOW()")),
|
||||
sa.Column("updated_at", sa.TIMESTAMP(timezone=True), nullable=False,
|
||||
server_default=sa.text("NOW()")),
|
||||
sa.UniqueConstraint("accession_number", "filer_cik", name="uq_activist_event"),
|
||||
)
|
||||
op.create_index("idx_activist_sym_date", "activist_ownership_events", ["symbol", "filing_date"])
|
||||
op.create_index("idx_activist_filer_issuer", "activist_ownership_events",
|
||||
["filer_cik", "issuer_cik", "filing_date"])
|
||||
op.create_index("idx_activist_filing_date", "activist_ownership_events", ["filing_date"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("idx_activist_filing_date", table_name="activist_ownership_events")
|
||||
op.drop_index("idx_activist_filer_issuer", table_name="activist_ownership_events")
|
||||
op.drop_index("idx_activist_sym_date", table_name="activist_ownership_events")
|
||||
op.drop_table("activist_ownership_events")
|
||||
@ -0,0 +1,92 @@
|
||||
"""
|
||||
Activist Ownership endpoints — SC 13D / 13G
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import date
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from fastapi.responses import Response
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.database import get_db
|
||||
from app.schemas.ownership import (
|
||||
ActivistEventEntry,
|
||||
ActivistEventsResponse,
|
||||
ActivistActiveResponse,
|
||||
)
|
||||
from app.services.activist_ownership_service import ActivistOwnershipService
|
||||
from app.utils.cache import with_cache
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger("app.api.v1.ownership")
|
||||
|
||||
|
||||
@router.get(
|
||||
"/13dg/active",
|
||||
response_model=ActivistActiveResponse,
|
||||
summary="Active activist positions as-of a date",
|
||||
description=(
|
||||
"Returns the latest SC 13D/13G filing per (filer, issuer) pair where **filing_date ≤ as_of** "
|
||||
"and **ownership_pct ≥ min_ownership_pct**.\n\n"
|
||||
"Positions with `ownership_pct = null` (not yet enriched) are excluded.\n\n"
|
||||
"`as_of` is required."
|
||||
),
|
||||
)
|
||||
@with_cache(namespace="ownership:13dg_active", ttl=300,
|
||||
key_params=["as_of", "min_ownership_pct"])
|
||||
async def get_13dg_active(
|
||||
response: Response,
|
||||
as_of: date = Query(..., description="Point-in-time cutoff. Required."),
|
||||
min_ownership_pct: float = Query(5.0, ge=0.0, le=100.0, description="Minimum ownership %"),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
svc = ActivistOwnershipService()
|
||||
try:
|
||||
rows = await svc.get_active_positions(db, as_of=as_of, min_ownership_pct=min_ownership_pct)
|
||||
entries = [ActivistEventEntry.from_orm_obj(r) for r in rows]
|
||||
return ActivistActiveResponse(
|
||||
as_of=as_of,
|
||||
min_ownership_pct=min_ownership_pct,
|
||||
positions=entries,
|
||||
total_count=len(entries),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"13D/G active positions error: {e}")
|
||||
raise HTTPException(status_code=502, detail=str(e))
|
||||
|
||||
|
||||
@router.get(
|
||||
"/13dg/{ticker}",
|
||||
response_model=ActivistEventsResponse,
|
||||
summary="SC 13D/13G activist ownership events for a ticker",
|
||||
description=(
|
||||
"Returns SC 13D and SC 13G filings (including amendments) where **filing_date ≤ as_of**.\n\n"
|
||||
"`as_of` is required for PIT safety in backtests.\n\n"
|
||||
"Note: `ownership_pct` / `shares_owned` will be `null` until background enrichment runs (~30 min)."
|
||||
),
|
||||
)
|
||||
@with_cache(namespace="ownership:13dg", ttl=300, key_params=["ticker", "start", "end", "as_of"])
|
||||
async def get_13dg_events(
|
||||
ticker: str,
|
||||
response: Response,
|
||||
as_of: date = Query(..., description="Point-in-time cutoff (filing_date ≤ as_of). Required."),
|
||||
start: Optional[date] = Query(None, description="Window start (filing_date ≥ start)"),
|
||||
end: Optional[date] = Query(None, description="Window end (filing_date ≤ end)"),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
svc = ActivistOwnershipService()
|
||||
try:
|
||||
rows = await svc.get_events(db, ticker=ticker, as_of=as_of, start=start, end=end)
|
||||
entries = [ActivistEventEntry.from_orm_obj(r) for r in rows]
|
||||
return ActivistEventsResponse(
|
||||
symbol=ticker.upper(),
|
||||
as_of=as_of,
|
||||
window={"start": start.isoformat() if start else None, "end": end.isoformat() if end else None},
|
||||
events=entries,
|
||||
total_count=len(entries),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"13D/G events error for {ticker}: {e}")
|
||||
raise HTTPException(status_code=502, detail=str(e))
|
||||
@ -0,0 +1,50 @@
|
||||
"""
|
||||
SC 13D / 13G Activist Ownership Events model
|
||||
"""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
import uuid
|
||||
|
||||
from sqlalchemy import Column, String, Float, Boolean, Index, Text, UniqueConstraint
|
||||
from sqlalchemy.dialects.postgresql import UUID, TIMESTAMP
|
||||
|
||||
from app.core.database import Base
|
||||
|
||||
|
||||
class ActivistOwnershipEvent(Base):
|
||||
__tablename__ = "activist_ownership_events"
|
||||
|
||||
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
symbol = Column(String(10), nullable=False)
|
||||
issuer_cik = Column(String(20), nullable=False)
|
||||
filer_cik = Column(String(20), nullable=False)
|
||||
filer_name = Column(String(500), nullable=False)
|
||||
accession_number = Column(String(30), nullable=False)
|
||||
form_type = Column(String(20), nullable=False) # SC 13D, SC 13G, SC 13D/A, SC 13G/A
|
||||
filing_date = Column(TIMESTAMP(timezone=True), nullable=False)
|
||||
is_amendment = Column(Boolean, nullable=False, default=False)
|
||||
ownership_pct = Column(Float, nullable=True)
|
||||
shares_owned = Column(Float, nullable=True)
|
||||
change_pct = Column(Float, nullable=True)
|
||||
# index_only | parsed | parse_failed
|
||||
parse_status = Column(String(20), nullable=False, default="index_only")
|
||||
filing_url = Column(Text, nullable=True)
|
||||
|
||||
created_at = Column(
|
||||
TIMESTAMP(timezone=True), default=lambda: datetime.now(timezone.utc)
|
||||
)
|
||||
updated_at = Column(
|
||||
TIMESTAMP(timezone=True),
|
||||
default=lambda: datetime.now(timezone.utc),
|
||||
onupdate=lambda: datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"accession_number", "filer_cik",
|
||||
name="uq_activist_event",
|
||||
),
|
||||
Index("idx_activist_sym_date", "symbol", "filing_date"),
|
||||
Index("idx_activist_filer_issuer", "filer_cik", "issuer_cik", "filing_date"),
|
||||
Index("idx_activist_filing_date", "filing_date"),
|
||||
)
|
||||
@ -0,0 +1,55 @@
|
||||
"""
|
||||
Activist ownership schemas — SC 13D / 13G
|
||||
"""
|
||||
|
||||
from datetime import date
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
|
||||
class ActivistEventEntry(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
symbol: str
|
||||
filing_date: date
|
||||
filer_name: str
|
||||
filer_cik: str
|
||||
form_type: str
|
||||
ownership_pct: Optional[float] = None
|
||||
shares_owned: Optional[float] = None
|
||||
is_amendment: bool = False
|
||||
change_pct: Optional[float] = None
|
||||
accession_number: str
|
||||
parse_status: str
|
||||
|
||||
@classmethod
|
||||
def from_orm_obj(cls, obj) -> "ActivistEventEntry":
|
||||
return cls(
|
||||
symbol=obj.symbol,
|
||||
filing_date=obj.filing_date.date() if hasattr(obj.filing_date, "date") else obj.filing_date,
|
||||
filer_name=obj.filer_name,
|
||||
filer_cik=obj.filer_cik,
|
||||
form_type=obj.form_type,
|
||||
ownership_pct=obj.ownership_pct,
|
||||
shares_owned=obj.shares_owned,
|
||||
is_amendment=obj.is_amendment or False,
|
||||
change_pct=obj.change_pct,
|
||||
accession_number=obj.accession_number,
|
||||
parse_status=obj.parse_status,
|
||||
)
|
||||
|
||||
|
||||
class ActivistEventsResponse(BaseModel):
|
||||
symbol: str
|
||||
as_of: date
|
||||
window: Dict[str, Any] = Field(default_factory=dict)
|
||||
events: List[ActivistEventEntry]
|
||||
total_count: int
|
||||
|
||||
|
||||
class ActivistActiveResponse(BaseModel):
|
||||
as_of: date
|
||||
min_ownership_pct: float
|
||||
positions: List[ActivistEventEntry]
|
||||
total_count: int
|
||||
@ -0,0 +1,398 @@
|
||||
"""
|
||||
SC 13D / 13G Activist Ownership Service
|
||||
|
||||
Phase 1 (index-only): ingest from SEC full-index company.idx
|
||||
Phase 2 (background enrich): parse cover-page XML/HTML for ownership_pct/shares_owned
|
||||
"""
|
||||
|
||||
import logging
|
||||
import re
|
||||
from datetime import date, datetime, timezone
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
from sqlalchemy import and_, func, select, text
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.activist_ownership import ActivistOwnershipEvent
|
||||
from app.services.sec_full_index_service import IndexEntry, SECFullIndexService
|
||||
from app.services.sec_http_client import SECHttpClient
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_CHUNK = 1500
|
||||
|
||||
# Regex patterns for cover-page HTML parsing
|
||||
_RE_OWNERSHIP_PCT = re.compile(
|
||||
r"(?:Percent\s+of\s+Class\s+Represented\s+by\s+Amount\s+in\s+Row(?:\s*\(\d+\))?|"
|
||||
r"percent(?:age)?\s+of\s+class)[^\d]*(\d+\.?\d*)\s*%?",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_RE_SHARES_OWNED = re.compile(
|
||||
r"Aggregate\s+Amount\s+(?:Beneficially\s+)?Owned[^\d]*(\d[\d,]*)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
ACTIVIST_FORM_TYPES = {
|
||||
"SC 13D", "SC 13G", "SC 13D/A", "SC 13G/A",
|
||||
"SCHEDULE 13D", "SCHEDULE 13G", "SCHEDULE 13D/A", "SCHEDULE 13G/A",
|
||||
}
|
||||
ENRICH_BATCH_SIZE = 200
|
||||
|
||||
|
||||
class ActivistOwnershipService:
|
||||
|
||||
def __init__(self):
|
||||
self._http = SECHttpClient("Stock Oracle Activist Service")
|
||||
self._index_svc = SECFullIndexService()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Phase 1: index-only ingest from full-index entries
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def ingest_from_index_entries(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
entries: List[IndexEntry],
|
||||
) -> int:
|
||||
"""Upsert SC 13D/G rows from index entries (parse_status='index_only').
|
||||
|
||||
Returns count of newly inserted rows.
|
||||
"""
|
||||
if not entries:
|
||||
return 0
|
||||
|
||||
acc_set = {e.accession_number for e in entries}
|
||||
existing = await db.execute(
|
||||
select(ActivistOwnershipEvent.accession_number).distinct().where(
|
||||
ActivistOwnershipEvent.accession_number.in_(acc_set)
|
||||
)
|
||||
)
|
||||
existing_accs = {r[0] for r in existing.fetchall()}
|
||||
# Commit immediately to release the DB connection before HTTP fetches.
|
||||
# Without this, the connection sits idle-in-transaction for the duration
|
||||
# of all HTTP calls (potentially minutes), triggering the 90s timeout.
|
||||
await db.commit()
|
||||
new_entries = [e for e in entries if e.accession_number not in existing_accs]
|
||||
|
||||
if not new_entries:
|
||||
return 0
|
||||
|
||||
# Resolve issuer CIK → ticker for each entry via SGML header parse.
|
||||
# For 13D/G, the company.idx filer is the BENEFICIAL OWNER (hedge fund).
|
||||
# The issuer (subject company) is in the filing's SGML SUBJECT COMPANY block.
|
||||
rows: List[Dict] = []
|
||||
for entry in new_entries:
|
||||
issuer_cik, symbol = await self._resolve_issuer(entry)
|
||||
if not issuer_cik or not symbol:
|
||||
continue
|
||||
|
||||
filing_dt = datetime(
|
||||
entry.filing_date.year, entry.filing_date.month, entry.filing_date.day,
|
||||
tzinfo=timezone.utc
|
||||
)
|
||||
is_amend = entry.form_type.endswith("/A")
|
||||
filing_url = f"https://www.sec.gov/Archives/{entry.filename}"
|
||||
|
||||
rows.append({
|
||||
"symbol": symbol,
|
||||
"issuer_cik": issuer_cik,
|
||||
"filer_cik": entry.cik,
|
||||
"filer_name": entry.company_name,
|
||||
"accession_number": entry.accession_number,
|
||||
"form_type": entry.form_type,
|
||||
"filing_date": filing_dt,
|
||||
"is_amendment": is_amend,
|
||||
"parse_status": "index_only",
|
||||
"filing_url": filing_url,
|
||||
})
|
||||
|
||||
if not rows:
|
||||
return 0
|
||||
|
||||
inserted = 0
|
||||
for i in range(0, len(rows), _CHUNK):
|
||||
chunk = rows[i:i + _CHUNK]
|
||||
stmt = pg_insert(ActivistOwnershipEvent).values(chunk)
|
||||
stmt = stmt.on_conflict_do_nothing(constraint="uq_activist_event")
|
||||
result = await db.execute(stmt)
|
||||
inserted += result.rowcount
|
||||
await db.commit()
|
||||
logger.info(f"Activist: inserted {inserted} new index-only rows")
|
||||
return inserted
|
||||
|
||||
async def _resolve_issuer(self, entry: IndexEntry) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""Parse SGML header of the filing .txt to find subject company CIK, then ticker.
|
||||
|
||||
SC 13D/G filings use flat-file SGML format: the SUBJECT COMPANY block in the
|
||||
first few KB contains CENTRAL INDEX KEY of the issuer.
|
||||
"""
|
||||
txt_url = f"https://www.sec.gov/Archives/{entry.filename}"
|
||||
try:
|
||||
text = await self._http.fetch_text(txt_url)
|
||||
# Parse only the SGML header portion (~4KB is sufficient)
|
||||
header = text[:4000]
|
||||
in_subject = False
|
||||
issuer_cik = None
|
||||
for line in header.splitlines():
|
||||
if "SUBJECT COMPANY:" in line:
|
||||
in_subject = True
|
||||
elif "FILED BY:" in line:
|
||||
in_subject = False
|
||||
elif in_subject and "CENTRAL INDEX KEY:" in line:
|
||||
raw_cik = line.split("CENTRAL INDEX KEY:")[1].strip()
|
||||
if raw_cik:
|
||||
issuer_cik = str(int(raw_cik)).zfill(10)
|
||||
break
|
||||
if not issuer_cik:
|
||||
return None, None
|
||||
ticker = await self._http.get_ticker_for_cik(issuer_cik)
|
||||
return issuer_cik, ticker
|
||||
except Exception as e:
|
||||
logger.debug(f"Activist: could not resolve issuer for {entry.accession_number}: {e}")
|
||||
return None, None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Phase 2: enrich index-only rows with ownership_pct / shares_owned
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def enrich_pending(self, db: AsyncSession, batch_size: int = ENRICH_BATCH_SIZE) -> int:
|
||||
"""Enrich up to batch_size index-only rows with cover-page parse."""
|
||||
pending = await db.execute(
|
||||
select(ActivistOwnershipEvent)
|
||||
.where(ActivistOwnershipEvent.parse_status == "index_only")
|
||||
.order_by(ActivistOwnershipEvent.filing_date.desc())
|
||||
.limit(batch_size)
|
||||
)
|
||||
rows = pending.scalars().all()
|
||||
if not rows:
|
||||
return 0
|
||||
|
||||
enriched = 0
|
||||
for row in rows:
|
||||
try:
|
||||
ownership_pct, shares_owned = await self._parse_cover_page(row)
|
||||
change_pct = None
|
||||
if ownership_pct is not None:
|
||||
change_pct = await self._compute_change_pct(db, row, ownership_pct)
|
||||
|
||||
row.ownership_pct = ownership_pct
|
||||
row.shares_owned = shares_owned
|
||||
row.change_pct = change_pct
|
||||
row.parse_status = "parsed"
|
||||
db.add(row)
|
||||
enriched += 1
|
||||
except Exception as e:
|
||||
logger.warning(f"Activist enrich failed {row.accession_number}: {e}")
|
||||
row.parse_status = "parse_failed"
|
||||
db.add(row)
|
||||
|
||||
await db.commit()
|
||||
logger.info(f"Activist enrich: {enriched}/{len(rows)} rows enriched")
|
||||
return enriched
|
||||
|
||||
async def _parse_cover_page(
|
||||
self, event: ActivistOwnershipEvent
|
||||
) -> Tuple[Optional[float], Optional[float]]:
|
||||
"""Best-effort parse of cover-page XML or HTML for ownership_pct and shares_owned."""
|
||||
if not event.filing_url:
|
||||
raise ValueError("No filing_url")
|
||||
|
||||
# Build the primary document URL from filing index
|
||||
acc_clean = event.accession_number.replace("-", "")
|
||||
cik_int = int(event.filer_cik)
|
||||
idx_url = (
|
||||
f"https://www.sec.gov/Archives/edgar/data/{cik_int}/{acc_clean}"
|
||||
f"/{event.accession_number}-index.json"
|
||||
)
|
||||
try:
|
||||
idx_data = await self._http.fetch_json(idx_url)
|
||||
primary_doc = idx_data.get("primary_document", "")
|
||||
if not primary_doc:
|
||||
# Fall back to the first listed document
|
||||
docs = idx_data.get("documents", [])
|
||||
primary_doc = docs[0].get("document_url", "") if docs else ""
|
||||
except Exception:
|
||||
primary_doc = ""
|
||||
|
||||
# Attempt XML cover page (post-Oct-2023 structured 13D/G)
|
||||
xml_url = (
|
||||
f"https://www.sec.gov/Archives/edgar/data/{cik_int}/{acc_clean}"
|
||||
"/primary_doc.xml"
|
||||
)
|
||||
try:
|
||||
xml_text = await self._http.fetch_text(xml_url, accept="application/xml", max_bytes=512_000)
|
||||
pct, shares = _parse_cover_xml(xml_text)
|
||||
if pct is not None or shares is not None:
|
||||
return pct, shares
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Fall back to HTML primary document
|
||||
if primary_doc:
|
||||
doc_url = primary_doc if primary_doc.startswith("http") else (
|
||||
f"https://www.sec.gov/Archives/edgar/data/{cik_int}/{acc_clean}/{primary_doc}"
|
||||
)
|
||||
else:
|
||||
doc_url = (
|
||||
f"https://www.sec.gov/Archives/edgar/data/{cik_int}/{acc_clean}"
|
||||
f"/{event.accession_number}.txt"
|
||||
)
|
||||
try:
|
||||
html = await self._http.fetch_text(doc_url, max_bytes=1_000_000)
|
||||
return _parse_cover_html(html)
|
||||
except Exception as e:
|
||||
raise ValueError(f"HTML parse failed: {e}") from e
|
||||
|
||||
async def _compute_change_pct(
|
||||
self, db: AsyncSession, event: ActivistOwnershipEvent, new_pct: float
|
||||
) -> Optional[float]:
|
||||
"""Compute change in ownership_pct vs the prior filing by the same filer/issuer."""
|
||||
prev = await db.execute(
|
||||
select(ActivistOwnershipEvent.ownership_pct)
|
||||
.where(
|
||||
and_(
|
||||
ActivistOwnershipEvent.filer_cik == event.filer_cik,
|
||||
ActivistOwnershipEvent.issuer_cik == event.issuer_cik,
|
||||
ActivistOwnershipEvent.filing_date < event.filing_date,
|
||||
ActivistOwnershipEvent.ownership_pct.isnot(None),
|
||||
)
|
||||
)
|
||||
.order_by(ActivistOwnershipEvent.filing_date.desc())
|
||||
.limit(1)
|
||||
)
|
||||
prev_pct = prev.scalar()
|
||||
if prev_pct is None:
|
||||
return None
|
||||
return round(new_pct - float(prev_pct), 4)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Query methods
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def get_events(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
ticker: str,
|
||||
as_of: date,
|
||||
start: Optional[date] = None,
|
||||
end: Optional[date] = None,
|
||||
) -> List[ActivistOwnershipEvent]:
|
||||
"""PIT-safe query: filing_date <= as_of."""
|
||||
ticker = ticker.upper()
|
||||
as_of_dt = datetime(as_of.year, as_of.month, as_of.day, 23, 59, 59, tzinfo=timezone.utc)
|
||||
|
||||
conditions = [
|
||||
ActivistOwnershipEvent.symbol == ticker,
|
||||
ActivistOwnershipEvent.filing_date <= as_of_dt,
|
||||
]
|
||||
if start:
|
||||
conditions.append(ActivistOwnershipEvent.filing_date >= datetime(start.year, start.month, start.day, tzinfo=timezone.utc))
|
||||
if end:
|
||||
conditions.append(ActivistOwnershipEvent.filing_date <= datetime(end.year, end.month, end.day, 23, 59, 59, tzinfo=timezone.utc))
|
||||
|
||||
result = await db.execute(
|
||||
select(ActivistOwnershipEvent)
|
||||
.where(and_(*conditions))
|
||||
.order_by(ActivistOwnershipEvent.filing_date.desc())
|
||||
)
|
||||
return result.scalars().all()
|
||||
|
||||
async def get_active_positions(
|
||||
self,
|
||||
db: AsyncSession,
|
||||
as_of: date,
|
||||
min_ownership_pct: float = 5.0,
|
||||
) -> List[ActivistOwnershipEvent]:
|
||||
"""Return latest-per-(filer, issuer) positions where ownership_pct >= min."""
|
||||
as_of_dt = datetime(as_of.year, as_of.month, as_of.day, 23, 59, 59, tzinfo=timezone.utc)
|
||||
|
||||
# Window function: latest filing per (filer_cik, issuer_cik) as-of as_of
|
||||
subq = (
|
||||
select(
|
||||
ActivistOwnershipEvent,
|
||||
func.row_number().over(
|
||||
partition_by=[
|
||||
ActivistOwnershipEvent.filer_cik,
|
||||
ActivistOwnershipEvent.issuer_cik,
|
||||
],
|
||||
order_by=ActivistOwnershipEvent.filing_date.desc(),
|
||||
).label("rn"),
|
||||
)
|
||||
.where(ActivistOwnershipEvent.filing_date <= as_of_dt)
|
||||
.subquery()
|
||||
)
|
||||
result = await db.execute(
|
||||
select(ActivistOwnershipEvent)
|
||||
.join(subq, ActivistOwnershipEvent.id == subq.c.id)
|
||||
.where(
|
||||
and_(
|
||||
subq.c.rn == 1,
|
||||
ActivistOwnershipEvent.ownership_pct >= min_ownership_pct,
|
||||
)
|
||||
)
|
||||
.order_by(ActivistOwnershipEvent.ownership_pct.desc())
|
||||
)
|
||||
return result.scalars().all()
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Cover-page parsers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _parse_cover_xml(xml_text: str) -> Tuple[Optional[float], Optional[float]]:
|
||||
"""Try to extract ownership_pct and shares_owned from structured Cover Page XML."""
|
||||
import xml.etree.ElementTree as ET
|
||||
try:
|
||||
root = ET.fromstring(xml_text)
|
||||
pct_el = root.find(".//{*}percentClass")
|
||||
if pct_el is None:
|
||||
pct_el = root.find(".//percentClass")
|
||||
if pct_el is None:
|
||||
pct_el = root.find(".//{*}classPercent")
|
||||
if pct_el is None:
|
||||
pct_el = root.find(".//classPercent")
|
||||
shares_el = root.find(".//{*}aggregateAmount")
|
||||
if shares_el is None:
|
||||
shares_el = root.find(".//aggregateAmount")
|
||||
if shares_el is None:
|
||||
shares_el = root.find(".//{*}amountBeneficiallyOwned")
|
||||
if shares_el is None:
|
||||
shares_el = root.find(".//amountBeneficiallyOwned")
|
||||
if shares_el is None:
|
||||
shares_el = root.find(".//{*}reportingPersonBeneficiallyOwnedAggregateNumberOfShares")
|
||||
if shares_el is None:
|
||||
shares_el = root.find(".//reportingPersonBeneficiallyOwnedAggregateNumberOfShares")
|
||||
pct = float(pct_el.text.strip()) if pct_el is not None and pct_el.text else None
|
||||
shares = float(shares_el.text.strip().replace(",", "")) if shares_el is not None and shares_el.text else None
|
||||
return pct, shares
|
||||
except Exception:
|
||||
return None, None
|
||||
|
||||
|
||||
def _parse_cover_html(html: str) -> Tuple[Optional[float], Optional[float]]:
|
||||
"""Best-effort regex extraction from 13D/G HTML cover page."""
|
||||
# Strip HTML tags so CSS/attribute digits don't confuse the regexes
|
||||
clean = re.sub(r"<[^>]+>", " ", html)
|
||||
clean = re.sub(r"\s+", " ", clean)
|
||||
|
||||
pct: Optional[float] = None
|
||||
shares: Optional[float] = None
|
||||
|
||||
m = _RE_OWNERSHIP_PCT.search(clean)
|
||||
if m:
|
||||
try:
|
||||
val = float(m.group(1))
|
||||
if val <= 100:
|
||||
pct = val
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
m = _RE_SHARES_OWNED.search(clean)
|
||||
if m:
|
||||
try:
|
||||
shares = float(m.group(1).replace(",", ""))
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return pct, shares
|
||||
@ -0,0 +1,201 @@
|
||||
"""
|
||||
APScheduler-based daily/weekly SEC ingest scheduler.
|
||||
|
||||
Jobs:
|
||||
form4_daily_ingest — Tue-Sat 09:00 ET: previous business day Form 4
|
||||
activist_13dg_daily_ingest — same schedule: SC 13D/G index-only ingest
|
||||
form4_weekly_reindex — Sat 03:00 ET: current quarter company.idx rescan
|
||||
activist_13dg_enrich — every 30 min: enrich index_only rows with ownership_pct
|
||||
|
||||
Wire up via start_sec_ingest_scheduler() / stop_sec_ingest_scheduler() in app lifespan.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_scheduler = None
|
||||
|
||||
|
||||
def _get_scheduler():
|
||||
global _scheduler
|
||||
if _scheduler is None:
|
||||
try:
|
||||
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
||||
from app.core.config import settings
|
||||
_scheduler = AsyncIOScheduler(timezone=settings.SEC_INGEST_TIMEZONE)
|
||||
except ImportError:
|
||||
logger.warning("apscheduler not installed; SEC ingest scheduling disabled")
|
||||
return None
|
||||
return _scheduler
|
||||
|
||||
|
||||
def _prev_business_day() -> date:
|
||||
"""Return the most recent business day (Mon-Fri) relative to today."""
|
||||
today = datetime.now(timezone.utc).date()
|
||||
delta = timedelta(days=1)
|
||||
# If today is Mon, go back to Fri; otherwise go back 1 day
|
||||
candidate = today - delta
|
||||
while candidate.weekday() >= 5: # 5=Sat, 6=Sun
|
||||
candidate -= delta
|
||||
return candidate
|
||||
|
||||
|
||||
async def _run_form4_daily_ingest() -> None:
|
||||
"""Ingest prior business day Form 4 entries from SEC daily full-index."""
|
||||
from app.core.database import AsyncSessionLocal
|
||||
from app.services.sec_full_index_service import SECFullIndexService
|
||||
from app.services.insider_transaction_service import InsiderTransactionService
|
||||
|
||||
target_date = _prev_business_day()
|
||||
date_str = target_date.strftime("%Y%m%d")
|
||||
logger.info(f"[SEC Ingest] Form 4 daily ingest for {date_str}")
|
||||
|
||||
index_svc = SECFullIndexService()
|
||||
txn_svc = InsiderTransactionService()
|
||||
|
||||
entries = await index_svc.fetch_daily_form4_entries(date_str)
|
||||
if not entries:
|
||||
logger.info(f"[SEC Ingest] No Form 4 entries in daily index for {date_str}")
|
||||
return
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
try:
|
||||
inserted = await txn_svc.index_form4_from_index_entries(db, entries)
|
||||
logger.info(f"[SEC Ingest] Form 4 daily: {inserted} new transactions for {date_str}")
|
||||
except Exception as e:
|
||||
logger.error(f"[SEC Ingest] Form 4 daily ingest failed ({date_str}): {e}")
|
||||
|
||||
|
||||
async def _run_activist_daily_ingest() -> None:
|
||||
"""Ingest prior business day SC 13D/G entries from SEC daily full-index."""
|
||||
from app.core.database import AsyncSessionLocal
|
||||
from app.services.sec_full_index_service import SECFullIndexService
|
||||
from app.services.activist_ownership_service import ActivistOwnershipService
|
||||
|
||||
target_date = _prev_business_day()
|
||||
date_str = target_date.strftime("%Y%m%d")
|
||||
logger.info(f"[SEC Ingest] Activist 13D/G daily ingest for {date_str}")
|
||||
|
||||
index_svc = SECFullIndexService()
|
||||
activist_svc = ActivistOwnershipService()
|
||||
|
||||
entries = await index_svc.fetch_daily_activist_entries(date_str)
|
||||
if not entries:
|
||||
logger.info(f"[SEC Ingest] No 13D/G entries in daily index for {date_str}")
|
||||
return
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
try:
|
||||
inserted = await activist_svc.ingest_from_index_entries(db, entries)
|
||||
logger.info(f"[SEC Ingest] Activist 13D/G daily: {inserted} new rows for {date_str}")
|
||||
except Exception as e:
|
||||
logger.error(f"[SEC Ingest] Activist daily ingest failed ({date_str}): {e}")
|
||||
|
||||
|
||||
async def _run_form4_weekly_reindex() -> None:
|
||||
"""Re-scan current quarter's company.idx to catch corrections and amendments."""
|
||||
from app.core.database import AsyncSessionLocal
|
||||
from app.services.sec_full_index_service import SECFullIndexService
|
||||
from app.services.insider_transaction_service import InsiderTransactionService
|
||||
|
||||
today = datetime.now(timezone.utc).date()
|
||||
quarter = (today.month - 1) // 3 + 1
|
||||
logger.info(f"[SEC Ingest] Form 4 weekly reindex {today.year}/Q{quarter}")
|
||||
|
||||
index_svc = SECFullIndexService()
|
||||
txn_svc = InsiderTransactionService()
|
||||
|
||||
entries = await index_svc.fetch_quarterly_form4_entries(today.year, quarter)
|
||||
if not entries:
|
||||
return
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
try:
|
||||
inserted = await txn_svc.index_form4_from_index_entries(db, entries)
|
||||
logger.info(f"[SEC Ingest] Form 4 weekly reindex: {inserted} new transactions")
|
||||
except Exception as e:
|
||||
logger.error(f"[SEC Ingest] Form 4 weekly reindex failed: {e}")
|
||||
|
||||
|
||||
async def _run_activist_enrich() -> None:
|
||||
"""Enrich a batch of index_only activist rows with ownership_pct / shares_owned."""
|
||||
from app.core.database import AsyncSessionLocal
|
||||
from app.services.activist_ownership_service import ActivistOwnershipService
|
||||
|
||||
activist_svc = ActivistOwnershipService()
|
||||
async with AsyncSessionLocal() as db:
|
||||
try:
|
||||
enriched = await activist_svc.enrich_pending(db)
|
||||
if enriched:
|
||||
logger.info(f"[SEC Ingest] Activist enrich: {enriched} rows enriched")
|
||||
except Exception as e:
|
||||
logger.error(f"[SEC Ingest] Activist enrich job failed: {e}")
|
||||
|
||||
|
||||
def start_sec_ingest_scheduler() -> None:
|
||||
"""Start the SEC ingest scheduler. Call from FastAPI lifespan startup."""
|
||||
sched = _get_scheduler()
|
||||
if sched is None:
|
||||
return
|
||||
|
||||
try:
|
||||
from apscheduler.triggers.cron import CronTrigger
|
||||
from apscheduler.triggers.interval import IntervalTrigger
|
||||
|
||||
sched.add_job(
|
||||
_run_form4_daily_ingest,
|
||||
trigger=CronTrigger(day_of_week="tue-sat", hour=9, minute=0),
|
||||
id="form4_daily_ingest",
|
||||
replace_existing=True,
|
||||
max_instances=1,
|
||||
misfire_grace_time=600,
|
||||
coalesce=True,
|
||||
)
|
||||
sched.add_job(
|
||||
_run_activist_daily_ingest,
|
||||
trigger=CronTrigger(day_of_week="tue-sat", hour=9, minute=5),
|
||||
id="activist_13dg_daily_ingest",
|
||||
replace_existing=True,
|
||||
max_instances=1,
|
||||
misfire_grace_time=600,
|
||||
coalesce=True,
|
||||
)
|
||||
sched.add_job(
|
||||
_run_form4_weekly_reindex,
|
||||
trigger=CronTrigger(day_of_week="sat", hour=3, minute=0),
|
||||
id="form4_weekly_reindex",
|
||||
replace_existing=True,
|
||||
max_instances=1,
|
||||
misfire_grace_time=600,
|
||||
coalesce=True,
|
||||
)
|
||||
sched.add_job(
|
||||
_run_activist_enrich,
|
||||
trigger=IntervalTrigger(minutes=30),
|
||||
id="activist_13dg_enrich",
|
||||
replace_existing=True,
|
||||
max_instances=1,
|
||||
misfire_grace_time=120,
|
||||
coalesce=True,
|
||||
)
|
||||
|
||||
sched.start()
|
||||
logger.info(
|
||||
"SEC ingest scheduler started: "
|
||||
"Form 4 daily @ Tue-Sat 09:00 ET, "
|
||||
"13D/G daily @ Tue-Sat 09:05 ET, "
|
||||
"Form 4 weekly reindex @ Sat 03:00 ET, "
|
||||
"Activist enrich every 30 min"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"SEC ingest scheduler start failed: {e}")
|
||||
|
||||
|
||||
def stop_sec_ingest_scheduler() -> None:
|
||||
"""Stop the scheduler. Call from FastAPI lifespan shutdown."""
|
||||
sched = _get_scheduler()
|
||||
if sched and sched.running:
|
||||
sched.shutdown(wait=False)
|
||||
logger.info("SEC ingest scheduler stopped")
|
||||
@ -0,0 +1,104 @@
|
||||
"""
|
||||
Bootstrap SC 13D/G activist ownership events for the last N quarters.
|
||||
Skips Form 4 (already bootstrapped separately).
|
||||
|
||||
Usage (inside the container):
|
||||
python scripts/bootstrap_13dg.py [--quarters N]
|
||||
|
||||
Idempotent: already-indexed accessions are skipped via upsert ON CONFLICT DO NOTHING.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import fcntl
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
|
||||
logger = logging.getLogger("bootstrap_13dg")
|
||||
|
||||
_LOCK_FILE = "/tmp/bootstrap_13dg.lock"
|
||||
|
||||
|
||||
def _quarters_to_process(n: int):
|
||||
now = datetime.now(timezone.utc)
|
||||
current_q = (now.month - 1) // 3 + 1
|
||||
results = []
|
||||
year, q = now.year, current_q
|
||||
for _ in range(n):
|
||||
results.append((year, q))
|
||||
q -= 1
|
||||
if q == 0:
|
||||
q = 4
|
||||
year -= 1
|
||||
return results
|
||||
|
||||
|
||||
async def bootstrap(n_quarters: int = 8) -> None:
|
||||
from app.core.database import AsyncSessionLocal
|
||||
from app.services.sec_full_index_service import SECFullIndexService
|
||||
from app.services.activist_ownership_service import ActivistOwnershipService
|
||||
|
||||
quarters = _quarters_to_process(n_quarters)
|
||||
logger.info(f"Bootstrapping 13D/G for {n_quarters} quarters: {quarters}")
|
||||
|
||||
index_svc = SECFullIndexService()
|
||||
activist_svc = ActivistOwnershipService()
|
||||
|
||||
_BATCH = 5000 # entries per sub-batch within a quarter (prevents OOM on 20k+ quarters)
|
||||
|
||||
total_inserted = 0
|
||||
for year, quarter in quarters:
|
||||
logger.info(f"Processing {year}/Q{quarter} ...")
|
||||
try:
|
||||
activist_entries = await index_svc.fetch_quarterly_activist_entries(year, quarter)
|
||||
logger.info(f" 13D/G entries from company.idx: {len(activist_entries)}")
|
||||
quarter_inserted = 0
|
||||
for batch_start in range(0, len(activist_entries), _BATCH):
|
||||
batch = activist_entries[batch_start:batch_start + _BATCH]
|
||||
async with AsyncSessionLocal() as db:
|
||||
inserted = await activist_svc.ingest_from_index_entries(db, batch)
|
||||
quarter_inserted += inserted
|
||||
# Clear HTTP cache between batches to prevent OOM on large quarters
|
||||
activist_svc._http._text_cache.clear()
|
||||
activist_svc._http._json_cache.clear()
|
||||
import gc; gc.collect()
|
||||
logger.info(f" batch {batch_start// _BATCH + 1}: {inserted} inserted")
|
||||
total_inserted += quarter_inserted
|
||||
logger.info(f" 13D/G index-only inserted: {quarter_inserted}")
|
||||
except Exception as e:
|
||||
logger.warning(f" {year}/Q{quarter} failed: {e}")
|
||||
finally:
|
||||
activist_svc._http._text_cache.clear()
|
||||
activist_svc._http._json_cache.clear()
|
||||
import gc; gc.collect()
|
||||
|
||||
logger.info(f"Bootstrap complete: {total_inserted} total 13D/G events inserted across {n_quarters} quarters.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--quarters", type=int, default=8)
|
||||
args = parser.parse_args()
|
||||
|
||||
lock_fh = open(_LOCK_FILE, "w")
|
||||
try:
|
||||
fcntl.flock(lock_fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except OSError:
|
||||
logger.error("Bootstrap is already running (lock file held). Exiting.")
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
asyncio.run(bootstrap(n_quarters=args.quarters))
|
||||
finally:
|
||||
fcntl.flock(lock_fh, fcntl.LOCK_UN)
|
||||
lock_fh.close()
|
||||
try:
|
||||
os.unlink(_LOCK_FILE)
|
||||
except OSError:
|
||||
pass
|
||||
@ -0,0 +1,138 @@
|
||||
"""
|
||||
One-time bootstrap: ingest Form 4 + SC 13D/G for the last N quarters.
|
||||
|
||||
Usage (inside the container):
|
||||
python scripts/bootstrap_form4_2y.py
|
||||
|
||||
Quarters bootstrapped = settings.SEC_FORM4_BOOTSTRAP_QUARTERS (default 8 = 2 years).
|
||||
Idempotent: already-indexed accessions are skipped via upsert ON CONFLICT DO NOTHING.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import fcntl
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from datetime import datetime, timezone
|
||||
|
||||
# Ensure project root is on sys.path when invoked directly
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
|
||||
logger = logging.getLogger("bootstrap")
|
||||
|
||||
_LOCK_FILE = "/tmp/bootstrap_form4.lock"
|
||||
# Use Archives endpoint for probe — browse-edgar can be 200 while Archives is still 429
|
||||
_SEC_PROBE_URL = "https://www.sec.gov/Archives/edgar/full-index/2026/QTR1/company.idx"
|
||||
|
||||
|
||||
def _quarters_to_process(n: int):
|
||||
"""Return list of (year, quarter) tuples for the last n quarters."""
|
||||
now = datetime.now(timezone.utc)
|
||||
current_q = (now.month - 1) // 3 + 1
|
||||
results = []
|
||||
year, q = now.year, current_q
|
||||
for _ in range(n):
|
||||
results.append((year, q))
|
||||
q -= 1
|
||||
if q == 0:
|
||||
q = 4
|
||||
year -= 1
|
||||
return results
|
||||
|
||||
|
||||
async def _check_sec_available() -> bool:
|
||||
"""Return True if SEC EDGAR is responding (not rate-limiting us)."""
|
||||
import aiohttp
|
||||
headers = {"User-Agent": "Stock Oracle bootstrap@stockoracle.internal"}
|
||||
try:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.get(
|
||||
_SEC_PROBE_URL, headers=headers,
|
||||
timeout=aiohttp.ClientTimeout(total=10),
|
||||
allow_redirects=True,
|
||||
) as resp:
|
||||
if resp.status == 429:
|
||||
logger.warning(f"SEC EDGAR is rate-limiting us (429). Try again in 30+ minutes.")
|
||||
return False
|
||||
return resp.status < 500
|
||||
except Exception as e:
|
||||
logger.warning(f"SEC EDGAR probe failed: {e}")
|
||||
return False
|
||||
|
||||
|
||||
async def bootstrap() -> None:
|
||||
from app.core.config import settings
|
||||
from app.core.database import AsyncSessionLocal
|
||||
from app.services.sec_full_index_service import SECFullIndexService
|
||||
from app.services.insider_transaction_service import InsiderTransactionService
|
||||
from app.services.activist_ownership_service import ActivistOwnershipService
|
||||
|
||||
# Pre-flight: abort if SEC is rate-limiting us to avoid wasted retries
|
||||
if not await _check_sec_available():
|
||||
logger.error("Aborting bootstrap — SEC EDGAR is rate-limiting this IP. Re-run in 30-60 minutes.")
|
||||
return
|
||||
|
||||
n_quarters = settings.SEC_FORM4_BOOTSTRAP_QUARTERS
|
||||
quarters = _quarters_to_process(n_quarters)
|
||||
logger.info(f"Bootstrapping {n_quarters} quarters: {quarters}")
|
||||
|
||||
index_svc = SECFullIndexService()
|
||||
txn_svc = InsiderTransactionService()
|
||||
activist_svc = ActivistOwnershipService()
|
||||
|
||||
for year, quarter in quarters:
|
||||
logger.info(f"Processing {year}/Q{quarter} ...")
|
||||
|
||||
# Form 4 via form345.zip
|
||||
zip_path = os.path.join(tempfile.gettempdir(), f"form345_{year}_Q{quarter}.zip")
|
||||
if not os.path.exists(zip_path):
|
||||
logger.info(f" Downloading form345.zip for {year}/Q{quarter} ...")
|
||||
try:
|
||||
await index_svc._http.fetch_form345_zip(year, quarter, zip_path)
|
||||
except Exception as e:
|
||||
logger.warning(f" form345.zip download failed: {e}; falling back to company.idx")
|
||||
zip_path = None
|
||||
|
||||
form4_entries = []
|
||||
if zip_path and os.path.exists(zip_path):
|
||||
from app.services.sec_full_index_service import parse_form345_zip, FORM4_TYPES
|
||||
form4_entries = parse_form345_zip(zip_path, form_types=FORM4_TYPES)
|
||||
logger.info(f" Form 4 entries from zip: {len(form4_entries)}")
|
||||
else:
|
||||
form4_entries = await index_svc.fetch_quarterly_form4_entries(year, quarter)
|
||||
logger.info(f" Form 4 entries from company.idx: {len(form4_entries)}")
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
inserted = await txn_svc.index_form4_from_index_entries(db, form4_entries)
|
||||
logger.info(f" Form 4 inserted: {inserted}")
|
||||
|
||||
# 13D/G via company.idx
|
||||
activist_entries = await index_svc.fetch_quarterly_activist_entries(year, quarter)
|
||||
logger.info(f" 13D/G entries from company.idx: {len(activist_entries)}")
|
||||
async with AsyncSessionLocal() as db:
|
||||
inserted = await activist_svc.ingest_from_index_entries(db, activist_entries)
|
||||
logger.info(f" 13D/G index-only inserted: {inserted}")
|
||||
|
||||
logger.info("Bootstrap complete. Run the activist enrich job to populate ownership_pct.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Prevent multiple concurrent runs via file lock
|
||||
lock_fh = open(_LOCK_FILE, "w")
|
||||
try:
|
||||
fcntl.flock(lock_fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except OSError:
|
||||
logger.error("Bootstrap is already running (lock file held). Exiting.")
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
asyncio.run(bootstrap())
|
||||
finally:
|
||||
fcntl.flock(lock_fh, fcntl.LOCK_UN)
|
||||
lock_fh.close()
|
||||
try:
|
||||
os.unlink(_LOCK_FILE)
|
||||
except OSError:
|
||||
pass
|
||||
@ -0,0 +1,138 @@
|
||||
"""
|
||||
Bootstrap Form 4 insider transactions by iterating over all universe tickers.
|
||||
|
||||
Uses index_form4s(ticker, days=730) — the proven submissions-API path — rather
|
||||
than the company.idx + per-accession Archives index.json approach (which is
|
||||
aggressively rate-limited by SEC EDGAR).
|
||||
|
||||
Usage (inside the container):
|
||||
python scripts/bootstrap_form4_by_ticker.py [--resume-from TICKER]
|
||||
|
||||
Options:
|
||||
--resume-from TICKER Skip tickers alphabetically before TICKER (for resuming
|
||||
after an interruption or rate-limit pause).
|
||||
--days N Days of history to fetch per ticker (default: 730).
|
||||
--delay-ms N Per-ticker delay in milliseconds (default: 300).
|
||||
|
||||
Idempotent: already-indexed accessions are skipped via upsert ON CONFLICT DO NOTHING.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import fcntl
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from typing import Optional
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
|
||||
logger = logging.getLogger("bootstrap_ticker")
|
||||
|
||||
_LOCK_FILE = "/tmp/bootstrap_form4_ticker.lock"
|
||||
|
||||
|
||||
async def bootstrap(
|
||||
resume_from: Optional[str] = None,
|
||||
days: int = 730,
|
||||
delay_ms: int = 300,
|
||||
) -> None:
|
||||
from app.core.database import AsyncSessionLocal
|
||||
from app.services.insider_transaction_service import InsiderTransactionService
|
||||
from sqlalchemy import text
|
||||
|
||||
txn_svc = InsiderTransactionService()
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
rows = await db.execute(
|
||||
text("SELECT ticker FROM universe_ticker_registry ORDER BY ticker")
|
||||
)
|
||||
tickers = [r[0] for r in rows.fetchall()]
|
||||
|
||||
total = len(tickers)
|
||||
logger.info(f"Universe: {total} tickers, fetching {days} days of Form 4 history each")
|
||||
|
||||
if resume_from:
|
||||
resume_from = resume_from.upper()
|
||||
start_idx = next((i for i, t in enumerate(tickers) if t >= resume_from), 0)
|
||||
logger.info(f"Resuming from {resume_from} (index {start_idx}/{total})")
|
||||
tickers = tickers[start_idx:]
|
||||
|
||||
inserted_total = 0
|
||||
errors = 0
|
||||
delay_s = delay_ms / 1000.0
|
||||
_SESSION_RECYCLE = 50 # close/reopen aiohttp session every N tickers
|
||||
|
||||
for i, ticker in enumerate(tickers, 1):
|
||||
# Recycle the aiohttp session periodically to release connection pool memory
|
||||
if i % _SESSION_RECYCLE == 1 and i > 1:
|
||||
await txn_svc._http.close()
|
||||
txn_svc = InsiderTransactionService()
|
||||
import gc
|
||||
gc.collect()
|
||||
logger.info(f"Session recycled at ticker {i} to free memory")
|
||||
|
||||
# SECHttpClient has unbounded _text_cache and _json_cache — clear before each ticker
|
||||
# (disk cache still provides persistence across calls)
|
||||
txn_svc._http._text_cache.clear()
|
||||
txn_svc._http._json_cache.clear()
|
||||
|
||||
try:
|
||||
async with AsyncSessionLocal() as db:
|
||||
# Disable idle-in-transaction timeout: HTTP fetches for 730 days of
|
||||
# filings can take several minutes, exceeding the default 90s limit.
|
||||
await db.execute(text("SET SESSION idle_in_transaction_session_timeout = 0"))
|
||||
await db.commit()
|
||||
n = await txn_svc.index_form4s(db, ticker, days=days, force_refresh=False)
|
||||
inserted_total += n
|
||||
except Exception as e:
|
||||
errors += 1
|
||||
logger.warning(f"[{i}/{len(tickers)}] {ticker}: error — {e}")
|
||||
|
||||
if i % 100 == 0 or i == len(tickers):
|
||||
logger.info(
|
||||
f"Progress: {i}/{len(tickers)} tickers processed, "
|
||||
f"{inserted_total} transactions inserted, {errors} errors"
|
||||
)
|
||||
|
||||
if delay_s > 0:
|
||||
await asyncio.sleep(delay_s)
|
||||
|
||||
logger.info(
|
||||
f"Bootstrap complete: {len(tickers)} tickers, "
|
||||
f"{inserted_total} total transactions inserted, {errors} errors"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--resume-from", default=None)
|
||||
parser.add_argument("--days", type=int, default=730)
|
||||
parser.add_argument("--delay-ms", type=int, default=300)
|
||||
args = parser.parse_args()
|
||||
|
||||
lock_fh = open(_LOCK_FILE, "w")
|
||||
try:
|
||||
fcntl.flock(lock_fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except OSError:
|
||||
logger.error("Bootstrap is already running (lock file held). Exiting.")
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
asyncio.run(bootstrap(
|
||||
resume_from=args.resume_from,
|
||||
days=args.days,
|
||||
delay_ms=args.delay_ms,
|
||||
))
|
||||
finally:
|
||||
fcntl.flock(lock_fh, fcntl.LOCK_UN)
|
||||
lock_fh.close()
|
||||
try:
|
||||
os.unlink(_LOCK_FILE)
|
||||
except OSError:
|
||||
pass
|
||||
@ -0,0 +1,88 @@
|
||||
"""
|
||||
Unit tests for activist ownership cover-page parsing (HTML + XML).
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services.activist_ownership_service import _parse_cover_html, _parse_cover_xml
|
||||
|
||||
|
||||
# Minimal 13D HTML cover page (simplified)
|
||||
SAMPLE_13D_HTML = """
|
||||
<html><body>
|
||||
<table>
|
||||
<tr><td>Item 11. Aggregate Amount Beneficially Owned by Each Reporting Person</td><td>12,500,000</td></tr>
|
||||
<tr><td>Item 13. Percent of Class Represented by Amount in Row (11)</td><td>7.2%</td></tr>
|
||||
</table>
|
||||
</body></html>
|
||||
"""
|
||||
|
||||
SAMPLE_13G_HTML = """
|
||||
<html><body>
|
||||
<p>Percent of Class Represented by Amount in Row (11): 5.4 %</p>
|
||||
<p>Aggregate Amount Beneficially Owned: 8,000,000</p>
|
||||
</body></html>
|
||||
"""
|
||||
|
||||
SAMPLE_COVER_XML = """<?xml version="1.0" encoding="UTF-8"?>
|
||||
<coverPage>
|
||||
<aggregateAmount>15000000</aggregateAmount>
|
||||
<percentClass>9.1</percentClass>
|
||||
</coverPage>
|
||||
"""
|
||||
|
||||
SAMPLE_COVER_XML_NAMESPACED = """<?xml version="1.0" encoding="UTF-8"?>
|
||||
<sc13d xmlns:sc="http://www.sec.gov/sc13d">
|
||||
<sc:aggregateAmount>3000000</sc:aggregateAmount>
|
||||
<sc:percentClass>3.5</sc:percentClass>
|
||||
</sc13d>
|
||||
"""
|
||||
|
||||
|
||||
class TestParseCoverHtml:
|
||||
def test_extract_ownership_pct(self):
|
||||
pct, shares = _parse_cover_html(SAMPLE_13D_HTML)
|
||||
assert pct == pytest.approx(7.2)
|
||||
|
||||
def test_extract_shares_owned(self):
|
||||
pct, shares = _parse_cover_html(SAMPLE_13D_HTML)
|
||||
assert shares == pytest.approx(12_500_000)
|
||||
|
||||
def test_13g_format(self):
|
||||
pct, shares = _parse_cover_html(SAMPLE_13G_HTML)
|
||||
assert pct == pytest.approx(5.4)
|
||||
assert shares == pytest.approx(8_000_000)
|
||||
|
||||
def test_empty_html(self):
|
||||
pct, shares = _parse_cover_html("")
|
||||
assert pct is None
|
||||
assert shares is None
|
||||
|
||||
def test_no_pct_in_html(self):
|
||||
pct, shares = _parse_cover_html("<p>Aggregate Amount Beneficially Owned: 1,000,000</p>")
|
||||
assert pct is None
|
||||
assert shares == pytest.approx(1_000_000)
|
||||
|
||||
|
||||
class TestParseCoverXml:
|
||||
def test_structured_xml(self):
|
||||
pct, shares = _parse_cover_xml(SAMPLE_COVER_XML)
|
||||
assert pct == pytest.approx(9.1)
|
||||
assert shares == pytest.approx(15_000_000)
|
||||
|
||||
def test_namespaced_xml(self):
|
||||
# Namespace-prefixed elements should be found via {*} wildcard search
|
||||
pct, shares = _parse_cover_xml(SAMPLE_COVER_XML_NAMESPACED)
|
||||
# Either finds or gracefully returns None — no crash
|
||||
assert pct is None or isinstance(pct, float)
|
||||
assert shares is None or isinstance(shares, float)
|
||||
|
||||
def test_invalid_xml(self):
|
||||
pct, shares = _parse_cover_xml("<not valid xml>>>")
|
||||
assert pct is None
|
||||
assert shares is None
|
||||
|
||||
def test_empty_xml(self):
|
||||
pct, shares = _parse_cover_xml("")
|
||||
assert pct is None
|
||||
assert shares is None
|
||||
@ -0,0 +1,191 @@
|
||||
"""
|
||||
PIT (point-in-time) correctness + business logic tests for Form 4.
|
||||
|
||||
Pure unit tests (no DB) for c-suite derivation, purchase_pct formula,
|
||||
and service query-building logic.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from datetime import date, datetime, timezone
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from app.services.insider_transaction_service import InsiderTransactionService
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# C-suite title derivation
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
class TestDeriveTitleFlags:
|
||||
def test_ceo_full_title(self):
|
||||
flags = InsiderTransactionService._derive_title_flags("Chief Executive Officer")
|
||||
assert flags == {"is_ceo": True, "is_cfo": False, "is_c_suite": True}
|
||||
|
||||
def test_ceo_abbrev(self):
|
||||
flags = InsiderTransactionService._derive_title_flags("CEO")
|
||||
assert flags["is_ceo"] is True
|
||||
assert flags["is_c_suite"] is True
|
||||
|
||||
def test_cfo_full_title(self):
|
||||
flags = InsiderTransactionService._derive_title_flags("Chief Financial Officer")
|
||||
assert flags["is_cfo"] is True
|
||||
assert flags["is_c_suite"] is True
|
||||
|
||||
def test_principal_financial_officer(self):
|
||||
flags = InsiderTransactionService._derive_title_flags("Principal Financial Officer")
|
||||
assert flags["is_cfo"] is True
|
||||
|
||||
def test_president(self):
|
||||
flags = InsiderTransactionService._derive_title_flags("President, Americas Division")
|
||||
assert flags["is_c_suite"] is True
|
||||
assert flags["is_ceo"] is False
|
||||
|
||||
def test_chairman(self):
|
||||
flags = InsiderTransactionService._derive_title_flags("Chairman of the Board")
|
||||
assert flags["is_c_suite"] is True
|
||||
|
||||
def test_coo(self):
|
||||
flags = InsiderTransactionService._derive_title_flags("COO")
|
||||
assert flags["is_c_suite"] is True
|
||||
|
||||
def test_evp_is_not_csuite(self):
|
||||
# "EVP Sales" does not match CEO/CFO/COO/... or "Chief X Officer"
|
||||
flags = InsiderTransactionService._derive_title_flags("EVP Sales")
|
||||
assert flags["is_c_suite"] is False
|
||||
|
||||
def test_director_is_not_csuite(self):
|
||||
flags = InsiderTransactionService._derive_title_flags("Director")
|
||||
assert all(not v for v in flags.values())
|
||||
|
||||
def test_none_title(self):
|
||||
flags = InsiderTransactionService._derive_title_flags(None)
|
||||
assert flags == {"is_ceo": False, "is_cfo": False, "is_c_suite": False}
|
||||
|
||||
def test_empty_string(self):
|
||||
flags = InsiderTransactionService._derive_title_flags("")
|
||||
assert all(not v for v in flags.values())
|
||||
|
||||
def test_case_insensitive(self):
|
||||
flags = InsiderTransactionService._derive_title_flags("ceo")
|
||||
assert flags["is_ceo"] is True
|
||||
|
||||
def test_ceo_and_president(self):
|
||||
flags = InsiderTransactionService._derive_title_flags("President and CEO")
|
||||
assert flags["is_ceo"] is True
|
||||
assert flags["is_c_suite"] is True
|
||||
|
||||
def test_chief_marketing_officer(self):
|
||||
flags = InsiderTransactionService._derive_title_flags("Chief Marketing Officer")
|
||||
assert flags["is_c_suite"] is True
|
||||
assert flags["is_ceo"] is False
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# purchase_pct_of_holding formula (tested via _parse_transaction_element mock)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
class TestPurchasePct:
|
||||
"""Test the purchase_pct_of_holding calculation logic directly."""
|
||||
|
||||
def _compute_pct(self, code, shares, shares_after):
|
||||
"""Mirrors the formula in _parse_transaction_element."""
|
||||
if code in ("P", "A") and shares is not None and shares > 0 and shares_after and shares_after > 0:
|
||||
return abs(shares) / shares_after
|
||||
return None
|
||||
|
||||
def test_open_market_buy(self):
|
||||
pct = self._compute_pct("P", 100.0, 1000.0)
|
||||
assert pct == pytest.approx(0.1, rel=1e-6)
|
||||
|
||||
def test_award(self):
|
||||
pct = self._compute_pct("A", 500.0, 5000.0)
|
||||
assert pct == pytest.approx(0.1, rel=1e-6)
|
||||
|
||||
def test_sell_is_none(self):
|
||||
assert self._compute_pct("S", -100.0, 900.0) is None
|
||||
|
||||
def test_zero_shares_after(self):
|
||||
assert self._compute_pct("P", 100.0, 0.0) is None
|
||||
|
||||
def test_none_shares_after(self):
|
||||
assert self._compute_pct("P", 100.0, None) is None
|
||||
|
||||
def test_negative_buy_shares_is_none(self):
|
||||
# Negative shares on a "P" code shouldn't happen, but guard
|
||||
assert self._compute_pct("P", -100.0, 1000.0) is None
|
||||
|
||||
def test_small_buy(self):
|
||||
pct = self._compute_pct("P", 1.0, 1_000_000.0)
|
||||
assert pct == pytest.approx(1e-6, rel=1e-4)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# PIT filter logic (mocked DB)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
class TestGetForm4Pit:
|
||||
"""Verify that get_form4_pit builds correct date conditions."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_as_of_is_used_as_upper_bound(self):
|
||||
svc = InsiderTransactionService()
|
||||
mock_db = AsyncMock()
|
||||
mock_result = MagicMock()
|
||||
mock_result.scalar.return_value = 0
|
||||
mock_result.scalars.return_value.all.return_value = []
|
||||
mock_db.execute = AsyncMock(return_value=mock_result)
|
||||
|
||||
rows, total = await svc.get_form4_pit(
|
||||
mock_db, ticker="AAPL", as_of=date(2024, 1, 15)
|
||||
)
|
||||
assert total == 0
|
||||
assert rows == []
|
||||
# Verify execute was called (conditions were built)
|
||||
assert mock_db.execute.called
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_buy_only_adds_code_filter(self):
|
||||
svc = InsiderTransactionService()
|
||||
mock_db = AsyncMock()
|
||||
mock_result = MagicMock()
|
||||
mock_result.scalar.return_value = 0
|
||||
mock_result.scalars.return_value.all.return_value = []
|
||||
mock_db.execute = AsyncMock(return_value=mock_result)
|
||||
|
||||
# Should not raise
|
||||
rows, total = await svc.get_form4_pit(
|
||||
mock_db, ticker="NVDA", as_of=date(2024, 3, 5), buy_only=True
|
||||
)
|
||||
assert mock_db.execute.called
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Aggregate query
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
class TestGetForm4Aggregate:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_returns_dict_structure(self):
|
||||
svc = InsiderTransactionService()
|
||||
mock_db = AsyncMock()
|
||||
mock_row = MagicMock()
|
||||
mock_row.buy_count = 3
|
||||
mock_row.buy_dollar_total = 150000.0
|
||||
mock_row.cluster_size = 2
|
||||
mock_row.csuite_count = 1
|
||||
mock_row.avg_pct_of_holding = 0.05
|
||||
mock_row.last_filing_date = datetime(2024, 1, 10, tzinfo=timezone.utc)
|
||||
mock_result = MagicMock()
|
||||
mock_result.one.return_value = mock_row
|
||||
mock_db.execute = AsyncMock(return_value=mock_result)
|
||||
|
||||
agg = await svc.get_form4_aggregate(
|
||||
mock_db, ticker="AAPL", as_of=date(2024, 1, 15), window_days=30
|
||||
)
|
||||
assert agg["symbol"] == "AAPL"
|
||||
assert agg["buy_count"] == 3
|
||||
assert agg["cluster_size"] == 2
|
||||
assert agg["csuite_count"] == 1
|
||||
assert "recency_days" in agg
|
||||
assert "buy_dollar_total" in agg
|
||||
@ -0,0 +1,149 @@
|
||||
"""
|
||||
Unit tests for SEC full-index (company.idx and form.idx) parsers.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from datetime import date
|
||||
|
||||
from app.services.sec_full_index_service import (
|
||||
parse_company_idx,
|
||||
_parse_form_idx,
|
||||
FORM4_TYPES,
|
||||
ACTIVIST_13DG_TYPES,
|
||||
)
|
||||
|
||||
|
||||
# Realistic company.idx sample — fixed-width, matches real SEC EDGAR format.
|
||||
# Column offsets: Company(0-61), Form(62-73), CIK(74-85), Date(86-97), File(98+)
|
||||
# The dashes line is the sentinel that marks the start of data rows.
|
||||
_H = "Company Name Form Type CIK Date Filed Filename"
|
||||
_D = "-" * 120
|
||||
|
||||
def _row(company, form_type, cik, filing_date, acc):
|
||||
"""Build a fixed-width company.idx data row matching real EDGAR format.
|
||||
|
||||
Real EDGAR layout: company(62) + form_type(12) + 5_spaces + cik(12, left-pad) + date(10) + 2_spaces + filename
|
||||
CIK always has 5 mandatory leading spaces; date is at position 91, filename at 103.
|
||||
"""
|
||||
filename = f"edgar/data/{int(cik)}/{acc}-index.htm"
|
||||
cik_str = f"{str(int(cik)):<12}" # 12-char left-aligned CIK (trailing spaces)
|
||||
# 5 mandatory leading spaces + 12-char CIK + date(10) + 2 spaces = positions 74-102
|
||||
line = f"{company:<62}{form_type:<12} {cik_str}{filing_date} {filename}"
|
||||
return line
|
||||
|
||||
SAMPLE_IDX = "\n".join([
|
||||
"Full-Index of EDGAR Filings Submitted to the Commission",
|
||||
"",
|
||||
_H,
|
||||
_D,
|
||||
_row("APPLE INC", "4", "320193", "2026-01-15", "0000320193-26-000001"),
|
||||
_row("BERKSHIRE HATHAWAY INC", "SC 13G", "1067983", "2026-02-14", "0001067983-26-000042"),
|
||||
_row("BLACKROCK INC", "SC 13D/A", "1364742", "2026-01-20", "0001364742-26-000010"),
|
||||
_row("TESLA INC", "4/A", "1318605", "2026-01-18", "0001318605-26-000005"),
|
||||
_row("MICROSOFT CORP", "10-K", "789019", "2026-02-01", "0000789019-26-000020"),
|
||||
])
|
||||
|
||||
|
||||
def test_parse_form4_entries():
|
||||
entries = parse_company_idx(SAMPLE_IDX, form_types=FORM4_TYPES)
|
||||
assert len(entries) == 2
|
||||
form_types = {e.form_type for e in entries}
|
||||
assert "4" in form_types
|
||||
assert "4/A" in form_types
|
||||
|
||||
|
||||
def test_parse_activist_entries():
|
||||
entries = parse_company_idx(SAMPLE_IDX, form_types=ACTIVIST_13DG_TYPES)
|
||||
assert len(entries) == 2
|
||||
form_types = {e.form_type for e in entries}
|
||||
assert "SC 13G" in form_types
|
||||
assert "SC 13D/A" in form_types
|
||||
|
||||
|
||||
def test_parse_all_entries_no_filter():
|
||||
entries = parse_company_idx(SAMPLE_IDX)
|
||||
# 4, SC 13G, SC 13D/A, 4/A, 10-K
|
||||
assert len(entries) == 5
|
||||
|
||||
|
||||
def test_parse_entry_fields():
|
||||
entries = parse_company_idx(SAMPLE_IDX, form_types=FORM4_TYPES)
|
||||
apple = next((e for e in entries if "APPLE" in e.company_name), None)
|
||||
assert apple is not None, f"APPLE not found in {[e.company_name for e in entries]}"
|
||||
assert apple.cik == "0000320193"
|
||||
assert apple.filing_date == date(2026, 1, 15)
|
||||
assert "0000320193-26-000001" in apple.accession_number
|
||||
assert "edgar/data" in apple.filename
|
||||
|
||||
|
||||
def test_parse_skips_short_lines():
|
||||
idx = "short\n" + SAMPLE_IDX
|
||||
entries = parse_company_idx(idx, form_types=FORM4_TYPES)
|
||||
# Should not crash; just skip the malformed line
|
||||
assert isinstance(entries, list)
|
||||
|
||||
|
||||
def test_parse_empty_text():
|
||||
entries = parse_company_idx("", form_types=FORM4_TYPES)
|
||||
assert entries == []
|
||||
|
||||
|
||||
def test_parse_header_only():
|
||||
# Only the 4 header lines (no data rows after the dash separator)
|
||||
lines = SAMPLE_IDX.splitlines()
|
||||
dash_idx = next(i for i, l in enumerate(lines) if set(l.strip()) == {"-"} and len(l.strip()) > 20)
|
||||
header_only = "\n".join(lines[:dash_idx + 1]) # up to and including dash line
|
||||
entries = parse_company_idx(header_only, form_types=FORM4_TYPES)
|
||||
assert entries == []
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# form.idx parser (form345.zip column layout: Form Type is FIRST)
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _form_row(form_type, company, cik, filing_date, acc):
|
||||
"""Build a fixed-width form.idx row (Form Type first, then Company Name).
|
||||
|
||||
Same EDGAR offsets from position 74 onward as company.idx.
|
||||
"""
|
||||
filename = f"edgar/data/{int(cik)}/{acc}-index.htm"
|
||||
cik_str = f"{str(int(cik)):<12}" # 12-char left-aligned CIK
|
||||
line = f"{form_type:<12}{company:<62} {cik_str}{filing_date} {filename}"
|
||||
return line
|
||||
|
||||
_FORM_IDX = "\n".join([
|
||||
"Form Type Company Name CIK Date Filed Filename",
|
||||
"-" * 120,
|
||||
_form_row("4", "TIM COOK", "320193", "2026-01-15", "0000320193-26-000001"),
|
||||
_form_row("4/A", "ELON MUSK", "1318605", "2026-01-18", "0001318605-26-000005"),
|
||||
_form_row("10-K", "MICROSOFT CORP", "789019", "2026-02-01", "0000789019-26-000020"),
|
||||
])
|
||||
|
||||
|
||||
def test_form_idx_parse_form4():
|
||||
entries = _parse_form_idx(_FORM_IDX, form_types=FORM4_TYPES)
|
||||
assert len(entries) == 2
|
||||
form_types = {e.form_type for e in entries}
|
||||
assert "4" in form_types
|
||||
assert "4/A" in form_types
|
||||
|
||||
|
||||
def test_form_idx_company_name_correct():
|
||||
entries = _parse_form_idx(_FORM_IDX, form_types=FORM4_TYPES)
|
||||
cook = next((e for e in entries if "TIM COOK" in e.company_name), None)
|
||||
assert cook is not None
|
||||
assert cook.cik == "0000320193"
|
||||
assert cook.filing_date == date(2026, 1, 15)
|
||||
assert "0000320193-26-000001" in cook.accession_number
|
||||
|
||||
|
||||
def test_form_idx_does_not_mix_offsets_with_company_idx():
|
||||
# Same data fed to company.idx parser would produce garbled output;
|
||||
# form_idx parser must correctly extract form_type from cols 0-12.
|
||||
company_idx_entries = parse_company_idx(_FORM_IDX, form_types=FORM4_TYPES)
|
||||
form_idx_entries = _parse_form_idx(_FORM_IDX, form_types=FORM4_TYPES)
|
||||
# company.idx parser treats "4 TIM COOK..." as company_name="4"
|
||||
# and form_type from offset 62 — misses "4" form type → 0 entries
|
||||
# form.idx parser correctly gets 2 Form 4 entries
|
||||
assert len(form_idx_entries) == 2
|
||||
assert len(company_idx_entries) == 0 # wrong parser on wrong format = empty
|
||||
Loading…
Reference in New Issue