|
|
"""
|
|
|
SEC Form 4 — Insider Transaction service
|
|
|
|
|
|
Downloads Form 4 filings from SEC EDGAR, parses the XML, and stores
|
|
|
individual transactions in the insider_transactions table.
|
|
|
"""
|
|
|
|
|
|
import logging
|
|
|
import xml.etree.ElementTree as ET
|
|
|
from datetime import datetime, timedelta, timezone
|
|
|
from typing import Dict, List, Optional, Set, Tuple
|
|
|
|
|
|
from sqlalchemy import select, and_, func, desc, distinct, case
|
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
|
|
|
|
|
from app.models.insider_transaction import InsiderTransaction
|
|
|
from app.services.sec_http_client import SECHttpClient
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
# Chunk size for batch insert (asyncpg 32767-param limit)
|
|
|
_CHUNK = 1500 # ~20 cols × 1500 = 30000 params
|
|
|
|
|
|
|
|
|
class InsiderTransactionService:
|
|
|
|
|
|
def __init__(self):
|
|
|
self._http = SECHttpClient("Stock Oracle Insider Service")
|
|
|
|
|
|
# ------------------------------------------------------------------
|
|
|
# Index Form 4s from SEC EDGAR
|
|
|
# ------------------------------------------------------------------
|
|
|
|
|
|
async def index_form4s(
|
|
|
self,
|
|
|
db: AsyncSession,
|
|
|
ticker: str,
|
|
|
days: int = 365,
|
|
|
force_refresh: bool = False,
|
|
|
) -> int:
|
|
|
"""Fetch Form 4 filings from SEC and index transactions into DB."""
|
|
|
ticker = ticker.upper()
|
|
|
self._http.set_deadline(120.0)
|
|
|
try:
|
|
|
cik = await self._http.get_company_cik(ticker)
|
|
|
if not cik:
|
|
|
raise ValueError(f"Could not find CIK for ticker {ticker}")
|
|
|
|
|
|
cik_int = int(cik)
|
|
|
url = f"{self._http.sec_base_data}/submissions/CIK{cik_int:010d}.json"
|
|
|
data = await self._http.fetch_json(url)
|
|
|
|
|
|
cutoff = datetime.now(timezone.utc) - timedelta(days=days)
|
|
|
|
|
|
# Collect Form 4 filing metadata
|
|
|
form4_list: List[Dict] = []
|
|
|
|
|
|
def scan_block(block: dict) -> None:
|
|
|
forms = block.get("form", [])
|
|
|
dates = block.get("filingDate", [])
|
|
|
accessions = block.get("accessionNumber", [])
|
|
|
primary_docs = block.get("primaryDocument", [])
|
|
|
for i, (form, dt_str, acc) in enumerate(zip(forms, dates, accessions)):
|
|
|
if form not in ("4", "4/A"):
|
|
|
continue
|
|
|
try:
|
|
|
filing_date = datetime.strptime(dt_str, "%Y-%m-%d").replace(
|
|
|
tzinfo=timezone.utc
|
|
|
)
|
|
|
except Exception:
|
|
|
continue
|
|
|
if filing_date < cutoff:
|
|
|
continue
|
|
|
pri_doc = primary_docs[i] if i < len(primary_docs) else None
|
|
|
form4_list.append({
|
|
|
"accession_number": acc,
|
|
|
"filing_date": filing_date,
|
|
|
"primary_document": pri_doc,
|
|
|
"cik_int": cik_int,
|
|
|
})
|
|
|
|
|
|
recent = data.get("filings", {}).get("recent", {})
|
|
|
scan_block(recent)
|
|
|
|
|
|
# Older filing pages (up to 10 — covers 10+ years of Form 4s)
|
|
|
for meta in (data.get("filings", {}).get("files", []) or [])[:10]:
|
|
|
name = meta.get("name")
|
|
|
if not name:
|
|
|
continue
|
|
|
try:
|
|
|
older = await self._http.fetch_json(
|
|
|
f"{self._http.sec_base_data}/submissions/{name}"
|
|
|
)
|
|
|
scan_block(older)
|
|
|
except Exception:
|
|
|
continue
|
|
|
|
|
|
if not form4_list:
|
|
|
return 0
|
|
|
|
|
|
# Skip already-indexed accession numbers
|
|
|
if not force_refresh:
|
|
|
existing = await db.execute(
|
|
|
select(InsiderTransaction.accession_number)
|
|
|
.where(InsiderTransaction.ticker == ticker)
|
|
|
.distinct()
|
|
|
)
|
|
|
existing_accs: Set[str] = {r[0] for r in existing.fetchall()}
|
|
|
form4_list = [f for f in form4_list if f["accession_number"] not in existing_accs]
|
|
|
|
|
|
if not form4_list:
|
|
|
return 0
|
|
|
|
|
|
logger.info(f"Insider: {ticker} — {len(form4_list)} Form 4s to process")
|
|
|
|
|
|
# Download and parse each Form 4 XML
|
|
|
all_rows: List[Dict] = []
|
|
|
for meta in form4_list:
|
|
|
acc = meta["accession_number"]
|
|
|
acc_clean = acc.replace("-", "")
|
|
|
pri_doc = meta["primary_document"]
|
|
|
cik_i = meta["cik_int"]
|
|
|
filing_date = meta["filing_date"]
|
|
|
|
|
|
# Build XML URL
|
|
|
# primary_document is often "xslF345X05/wk-form4_*.xml" (XSLT view);
|
|
|
# the actual raw XML lives at the base filename without the xsl prefix.
|
|
|
xml_filename = None
|
|
|
if pri_doc and pri_doc.endswith(".xml"):
|
|
|
# Strip xsl* prefix directory: "xslF345X05/wk-form4_123.xml" → "wk-form4_123.xml"
|
|
|
xml_filename = pri_doc.rsplit("/", 1)[-1]
|
|
|
if not xml_filename:
|
|
|
xml_filename = f"{acc_clean}.xml"
|
|
|
xml_url = f"https://www.sec.gov/Archives/edgar/data/{cik_i}/{acc_clean}/{xml_filename}"
|
|
|
|
|
|
try:
|
|
|
xml_text = await self._http.fetch_text(xml_url, accept="application/xml")
|
|
|
rows = self.parse_form4_xml(xml_text, ticker, cik, acc, filing_date)
|
|
|
all_rows.extend(rows)
|
|
|
except Exception as e:
|
|
|
logger.warning(f"Insider: failed to fetch/parse Form 4 {acc}: {e}")
|
|
|
continue
|
|
|
|
|
|
if not all_rows:
|
|
|
return 0
|
|
|
|
|
|
# Batch upsert
|
|
|
inserted = 0
|
|
|
for i in range(0, len(all_rows), _CHUNK):
|
|
|
chunk = all_rows[i:i + _CHUNK]
|
|
|
stmt = pg_insert(InsiderTransaction).values(chunk)
|
|
|
stmt = stmt.on_conflict_do_nothing(constraint="uq_insider_transaction")
|
|
|
result = await db.execute(stmt)
|
|
|
inserted += result.rowcount
|
|
|
await db.commit()
|
|
|
|
|
|
if inserted:
|
|
|
logger.info(f"Insider: {ticker} — inserted {inserted} transactions")
|
|
|
return inserted
|
|
|
|
|
|
finally:
|
|
|
self._http.clear_deadline()
|
|
|
|
|
|
# ------------------------------------------------------------------
|
|
|
# Form 4 XML parsing
|
|
|
# ------------------------------------------------------------------
|
|
|
|
|
|
def parse_form4_xml(
|
|
|
self,
|
|
|
xml_text: str,
|
|
|
ticker: str,
|
|
|
cik: str,
|
|
|
accession_number: str,
|
|
|
filing_date: datetime,
|
|
|
) -> List[Dict]:
|
|
|
"""Parse a Form 4 XML document and return transaction dicts."""
|
|
|
rows: List[Dict] = []
|
|
|
try:
|
|
|
root = ET.fromstring(xml_text)
|
|
|
except ET.ParseError as e:
|
|
|
logger.warning(f"Insider XML parse error for {accession_number}: {e}")
|
|
|
return []
|
|
|
|
|
|
# Extract reporting owners
|
|
|
owners = []
|
|
|
for ro in root.findall(".//reportingOwner"):
|
|
|
owner_id = ro.find("reportingOwnerId")
|
|
|
rel = ro.find("reportingOwnerRelationship")
|
|
|
name = _xml_text(owner_id, "rptOwnerName") if owner_id is not None else None
|
|
|
if not name:
|
|
|
continue
|
|
|
owners.append({
|
|
|
"owner_name": name,
|
|
|
"owner_cik": _xml_text(owner_id, "rptOwnerCik"),
|
|
|
"is_officer": _xml_bool(rel, "isOfficer"),
|
|
|
"is_director": _xml_bool(rel, "isDirector"),
|
|
|
"is_ten_percent_owner": _xml_bool(rel, "isTenPercentOwner"),
|
|
|
"officer_title": _xml_text(rel, "officerTitle") if rel is not None else None,
|
|
|
})
|
|
|
|
|
|
if not owners:
|
|
|
return []
|
|
|
|
|
|
# Parse non-derivative transactions
|
|
|
for txn in root.findall(".//nonDerivativeTransaction"):
|
|
|
row = self._parse_transaction_element(
|
|
|
txn, ticker, cik, accession_number, filing_date, is_derivative=False
|
|
|
)
|
|
|
if row:
|
|
|
for owner in owners:
|
|
|
rows.append({**row, **owner})
|
|
|
|
|
|
# Parse derivative transactions
|
|
|
for txn in root.findall(".//derivativeTransaction"):
|
|
|
row = self._parse_transaction_element(
|
|
|
txn, ticker, cik, accession_number, filing_date, is_derivative=True
|
|
|
)
|
|
|
if row:
|
|
|
for owner in owners:
|
|
|
rows.append({**row, **owner})
|
|
|
|
|
|
return rows
|
|
|
|
|
|
def _parse_transaction_element(
|
|
|
self,
|
|
|
txn,
|
|
|
ticker: str,
|
|
|
cik: str,
|
|
|
accession_number: str,
|
|
|
filing_date: datetime,
|
|
|
is_derivative: bool,
|
|
|
) -> Optional[Dict]:
|
|
|
"""Parse a single <nonDerivativeTransaction> or <derivativeTransaction>."""
|
|
|
security = _xml_value(txn, "securityTitle")
|
|
|
txn_date_str = _xml_value(txn, "transactionDate")
|
|
|
if not txn_date_str:
|
|
|
return None
|
|
|
|
|
|
try:
|
|
|
txn_date = datetime.strptime(txn_date_str, "%Y-%m-%d").replace(
|
|
|
tzinfo=timezone.utc
|
|
|
)
|
|
|
except ValueError:
|
|
|
return None
|
|
|
|
|
|
coding = txn.find(".//transactionCoding")
|
|
|
code = _xml_text(coding, "transactionCode") if coding is not None else None
|
|
|
if not code:
|
|
|
return None
|
|
|
|
|
|
amounts = txn.find("transactionAmounts")
|
|
|
shares = _xml_float(amounts, "transactionShares")
|
|
|
price = _xml_float(amounts, "transactionPricePerShare")
|
|
|
if shares is None:
|
|
|
return None
|
|
|
|
|
|
# Acquisition (A) vs Disposition (D)
|
|
|
acq_disp = _xml_value(amounts, "transactionAcquiredDisposedCode") if amounts is not None else None
|
|
|
if acq_disp == "D":
|
|
|
shares = -abs(shares)
|
|
|
|
|
|
post = txn.find("postTransactionAmounts")
|
|
|
shares_after = _xml_float(post, "sharesOwnedFollowingTransaction")
|
|
|
|
|
|
total_value = None
|
|
|
if price is not None and shares is not None:
|
|
|
total_value = round(abs(shares) * price, 2)
|
|
|
|
|
|
return {
|
|
|
"ticker": ticker,
|
|
|
"cik": cik,
|
|
|
"accession_number": accession_number,
|
|
|
"filing_date": filing_date,
|
|
|
"security_title": security,
|
|
|
"transaction_date": txn_date,
|
|
|
"transaction_code": code,
|
|
|
"shares": shares,
|
|
|
"price_per_share": price,
|
|
|
"total_value": total_value,
|
|
|
"shares_owned_after": shares_after,
|
|
|
"is_derivative": is_derivative,
|
|
|
}
|
|
|
|
|
|
# ------------------------------------------------------------------
|
|
|
# Query
|
|
|
# ------------------------------------------------------------------
|
|
|
|
|
|
async def get_transactions(
|
|
|
self,
|
|
|
db: AsyncSession,
|
|
|
ticker: str,
|
|
|
days: int = 90,
|
|
|
transaction_type: Optional[str] = None,
|
|
|
insider_title: Optional[str] = None,
|
|
|
limit: int = 50,
|
|
|
) -> Tuple[List[InsiderTransaction], int]:
|
|
|
"""Query insider transactions. Auto-indexes if no data found."""
|
|
|
ticker = ticker.upper()
|
|
|
cutoff = datetime.now(timezone.utc) - timedelta(days=days)
|
|
|
|
|
|
conditions = [
|
|
|
InsiderTransaction.ticker == ticker,
|
|
|
InsiderTransaction.transaction_date >= cutoff,
|
|
|
]
|
|
|
if transaction_type:
|
|
|
conditions.append(InsiderTransaction.transaction_code == transaction_type.upper())
|
|
|
if insider_title:
|
|
|
conditions.append(InsiderTransaction.officer_title.ilike(f"%{insider_title}%"))
|
|
|
|
|
|
# Count
|
|
|
count_q = await db.execute(
|
|
|
select(func.count(InsiderTransaction.id)).where(and_(*conditions))
|
|
|
)
|
|
|
total = count_q.scalar() or 0
|
|
|
|
|
|
# Auto-index if empty
|
|
|
if total == 0:
|
|
|
ingested = await self.index_form4s(db, ticker, days=max(days, 365))
|
|
|
if ingested > 0:
|
|
|
count_q = await db.execute(
|
|
|
select(func.count(InsiderTransaction.id)).where(and_(*conditions))
|
|
|
)
|
|
|
total = count_q.scalar() or 0
|
|
|
|
|
|
# Fetch
|
|
|
result = await db.execute(
|
|
|
select(InsiderTransaction)
|
|
|
.where(and_(*conditions))
|
|
|
.order_by(desc(InsiderTransaction.transaction_date))
|
|
|
.limit(limit)
|
|
|
)
|
|
|
rows = result.scalars().all()
|
|
|
return rows, total
|
|
|
|
|
|
async def get_summary(
|
|
|
self, db: AsyncSession, ticker: str
|
|
|
) -> Dict:
|
|
|
"""Aggregate insider buy/sell for 3, 6, 12 month periods."""
|
|
|
ticker = ticker.upper()
|
|
|
now = datetime.now(timezone.utc)
|
|
|
|
|
|
# Ensure data exists
|
|
|
count_q = await db.execute(
|
|
|
select(func.count(InsiderTransaction.id)).where(
|
|
|
InsiderTransaction.ticker == ticker
|
|
|
)
|
|
|
)
|
|
|
if (count_q.scalar() or 0) == 0:
|
|
|
await self.index_form4s(db, ticker, days=365)
|
|
|
|
|
|
periods = []
|
|
|
for label, months in [("3m", 3), ("6m", 6), ("12m", 12)]:
|
|
|
cutoff = now - timedelta(days=months * 30)
|
|
|
base_filter = and_(
|
|
|
InsiderTransaction.ticker == ticker,
|
|
|
InsiderTransaction.transaction_date >= cutoff,
|
|
|
InsiderTransaction.is_derivative == False,
|
|
|
)
|
|
|
|
|
|
result = await db.execute(
|
|
|
select(
|
|
|
func.sum(case(
|
|
|
(InsiderTransaction.transaction_code == "P", 1), else_=0
|
|
|
)).label("buy_count"),
|
|
|
func.sum(case(
|
|
|
(InsiderTransaction.transaction_code == "S", 1), else_=0
|
|
|
)).label("sell_count"),
|
|
|
func.sum(case(
|
|
|
(InsiderTransaction.transaction_code == "P",
|
|
|
func.abs(InsiderTransaction.shares)), else_=0
|
|
|
)).label("buy_shares"),
|
|
|
func.sum(case(
|
|
|
(InsiderTransaction.transaction_code == "S",
|
|
|
func.abs(InsiderTransaction.shares)), else_=0
|
|
|
)).label("sell_shares"),
|
|
|
func.sum(case(
|
|
|
(InsiderTransaction.transaction_code == "P",
|
|
|
InsiderTransaction.total_value), else_=0
|
|
|
)).label("buy_value"),
|
|
|
func.sum(case(
|
|
|
(InsiderTransaction.transaction_code == "S",
|
|
|
InsiderTransaction.total_value), else_=0
|
|
|
)).label("sell_value"),
|
|
|
func.count(distinct(case(
|
|
|
(InsiderTransaction.transaction_code == "P",
|
|
|
InsiderTransaction.owner_name), else_=None
|
|
|
))).label("unique_buyers"),
|
|
|
func.count(distinct(case(
|
|
|
(InsiderTransaction.transaction_code == "S",
|
|
|
InsiderTransaction.owner_name), else_=None
|
|
|
))).label("unique_sellers"),
|
|
|
).where(base_filter)
|
|
|
)
|
|
|
row = result.one()
|
|
|
buy_shares = float(row.buy_shares or 0)
|
|
|
sell_shares = float(row.sell_shares or 0)
|
|
|
buy_value = float(row.buy_value or 0)
|
|
|
sell_value = float(row.sell_value or 0)
|
|
|
|
|
|
periods.append({
|
|
|
"period_label": label,
|
|
|
"buy_count": int(row.buy_count or 0),
|
|
|
"sell_count": int(row.sell_count or 0),
|
|
|
"buy_shares": buy_shares,
|
|
|
"sell_shares": sell_shares,
|
|
|
"buy_value": round(buy_value, 2),
|
|
|
"sell_value": round(sell_value, 2),
|
|
|
"net_shares": round(buy_shares - sell_shares, 2),
|
|
|
"net_value": round(buy_value - sell_value, 2),
|
|
|
"unique_buyers": int(row.unique_buyers or 0),
|
|
|
"unique_sellers": int(row.unique_sellers or 0),
|
|
|
})
|
|
|
|
|
|
# Notable transactions (top 5 by value, non-derivative P or S)
|
|
|
notable_result = await db.execute(
|
|
|
select(InsiderTransaction)
|
|
|
.where(
|
|
|
and_(
|
|
|
InsiderTransaction.ticker == ticker,
|
|
|
InsiderTransaction.transaction_code.in_(["P", "S"]),
|
|
|
InsiderTransaction.is_derivative == False,
|
|
|
InsiderTransaction.total_value.isnot(None),
|
|
|
)
|
|
|
)
|
|
|
.order_by(desc(InsiderTransaction.total_value))
|
|
|
.limit(5)
|
|
|
)
|
|
|
notable = notable_result.scalars().all()
|
|
|
|
|
|
return {"periods": periods, "notable": notable}
|
|
|
|
|
|
|
|
|
# ------------------------------------------------------------------
|
|
|
# XML helpers
|
|
|
# ------------------------------------------------------------------
|
|
|
|
|
|
def _xml_text(parent, tag: str) -> Optional[str]:
|
|
|
"""Get text content of a direct child element."""
|
|
|
if parent is None:
|
|
|
return None
|
|
|
el = parent.find(tag)
|
|
|
return el.text.strip() if el is not None and el.text else None
|
|
|
|
|
|
|
|
|
def _xml_value(parent, tag: str) -> Optional[str]:
|
|
|
"""Get the <value> sub-element text (Form 4 XML pattern)."""
|
|
|
if parent is None:
|
|
|
return None
|
|
|
el = parent.find(f".//{tag}")
|
|
|
if el is None:
|
|
|
return None
|
|
|
val = el.find("value")
|
|
|
if val is not None and val.text:
|
|
|
return val.text.strip()
|
|
|
return el.text.strip() if el.text else None
|
|
|
|
|
|
|
|
|
def _xml_float(parent, tag: str) -> Optional[float]:
|
|
|
"""Get float from <value> sub-element."""
|
|
|
text = _xml_value(parent, tag)
|
|
|
if text is None:
|
|
|
return None
|
|
|
try:
|
|
|
return float(text)
|
|
|
except ValueError:
|
|
|
return None
|
|
|
|
|
|
|
|
|
def _xml_bool(parent, tag: str) -> bool:
|
|
|
"""Get boolean from element text (Form 4: '1'/'true' = True)."""
|
|
|
text = _xml_text(parent, tag) if parent is not None else None
|
|
|
if text is None:
|
|
|
return False
|
|
|
return text.lower() in ("1", "true", "yes")
|