|
|
"""
|
|
|
SEC Form 4 — Insider Transaction service
|
|
|
|
|
|
Downloads Form 4 filings from SEC EDGAR, parses the XML, and stores
|
|
|
individual transactions in the insider_transactions table.
|
|
|
"""
|
|
|
|
|
|
import logging
|
|
|
import re
|
|
|
import xml.etree.ElementTree as ET
|
|
|
from datetime import date, datetime, timedelta, timezone
|
|
|
from typing import Dict, List, Optional, Set, Tuple
|
|
|
|
|
|
from sqlalchemy import select, and_, func, desc, distinct, case
|
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
|
|
|
|
|
from app.models.insider_transaction import InsiderTransaction
|
|
|
from app.services.sec_http_client import SECHttpClient
|
|
|
from app.services.sec_full_index_service import IndexEntry
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
# Chunk size for batch insert (asyncpg 32767-param limit)
|
|
|
_CHUNK = 1200 # 26 cols × 1200 = 31200 → safely under asyncpg 32767-param limit
|
|
|
|
|
|
# C-suite title regex patterns (case-insensitive)
|
|
|
_RE_CEO = re.compile(r"\b(CEO|Chief\s+Executive\s+Officer)\b", re.IGNORECASE)
|
|
|
_RE_CFO = re.compile(r"\b(CFO|Chief\s+Financial\s+Officer|Principal\s+Financial\s+Officer)\b", re.IGNORECASE)
|
|
|
_RE_CSUITE = re.compile(
|
|
|
r"\b(CEO|Chief\s+Executive\s+Officer"
|
|
|
r"|CFO|Chief\s+Financial\s+Officer|Principal\s+Financial\s+Officer"
|
|
|
r"|COO|CTO|CIO|CLO|CMO|President|Chair(?:man|person|woman)?"
|
|
|
r"|Chief\s+\w+\s+Officer)\b",
|
|
|
re.IGNORECASE,
|
|
|
)
|
|
|
|
|
|
|
|
|
class InsiderTransactionService:
|
|
|
|
|
|
def __init__(self):
|
|
|
self._http = SECHttpClient("Stock Oracle Insider Service")
|
|
|
|
|
|
# ------------------------------------------------------------------
|
|
|
# C-suite title classification
|
|
|
# ------------------------------------------------------------------
|
|
|
|
|
|
@staticmethod
|
|
|
def _derive_title_flags(title: Optional[str]) -> Dict[str, bool]:
|
|
|
if not title:
|
|
|
return {"is_ceo": False, "is_cfo": False, "is_c_suite": False}
|
|
|
return {
|
|
|
"is_ceo": bool(_RE_CEO.search(title)),
|
|
|
"is_cfo": bool(_RE_CFO.search(title)),
|
|
|
"is_c_suite": bool(_RE_CSUITE.search(title)),
|
|
|
}
|
|
|
|
|
|
# ------------------------------------------------------------------
|
|
|
# Index Form 4s from SEC EDGAR
|
|
|
# ------------------------------------------------------------------
|
|
|
|
|
|
async def index_form4s(
|
|
|
self,
|
|
|
db: AsyncSession,
|
|
|
ticker: str,
|
|
|
days: int = 365,
|
|
|
force_refresh: bool = False,
|
|
|
) -> int:
|
|
|
"""Fetch Form 4 filings from SEC and index transactions into DB."""
|
|
|
ticker = ticker.upper()
|
|
|
self._http.set_deadline(120.0)
|
|
|
try:
|
|
|
cik = await self._http.get_company_cik(ticker)
|
|
|
if not cik:
|
|
|
raise ValueError(f"Could not find CIK for ticker {ticker}")
|
|
|
|
|
|
cik_int = int(cik)
|
|
|
url = f"{self._http.sec_base_data}/submissions/CIK{cik_int:010d}.json"
|
|
|
data = await self._http.fetch_json(url)
|
|
|
|
|
|
cutoff = datetime.now(timezone.utc) - timedelta(days=days)
|
|
|
|
|
|
# Collect Form 4 filing metadata
|
|
|
form4_list: List[Dict] = []
|
|
|
|
|
|
def scan_block(block: dict) -> None:
|
|
|
forms = block.get("form", [])
|
|
|
dates = block.get("filingDate", [])
|
|
|
accessions = block.get("accessionNumber", [])
|
|
|
primary_docs = block.get("primaryDocument", [])
|
|
|
for i, (form, dt_str, acc) in enumerate(zip(forms, dates, accessions)):
|
|
|
if form not in ("4", "4/A"):
|
|
|
continue
|
|
|
try:
|
|
|
filing_date = datetime.strptime(dt_str, "%Y-%m-%d").replace(
|
|
|
tzinfo=timezone.utc
|
|
|
)
|
|
|
except Exception:
|
|
|
continue
|
|
|
if filing_date < cutoff:
|
|
|
continue
|
|
|
pri_doc = primary_docs[i] if i < len(primary_docs) else None
|
|
|
form4_list.append({
|
|
|
"accession_number": acc,
|
|
|
"filing_date": filing_date,
|
|
|
"primary_document": pri_doc,
|
|
|
"cik_int": cik_int,
|
|
|
})
|
|
|
|
|
|
recent = data.get("filings", {}).get("recent", {})
|
|
|
scan_block(recent)
|
|
|
|
|
|
# Older filing pages (up to 10 — covers 10+ years of Form 4s)
|
|
|
for meta in (data.get("filings", {}).get("files", []) or [])[:10]:
|
|
|
name = meta.get("name")
|
|
|
if not name:
|
|
|
continue
|
|
|
try:
|
|
|
older = await self._http.fetch_json(
|
|
|
f"{self._http.sec_base_data}/submissions/{name}"
|
|
|
)
|
|
|
scan_block(older)
|
|
|
except Exception:
|
|
|
continue
|
|
|
|
|
|
if not form4_list:
|
|
|
return 0
|
|
|
|
|
|
# Skip already-indexed accession numbers
|
|
|
if not force_refresh:
|
|
|
existing = await db.execute(
|
|
|
select(InsiderTransaction.accession_number)
|
|
|
.where(InsiderTransaction.ticker == ticker)
|
|
|
.distinct()
|
|
|
)
|
|
|
existing_accs: Set[str] = {r[0] for r in existing.fetchall()}
|
|
|
form4_list = [f for f in form4_list if f["accession_number"] not in existing_accs]
|
|
|
|
|
|
if not form4_list:
|
|
|
return 0
|
|
|
|
|
|
logger.info(f"Insider: {ticker} — {len(form4_list)} Form 4s to process")
|
|
|
|
|
|
# Download and parse each Form 4 XML
|
|
|
all_rows: List[Dict] = []
|
|
|
for meta in form4_list:
|
|
|
acc = meta["accession_number"]
|
|
|
acc_clean = acc.replace("-", "")
|
|
|
pri_doc = meta["primary_document"]
|
|
|
cik_i = meta["cik_int"]
|
|
|
filing_date = meta["filing_date"]
|
|
|
|
|
|
# Build XML URL
|
|
|
# primary_document is often "xslF345X05/wk-form4_*.xml" (XSLT view);
|
|
|
# the actual raw XML lives at the base filename without the xsl prefix.
|
|
|
xml_filename = None
|
|
|
if pri_doc and pri_doc.endswith(".xml"):
|
|
|
# Strip xsl* prefix directory: "xslF345X05/wk-form4_123.xml" → "wk-form4_123.xml"
|
|
|
xml_filename = pri_doc.rsplit("/", 1)[-1]
|
|
|
if not xml_filename:
|
|
|
xml_filename = f"{acc_clean}.xml"
|
|
|
xml_url = f"https://www.sec.gov/Archives/edgar/data/{cik_i}/{acc_clean}/{xml_filename}"
|
|
|
|
|
|
try:
|
|
|
xml_text = await self._http.fetch_text(xml_url, accept="application/xml")
|
|
|
rows = self.parse_form4_xml(xml_text, ticker, cik, acc, filing_date)
|
|
|
all_rows.extend(rows)
|
|
|
except Exception as e:
|
|
|
logger.warning(f"Insider: failed to fetch/parse Form 4 {acc}: {e}")
|
|
|
continue
|
|
|
|
|
|
if not all_rows:
|
|
|
return 0
|
|
|
|
|
|
# Batch upsert
|
|
|
inserted = 0
|
|
|
for i in range(0, len(all_rows), _CHUNK):
|
|
|
chunk = all_rows[i:i + _CHUNK]
|
|
|
stmt = pg_insert(InsiderTransaction).values(chunk)
|
|
|
stmt = stmt.on_conflict_do_nothing(constraint="uq_insider_transaction")
|
|
|
result = await db.execute(stmt)
|
|
|
inserted += result.rowcount
|
|
|
await db.commit()
|
|
|
|
|
|
if inserted:
|
|
|
logger.info(f"Insider: {ticker} — inserted {inserted} transactions")
|
|
|
return inserted
|
|
|
|
|
|
finally:
|
|
|
self._http.clear_deadline()
|
|
|
|
|
|
# ------------------------------------------------------------------
|
|
|
# Bulk ingestion from full-index entries (daily/quarterly)
|
|
|
# ------------------------------------------------------------------
|
|
|
|
|
|
async def _get_form4_xml_url(self, cik_int: int, acc: str, acc_clean: str) -> Optional[str]:
|
|
|
"""Resolve the correct Form 4 XML URL by fetching the filing index JSON.
|
|
|
|
|
|
SEC Form 4 XML files use filer-defined names (e.g., 'form4.xml',
|
|
|
'wk-form4_*.xml', 'tm*_*.xml'). We fetch the filing index JSON to get
|
|
|
the primary document filename, then build the correct XML URL.
|
|
|
"""
|
|
|
idx_url = (
|
|
|
f"https://www.sec.gov/Archives/edgar/data/{cik_int}/{acc_clean}/{acc}-index.json"
|
|
|
)
|
|
|
try:
|
|
|
idx_data = await self._http.fetch_json(idx_url)
|
|
|
primary = idx_data.get("primary_document", "")
|
|
|
if not primary:
|
|
|
docs = idx_data.get("documents", [])
|
|
|
for doc in docs:
|
|
|
url = doc.get("document_url", "")
|
|
|
if url.endswith(".xml") and "xsl" not in url:
|
|
|
primary = url.rsplit("/", 1)[-1]
|
|
|
break
|
|
|
if primary:
|
|
|
filename = primary.rsplit("/", 1)[-1]
|
|
|
return (
|
|
|
f"https://www.sec.gov/Archives/edgar/data/{cik_int}/{acc_clean}/{filename}"
|
|
|
)
|
|
|
except Exception:
|
|
|
pass
|
|
|
return None
|
|
|
|
|
|
async def index_form4_from_index_entries(
|
|
|
self,
|
|
|
db: AsyncSession,
|
|
|
entries: List[IndexEntry],
|
|
|
commit_every: int = 3000,
|
|
|
) -> int:
|
|
|
"""Ingest Form 4 filings from SEC full-index IndexEntry list.
|
|
|
|
|
|
For each new accession: fetch the filing index JSON to get the correct
|
|
|
XML filename, then parse the XML (extracting ticker from issuerTradingSymbol).
|
|
|
Skips already-indexed accession numbers. Returns new transaction count.
|
|
|
Commits to DB every commit_every accumulated rows to show progress and limit memory.
|
|
|
"""
|
|
|
if not entries:
|
|
|
return 0
|
|
|
|
|
|
acc_set = {e.accession_number for e in entries}
|
|
|
existing = await db.execute(
|
|
|
select(InsiderTransaction.accession_number).distinct().where(
|
|
|
InsiderTransaction.accession_number.in_(acc_set)
|
|
|
)
|
|
|
)
|
|
|
existing_accs: Set[str] = {r[0] for r in existing.fetchall()}
|
|
|
new_entries = [e for e in entries if e.accession_number not in existing_accs]
|
|
|
|
|
|
if not new_entries:
|
|
|
return 0
|
|
|
|
|
|
total = len(new_entries)
|
|
|
logger.info(f"SEC full-index: processing {total} new Form 4 accessions")
|
|
|
|
|
|
inserted = 0
|
|
|
pending_rows: List[Dict] = []
|
|
|
|
|
|
async def _flush(rows: List[Dict]) -> int:
|
|
|
n = 0
|
|
|
for i in range(0, len(rows), _CHUNK):
|
|
|
chunk = rows[i:i + _CHUNK]
|
|
|
stmt = pg_insert(InsiderTransaction).values(chunk)
|
|
|
stmt = stmt.on_conflict_do_nothing(constraint="uq_insider_transaction")
|
|
|
result = await db.execute(stmt)
|
|
|
n += result.rowcount
|
|
|
await db.commit()
|
|
|
return n
|
|
|
|
|
|
for idx, entry in enumerate(new_entries, 1):
|
|
|
acc_clean = entry.accession_number.replace("-", "")
|
|
|
# entry.cik is the ISSUER CIK in company.idx for Form 4.
|
|
|
# Form 4 filings are stored under the issuer's CIK directory.
|
|
|
cik_int = int(entry.cik)
|
|
|
filing_date = datetime.combine(entry.filing_date, datetime.min.time()).replace(tzinfo=timezone.utc)
|
|
|
|
|
|
# Resolve XML URL via filing index JSON (avoids guessing the filename)
|
|
|
xml_url = await self._get_form4_xml_url(cik_int, entry.accession_number, acc_clean)
|
|
|
if not xml_url:
|
|
|
logger.debug(f"SEC full-index: no XML URL for {entry.accession_number}")
|
|
|
else:
|
|
|
try:
|
|
|
xml_text = await self._http.fetch_text(xml_url, accept="application/xml")
|
|
|
rows = self.parse_form4_xml(xml_text, None, entry.cik, entry.accession_number, filing_date)
|
|
|
pending_rows.extend(rows)
|
|
|
except Exception as e:
|
|
|
logger.warning(f"SEC full-index: failed {entry.accession_number}: {e}")
|
|
|
|
|
|
if len(pending_rows) >= commit_every:
|
|
|
n = await _flush(pending_rows)
|
|
|
inserted += n
|
|
|
pending_rows = []
|
|
|
logger.info(f"SEC full-index Form 4: progress {idx}/{total} accessions, {inserted} rows inserted so far")
|
|
|
|
|
|
if pending_rows:
|
|
|
inserted += await _flush(pending_rows)
|
|
|
|
|
|
logger.info(f"SEC full-index Form 4: done — inserted {inserted} transactions from {total} accessions")
|
|
|
return inserted
|
|
|
|
|
|
# ------------------------------------------------------------------
|
|
|
# PIT-safe queries (filing_date-based)
|
|
|
# ------------------------------------------------------------------
|
|
|
|
|
|
async def get_form4_pit(
|
|
|
self,
|
|
|
db: AsyncSession,
|
|
|
ticker: str,
|
|
|
as_of: date,
|
|
|
start: Optional[date] = None,
|
|
|
end: Optional[date] = None,
|
|
|
buy_only: bool = False,
|
|
|
csuite_only: bool = False,
|
|
|
limit: int = 500,
|
|
|
) -> Tuple[List[InsiderTransaction], int]:
|
|
|
"""PIT-safe Form 4 query. filing_date <= as_of is the lookahead guard."""
|
|
|
from datetime import date
|
|
|
ticker = ticker.upper()
|
|
|
as_of_dt = datetime(as_of.year, as_of.month, as_of.day, 23, 59, 59, tzinfo=timezone.utc)
|
|
|
|
|
|
conditions = [
|
|
|
InsiderTransaction.ticker == ticker,
|
|
|
InsiderTransaction.filing_date <= as_of_dt,
|
|
|
]
|
|
|
if start:
|
|
|
conditions.append(InsiderTransaction.filing_date >= datetime(start.year, start.month, start.day, tzinfo=timezone.utc))
|
|
|
if end:
|
|
|
conditions.append(InsiderTransaction.filing_date <= datetime(end.year, end.month, end.day, 23, 59, 59, tzinfo=timezone.utc))
|
|
|
if buy_only:
|
|
|
conditions.append(InsiderTransaction.transaction_code == "P")
|
|
|
conditions.append(InsiderTransaction.shares > 0)
|
|
|
if csuite_only:
|
|
|
conditions.append(InsiderTransaction.is_c_suite == True)
|
|
|
|
|
|
count_q = await db.execute(
|
|
|
select(func.count(InsiderTransaction.id)).where(and_(*conditions))
|
|
|
)
|
|
|
total = count_q.scalar() or 0
|
|
|
|
|
|
result = await db.execute(
|
|
|
select(InsiderTransaction)
|
|
|
.where(and_(*conditions))
|
|
|
.order_by(desc(InsiderTransaction.filing_date))
|
|
|
.limit(limit)
|
|
|
)
|
|
|
return result.scalars().all(), total
|
|
|
|
|
|
async def get_form4_by_date(
|
|
|
self,
|
|
|
db: AsyncSession,
|
|
|
filing_date: date,
|
|
|
buy_only: bool = False,
|
|
|
) -> List[InsiderTransaction]:
|
|
|
"""Return all Form 4 transactions with filing_date == the given date (cross-ticker)."""
|
|
|
from sqlalchemy import cast, Date as SADate
|
|
|
conditions = [
|
|
|
func.cast(InsiderTransaction.filing_date, SADate) == filing_date,
|
|
|
]
|
|
|
if buy_only:
|
|
|
conditions.append(InsiderTransaction.transaction_code == "P")
|
|
|
conditions.append(InsiderTransaction.shares > 0)
|
|
|
|
|
|
result = await db.execute(
|
|
|
select(InsiderTransaction)
|
|
|
.where(and_(*conditions))
|
|
|
.order_by(desc(InsiderTransaction.filing_date), InsiderTransaction.ticker)
|
|
|
)
|
|
|
return result.scalars().all()
|
|
|
|
|
|
async def get_form4_aggregate(
|
|
|
self,
|
|
|
db: AsyncSession,
|
|
|
ticker: str,
|
|
|
as_of: date,
|
|
|
window_days: int = 30,
|
|
|
) -> Dict:
|
|
|
"""Aggregate Form 4 buy activity for a ticker within [as_of-window_days, as_of]."""
|
|
|
from datetime import date, timedelta
|
|
|
ticker = ticker.upper()
|
|
|
as_of_dt = datetime(as_of.year, as_of.month, as_of.day, 23, 59, 59, tzinfo=timezone.utc)
|
|
|
window_start = as_of_dt - timedelta(days=window_days)
|
|
|
|
|
|
base = and_(
|
|
|
InsiderTransaction.ticker == ticker,
|
|
|
InsiderTransaction.filing_date > window_start,
|
|
|
InsiderTransaction.filing_date <= as_of_dt,
|
|
|
InsiderTransaction.transaction_code == "P",
|
|
|
InsiderTransaction.shares > 0,
|
|
|
InsiderTransaction.is_derivative == False,
|
|
|
)
|
|
|
|
|
|
agg = await db.execute(
|
|
|
select(
|
|
|
func.count(InsiderTransaction.id).label("buy_count"),
|
|
|
func.sum(InsiderTransaction.total_value).label("buy_dollar_total"),
|
|
|
func.count(distinct(InsiderTransaction.owner_cik)).label("cluster_size"),
|
|
|
func.sum(case((InsiderTransaction.is_c_suite == True, 1), else_=0)).label("csuite_count"),
|
|
|
func.avg(InsiderTransaction.purchase_pct_of_holding).label("avg_pct_of_holding"),
|
|
|
func.max(InsiderTransaction.filing_date).label("last_filing_date"),
|
|
|
).where(base)
|
|
|
)
|
|
|
row = agg.one()
|
|
|
|
|
|
recency_days = window_days
|
|
|
if row.last_filing_date:
|
|
|
delta = as_of_dt - row.last_filing_date.replace(tzinfo=timezone.utc) if row.last_filing_date.tzinfo is None else as_of_dt - row.last_filing_date
|
|
|
recency_days = max(0, delta.days)
|
|
|
|
|
|
return {
|
|
|
"symbol": ticker,
|
|
|
"as_of": as_of.isoformat(),
|
|
|
"window_days": window_days,
|
|
|
"buy_count": int(row.buy_count or 0),
|
|
|
"buy_dollar_total": round(float(row.buy_dollar_total or 0), 2),
|
|
|
"cluster_size": int(row.cluster_size or 0),
|
|
|
"csuite_count": int(row.csuite_count or 0),
|
|
|
"avg_pct_of_holding": round(float(row.avg_pct_of_holding), 4) if row.avg_pct_of_holding else None,
|
|
|
"recency_days": recency_days,
|
|
|
}
|
|
|
|
|
|
# ------------------------------------------------------------------
|
|
|
# Form 4 XML parsing
|
|
|
# ------------------------------------------------------------------
|
|
|
|
|
|
def parse_form4_xml(
|
|
|
self,
|
|
|
xml_text: str,
|
|
|
ticker: Optional[str],
|
|
|
cik: str,
|
|
|
accession_number: str,
|
|
|
filing_date: datetime,
|
|
|
) -> List[Dict]:
|
|
|
"""Parse a Form 4 XML document and return transaction dicts.
|
|
|
|
|
|
ticker may be None when called from full-index ingestion; in that case
|
|
|
issuerTradingSymbol from the XML is used as the ticker.
|
|
|
"""
|
|
|
rows: List[Dict] = []
|
|
|
try:
|
|
|
root = ET.fromstring(xml_text)
|
|
|
except ET.ParseError as e:
|
|
|
logger.warning(f"Insider XML parse error for {accession_number}: {e}")
|
|
|
return []
|
|
|
|
|
|
# Extract issuer ticker from XML if not provided (full-index ingestion path)
|
|
|
if not ticker:
|
|
|
issuer_el = root.find(".//issuer")
|
|
|
if issuer_el is not None:
|
|
|
sym = _xml_text(issuer_el, "issuerTradingSymbol")
|
|
|
ticker = sym.upper() if sym else None
|
|
|
if not ticker:
|
|
|
logger.debug(f"Insider: no ticker for {accession_number}, skipping")
|
|
|
return []
|
|
|
|
|
|
# Extract reporting owners
|
|
|
owners = []
|
|
|
for ro in root.findall(".//reportingOwner"):
|
|
|
owner_id = ro.find("reportingOwnerId")
|
|
|
rel = ro.find("reportingOwnerRelationship")
|
|
|
name = _xml_text(owner_id, "rptOwnerName") if owner_id is not None else None
|
|
|
if not name:
|
|
|
continue
|
|
|
is_officer = _xml_bool(rel, "isOfficer")
|
|
|
is_director = _xml_bool(rel, "isDirector")
|
|
|
is_ten_pct = _xml_bool(rel, "isTenPercentOwner")
|
|
|
officer_title = _xml_text(rel, "officerTitle") if rel is not None else None
|
|
|
# Build human-readable relationship string
|
|
|
parts = []
|
|
|
if is_officer:
|
|
|
parts.append(officer_title or "Officer")
|
|
|
if is_director:
|
|
|
parts.append("Director")
|
|
|
if is_ten_pct:
|
|
|
parts.append("10% Owner")
|
|
|
owner_rel = ", ".join(parts) if parts else None
|
|
|
flags = self._derive_title_flags(officer_title)
|
|
|
owners.append({
|
|
|
"owner_name": name,
|
|
|
"owner_cik": _xml_text(owner_id, "rptOwnerCik"),
|
|
|
"owner_relationship": owner_rel,
|
|
|
"is_officer": is_officer,
|
|
|
"is_director": is_director,
|
|
|
"is_ten_percent_owner": is_ten_pct,
|
|
|
"officer_title": officer_title,
|
|
|
**flags,
|
|
|
})
|
|
|
|
|
|
if not owners:
|
|
|
return []
|
|
|
|
|
|
# Parse non-derivative transactions
|
|
|
for txn in root.findall(".//nonDerivativeTransaction"):
|
|
|
row = self._parse_transaction_element(
|
|
|
txn, ticker, cik, accession_number, filing_date, is_derivative=False
|
|
|
)
|
|
|
if row:
|
|
|
for owner in owners:
|
|
|
rows.append({**row, **owner})
|
|
|
|
|
|
# Parse derivative transactions
|
|
|
for txn in root.findall(".//derivativeTransaction"):
|
|
|
row = self._parse_transaction_element(
|
|
|
txn, ticker, cik, accession_number, filing_date, is_derivative=True
|
|
|
)
|
|
|
if row:
|
|
|
for owner in owners:
|
|
|
rows.append({**row, **owner})
|
|
|
|
|
|
return rows
|
|
|
|
|
|
def _parse_transaction_element(
|
|
|
self,
|
|
|
txn,
|
|
|
ticker: str,
|
|
|
cik: str,
|
|
|
accession_number: str,
|
|
|
filing_date: datetime,
|
|
|
is_derivative: bool,
|
|
|
) -> Optional[Dict]:
|
|
|
"""Parse a single <nonDerivativeTransaction> or <derivativeTransaction>."""
|
|
|
security = _xml_value(txn, "securityTitle")
|
|
|
txn_date_str = _xml_value(txn, "transactionDate")
|
|
|
if not txn_date_str:
|
|
|
return None
|
|
|
|
|
|
try:
|
|
|
txn_date = datetime.strptime(txn_date_str, "%Y-%m-%d").replace(
|
|
|
tzinfo=timezone.utc
|
|
|
)
|
|
|
except ValueError:
|
|
|
return None
|
|
|
|
|
|
coding = txn.find(".//transactionCoding")
|
|
|
code = _xml_text(coding, "transactionCode") if coding is not None else None
|
|
|
if not code:
|
|
|
return None
|
|
|
|
|
|
amounts = txn.find("transactionAmounts")
|
|
|
shares = _xml_float(amounts, "transactionShares")
|
|
|
price = _xml_float(amounts, "transactionPricePerShare")
|
|
|
if shares is None:
|
|
|
return None
|
|
|
|
|
|
# Acquisition (A) vs Disposition (D)
|
|
|
acq_disp = _xml_value(amounts, "transactionAcquiredDisposedCode") if amounts is not None else None
|
|
|
if acq_disp == "D":
|
|
|
shares = -abs(shares)
|
|
|
|
|
|
post = txn.find("postTransactionAmounts")
|
|
|
shares_after = _xml_float(post, "sharesOwnedFollowingTransaction")
|
|
|
|
|
|
total_value = None
|
|
|
if price is not None and shares is not None:
|
|
|
total_value = round(abs(shares) * price, 2)
|
|
|
|
|
|
# purchase_pct_of_holding: only for open-market purchases (P) with known post-holding
|
|
|
purchase_pct = None
|
|
|
if code == "P" and shares is not None and shares > 0 and shares_after and shares_after > 0:
|
|
|
purchase_pct = abs(shares) / shares_after
|
|
|
|
|
|
return {
|
|
|
"ticker": ticker,
|
|
|
"cik": cik,
|
|
|
"accession_number": accession_number,
|
|
|
"filing_date": filing_date,
|
|
|
"security_title": security,
|
|
|
"transaction_date": txn_date,
|
|
|
"transaction_code": code,
|
|
|
"shares": shares,
|
|
|
"price_per_share": price,
|
|
|
"total_value": total_value,
|
|
|
"shares_owned_after": shares_after,
|
|
|
"purchase_pct_of_holding": purchase_pct,
|
|
|
"is_derivative": is_derivative,
|
|
|
}
|
|
|
|
|
|
# ------------------------------------------------------------------
|
|
|
# Query
|
|
|
# ------------------------------------------------------------------
|
|
|
|
|
|
async def get_transactions(
|
|
|
self,
|
|
|
db: AsyncSession,
|
|
|
ticker: str,
|
|
|
days: int = 90,
|
|
|
transaction_type: Optional[str] = None,
|
|
|
insider_title: Optional[str] = None,
|
|
|
limit: int = 50,
|
|
|
) -> Tuple[List[InsiderTransaction], int]:
|
|
|
"""Query insider transactions. Auto-indexes if no data found."""
|
|
|
ticker = ticker.upper()
|
|
|
cutoff = datetime.now(timezone.utc) - timedelta(days=days)
|
|
|
|
|
|
conditions = [
|
|
|
InsiderTransaction.ticker == ticker,
|
|
|
InsiderTransaction.transaction_date >= cutoff,
|
|
|
]
|
|
|
if transaction_type:
|
|
|
conditions.append(InsiderTransaction.transaction_code == transaction_type.upper())
|
|
|
if insider_title:
|
|
|
conditions.append(InsiderTransaction.officer_title.ilike(f"%{insider_title}%"))
|
|
|
|
|
|
# Count
|
|
|
count_q = await db.execute(
|
|
|
select(func.count(InsiderTransaction.id)).where(and_(*conditions))
|
|
|
)
|
|
|
total = count_q.scalar() or 0
|
|
|
|
|
|
# Auto-index if empty
|
|
|
if total == 0:
|
|
|
ingested = await self.index_form4s(db, ticker, days=max(days, 365))
|
|
|
if ingested > 0:
|
|
|
count_q = await db.execute(
|
|
|
select(func.count(InsiderTransaction.id)).where(and_(*conditions))
|
|
|
)
|
|
|
total = count_q.scalar() or 0
|
|
|
|
|
|
# Fetch
|
|
|
result = await db.execute(
|
|
|
select(InsiderTransaction)
|
|
|
.where(and_(*conditions))
|
|
|
.order_by(desc(InsiderTransaction.transaction_date))
|
|
|
.limit(limit)
|
|
|
)
|
|
|
rows = result.scalars().all()
|
|
|
return rows, total
|
|
|
|
|
|
async def get_summary(
|
|
|
self, db: AsyncSession, ticker: str
|
|
|
) -> Dict:
|
|
|
"""Aggregate insider buy/sell for 3, 6, 12 month periods."""
|
|
|
ticker = ticker.upper()
|
|
|
now = datetime.now(timezone.utc)
|
|
|
|
|
|
# Ensure data exists
|
|
|
count_q = await db.execute(
|
|
|
select(func.count(InsiderTransaction.id)).where(
|
|
|
InsiderTransaction.ticker == ticker
|
|
|
)
|
|
|
)
|
|
|
if (count_q.scalar() or 0) == 0:
|
|
|
await self.index_form4s(db, ticker, days=365)
|
|
|
|
|
|
periods = []
|
|
|
for label, months in [("3m", 3), ("6m", 6), ("12m", 12)]:
|
|
|
cutoff = now - timedelta(days=months * 30)
|
|
|
base_filter = and_(
|
|
|
InsiderTransaction.ticker == ticker,
|
|
|
InsiderTransaction.transaction_date >= cutoff,
|
|
|
InsiderTransaction.is_derivative == False,
|
|
|
)
|
|
|
|
|
|
result = await db.execute(
|
|
|
select(
|
|
|
func.sum(case(
|
|
|
(InsiderTransaction.transaction_code == "P", 1), else_=0
|
|
|
)).label("buy_count"),
|
|
|
func.sum(case(
|
|
|
(InsiderTransaction.transaction_code == "S", 1), else_=0
|
|
|
)).label("sell_count"),
|
|
|
func.sum(case(
|
|
|
(InsiderTransaction.transaction_code == "P",
|
|
|
func.abs(InsiderTransaction.shares)), else_=0
|
|
|
)).label("buy_shares"),
|
|
|
func.sum(case(
|
|
|
(InsiderTransaction.transaction_code == "S",
|
|
|
func.abs(InsiderTransaction.shares)), else_=0
|
|
|
)).label("sell_shares"),
|
|
|
func.sum(case(
|
|
|
(InsiderTransaction.transaction_code == "P",
|
|
|
InsiderTransaction.total_value), else_=0
|
|
|
)).label("buy_value"),
|
|
|
func.sum(case(
|
|
|
(InsiderTransaction.transaction_code == "S",
|
|
|
InsiderTransaction.total_value), else_=0
|
|
|
)).label("sell_value"),
|
|
|
func.count(distinct(case(
|
|
|
(InsiderTransaction.transaction_code == "P",
|
|
|
InsiderTransaction.owner_name), else_=None
|
|
|
))).label("unique_buyers"),
|
|
|
func.count(distinct(case(
|
|
|
(InsiderTransaction.transaction_code == "S",
|
|
|
InsiderTransaction.owner_name), else_=None
|
|
|
))).label("unique_sellers"),
|
|
|
).where(base_filter)
|
|
|
)
|
|
|
row = result.one()
|
|
|
buy_shares = float(row.buy_shares or 0)
|
|
|
sell_shares = float(row.sell_shares or 0)
|
|
|
buy_value = float(row.buy_value or 0)
|
|
|
sell_value = float(row.sell_value or 0)
|
|
|
|
|
|
periods.append({
|
|
|
"period_label": label,
|
|
|
"buy_count": int(row.buy_count or 0),
|
|
|
"sell_count": int(row.sell_count or 0),
|
|
|
"buy_shares": buy_shares,
|
|
|
"sell_shares": sell_shares,
|
|
|
"buy_value": round(buy_value, 2),
|
|
|
"sell_value": round(sell_value, 2),
|
|
|
"net_shares": round(buy_shares - sell_shares, 2),
|
|
|
"net_value": round(buy_value - sell_value, 2),
|
|
|
"unique_buyers": int(row.unique_buyers or 0),
|
|
|
"unique_sellers": int(row.unique_sellers or 0),
|
|
|
})
|
|
|
|
|
|
# Notable transactions (top 5 by value, non-derivative P or S)
|
|
|
notable_result = await db.execute(
|
|
|
select(InsiderTransaction)
|
|
|
.where(
|
|
|
and_(
|
|
|
InsiderTransaction.ticker == ticker,
|
|
|
InsiderTransaction.transaction_code.in_(["P", "S"]),
|
|
|
InsiderTransaction.is_derivative == False,
|
|
|
InsiderTransaction.total_value.isnot(None),
|
|
|
)
|
|
|
)
|
|
|
.order_by(desc(InsiderTransaction.total_value))
|
|
|
.limit(5)
|
|
|
)
|
|
|
notable = notable_result.scalars().all()
|
|
|
|
|
|
return {"periods": periods, "notable": notable}
|
|
|
|
|
|
|
|
|
# ------------------------------------------------------------------
|
|
|
# XML helpers
|
|
|
# ------------------------------------------------------------------
|
|
|
|
|
|
def _xml_text(parent, tag: str) -> Optional[str]:
|
|
|
"""Get text content of a direct child element."""
|
|
|
if parent is None:
|
|
|
return None
|
|
|
el = parent.find(tag)
|
|
|
return el.text.strip() if el is not None and el.text else None
|
|
|
|
|
|
|
|
|
def _xml_value(parent, tag: str) -> Optional[str]:
|
|
|
"""Get the <value> sub-element text (Form 4 XML pattern)."""
|
|
|
if parent is None:
|
|
|
return None
|
|
|
el = parent.find(f".//{tag}")
|
|
|
if el is None:
|
|
|
return None
|
|
|
val = el.find("value")
|
|
|
if val is not None and val.text:
|
|
|
return val.text.strip()
|
|
|
return el.text.strip() if el.text else None
|
|
|
|
|
|
|
|
|
def _xml_float(parent, tag: str) -> Optional[float]:
|
|
|
"""Get float from <value> sub-element."""
|
|
|
text = _xml_value(parent, tag)
|
|
|
if text is None:
|
|
|
return None
|
|
|
try:
|
|
|
return float(text)
|
|
|
except ValueError:
|
|
|
return None
|
|
|
|
|
|
|
|
|
def _xml_bool(parent, tag: str) -> bool:
|
|
|
"""Get boolean from element text (Form 4: '1'/'true' = True)."""
|
|
|
text = _xml_text(parent, tag) if parent is not None else None
|
|
|
if text is None:
|
|
|
return False
|
|
|
return text.lower() in ("1", "true", "yes")
|