You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
447 lines
16 KiB
Python
447 lines
16 KiB
Python
"""Research probe for SEC Form 4 insider-buy idle-alpha ideas.
|
|
|
|
This tool downloads the SEC's quarterly insider transaction flat files,
|
|
filters them to the current backtest universe, and tests simple
|
|
cluster-buy signals as standalone equal-weight curves.
|
|
|
|
It is intentionally conservative:
|
|
- only exact Form 4 filings are used (no amendments)
|
|
- only non-derivative P-code acquisitions are used
|
|
- signals enter on the next trading day's open
|
|
- signals exit on a fixed future close
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import csv
|
|
import datetime as dt
|
|
import io
|
|
import json
|
|
import urllib.request
|
|
import zipfile
|
|
from collections import defaultdict
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from apps.backtester.run import _build_merged_snapshot_store, load_manifest, resolve_config
|
|
from libs.backtest.domain import DailyPortfolioState
|
|
from libs.backtest.metrics import (
|
|
compute_max_drawdown_pct,
|
|
compute_sharpe_ratio,
|
|
compute_total_return_pct,
|
|
)
|
|
from libs.common.logging import configure_logging
|
|
|
|
|
|
_DEFAULT_USER_AGENT = "fithia2-form4-probe/1.0 (local research; contact: dev@example.com)"
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class InsiderSignalSpec:
|
|
name: str
|
|
min_owner_count: int
|
|
min_total_value: float
|
|
hold_days: int
|
|
require_officer_or_director: bool = False
|
|
max_positions: int = 10
|
|
max_new_per_day: int = 3
|
|
|
|
|
|
def _parse_date(value: str, *, is_end: bool = False) -> dt.date:
|
|
parts = value.split("-")
|
|
if len(parts) == 1 and len(value) == 4 and value.isdigit():
|
|
year = int(value)
|
|
return dt.date(year, 12, 31) if is_end else dt.date(year, 1, 1)
|
|
if len(parts) == 2 and all(part.isdigit() for part in parts):
|
|
year = int(parts[0])
|
|
month = int(parts[1])
|
|
if is_end:
|
|
next_month = dt.date(year + (month // 12), (month % 12) + 1, 1)
|
|
return next_month - dt.timedelta(days=1)
|
|
return dt.date(year, month, 1)
|
|
return dt.date.fromisoformat(value)
|
|
|
|
|
|
def _quarter_range(start_date: dt.date, end_date: dt.date) -> list[tuple[int, int]]:
|
|
year = start_date.year
|
|
quarter = (start_date.month - 1) // 3 + 1
|
|
end_key = (end_date.year, (end_date.month - 1) // 3 + 1)
|
|
quarters: list[tuple[int, int]] = []
|
|
while (year, quarter) <= end_key:
|
|
quarters.append((year, quarter))
|
|
quarter += 1
|
|
if quarter == 5:
|
|
year += 1
|
|
quarter = 1
|
|
return quarters
|
|
|
|
|
|
def _quarter_zip_path(cache_dir: Path, year: int, quarter: int) -> Path:
|
|
return cache_dir / f"{year}q{quarter}_form345.zip"
|
|
|
|
|
|
def _quarter_zip_url(year: int, quarter: int) -> str:
|
|
return (
|
|
"https://www.sec.gov/files/structureddata/data/"
|
|
f"insider-transactions-data-sets/{year}q{quarter}_form345.zip"
|
|
)
|
|
|
|
|
|
def _ensure_quarter_zip(
|
|
cache_dir: Path,
|
|
year: int,
|
|
quarter: int,
|
|
*,
|
|
user_agent: str,
|
|
) -> Path:
|
|
cache_dir.mkdir(parents=True, exist_ok=True)
|
|
path = _quarter_zip_path(cache_dir, year, quarter)
|
|
if path.exists() and path.stat().st_size > 0:
|
|
return path
|
|
request = urllib.request.Request(
|
|
_quarter_zip_url(year, quarter),
|
|
headers={"User-Agent": user_agent},
|
|
)
|
|
with urllib.request.urlopen(request, timeout=60) as response:
|
|
data = response.read()
|
|
path.write_bytes(data)
|
|
return path
|
|
|
|
|
|
def _parse_sec_date(value: str | None) -> dt.date | None:
|
|
if not value:
|
|
return None
|
|
try:
|
|
return dt.datetime.strptime(value, "%d-%b-%Y").date()
|
|
except ValueError:
|
|
return None
|
|
|
|
|
|
def _load_form4_cluster_events(
|
|
*,
|
|
cache_dir: Path,
|
|
start_date: dt.date,
|
|
end_date: dt.date,
|
|
allowed_symbols: set[str],
|
|
user_agent: str,
|
|
) -> dict[tuple[dt.date, str], dict[str, Any]]:
|
|
events: dict[tuple[dt.date, str], dict[str, Any]] = {}
|
|
for year, quarter in _quarter_range(start_date, end_date):
|
|
path = _ensure_quarter_zip(cache_dir, year, quarter, user_agent=user_agent)
|
|
with zipfile.ZipFile(path) as zf:
|
|
submissions: dict[str, dict[str, Any]] = {}
|
|
with zf.open("SUBMISSION.tsv") as handle:
|
|
reader = csv.DictReader(
|
|
io.TextIOWrapper(handle, encoding="utf-8", newline=""),
|
|
delimiter="\t",
|
|
)
|
|
for row in reader:
|
|
symbol = str(row.get("ISSUERTRADINGSYMBOL") or "").strip().upper()
|
|
if not symbol or symbol not in allowed_symbols:
|
|
continue
|
|
if str(row.get("DOCUMENT_TYPE") or "").strip().upper() != "4":
|
|
continue
|
|
filing_date = _parse_sec_date(row.get("FILING_DATE"))
|
|
if filing_date is None or filing_date < start_date or filing_date > end_date:
|
|
continue
|
|
submissions[str(row["ACCESSION_NUMBER"])] = {
|
|
"symbol": symbol,
|
|
"filing_date": filing_date,
|
|
}
|
|
|
|
if not submissions:
|
|
continue
|
|
|
|
relationships: dict[str, list[tuple[str, str]]] = defaultdict(list)
|
|
with zf.open("REPORTINGOWNER.tsv") as handle:
|
|
reader = csv.DictReader(
|
|
io.TextIOWrapper(handle, encoding="utf-8", newline=""),
|
|
delimiter="\t",
|
|
)
|
|
for row in reader:
|
|
accession = str(row["ACCESSION_NUMBER"])
|
|
if accession not in submissions:
|
|
continue
|
|
owner_cik = str(row.get("RPTOWNERCIK") or "").strip()
|
|
relationship = str(row.get("RPTOWNER_RELATIONSHIP") or "").strip().lower()
|
|
relationships[accession].append((owner_cik, relationship))
|
|
|
|
with zf.open("NONDERIV_TRANS.tsv") as handle:
|
|
reader = csv.DictReader(
|
|
io.TextIOWrapper(handle, encoding="utf-8", newline=""),
|
|
delimiter="\t",
|
|
)
|
|
for row in reader:
|
|
accession = str(row["ACCESSION_NUMBER"])
|
|
submission = submissions.get(accession)
|
|
if submission is None:
|
|
continue
|
|
if str(row.get("TRANS_CODE") or "").strip().upper() != "P":
|
|
continue
|
|
if str(row.get("TRANS_ACQUIRED_DISP_CD") or "").strip().upper() != "A":
|
|
continue
|
|
try:
|
|
shares = float(row.get("TRANS_SHARES") or 0.0)
|
|
price = float(row.get("TRANS_PRICEPERSHARE") or 0.0)
|
|
except (TypeError, ValueError):
|
|
continue
|
|
if shares <= 0 or price <= 0:
|
|
continue
|
|
|
|
key = (submission["filing_date"], submission["symbol"])
|
|
event = events.setdefault(
|
|
key,
|
|
{
|
|
"filing_date": submission["filing_date"],
|
|
"symbol": submission["symbol"],
|
|
"total_value": 0.0,
|
|
"transaction_count": 0,
|
|
"owners": set(),
|
|
"officer_or_director": set(),
|
|
},
|
|
)
|
|
event["total_value"] += shares * price
|
|
event["transaction_count"] += 1
|
|
for owner_cik, relationship in relationships.get(accession, []):
|
|
if not owner_cik:
|
|
continue
|
|
event["owners"].add(owner_cik)
|
|
if "officer" in relationship or "director" in relationship:
|
|
event["officer_or_director"].add(owner_cik)
|
|
return events
|
|
|
|
|
|
def _build_entries_for_spec(
|
|
*,
|
|
store: Any,
|
|
events: dict[tuple[dt.date, str], dict[str, Any]],
|
|
spec: InsiderSignalSpec,
|
|
) -> dict[dt.date, list[dict[str, Any]]]:
|
|
trading_days = store.all_trading_days()
|
|
trading_index = {date: idx for idx, date in enumerate(trading_days)}
|
|
entries: dict[dt.date, list[dict[str, Any]]] = defaultdict(list)
|
|
|
|
for event in events.values():
|
|
owner_count = len(event["owners"])
|
|
if owner_count < spec.min_owner_count:
|
|
continue
|
|
if float(event["total_value"]) < spec.min_total_value:
|
|
continue
|
|
if spec.require_officer_or_director and not event["officer_or_director"]:
|
|
continue
|
|
|
|
filing_date = event["filing_date"]
|
|
entry_date = next((date for date in trading_days if date > filing_date), None)
|
|
if entry_date is None:
|
|
continue
|
|
entry_bar = store.get_bar(event["symbol"], entry_date)
|
|
if not entry_bar or not entry_bar.get("open") or not entry_bar.get("close"):
|
|
continue
|
|
entry_index = trading_index.get(entry_date)
|
|
if entry_index is None or entry_index + spec.hold_days >= len(trading_days):
|
|
continue
|
|
exit_date = trading_days[entry_index + spec.hold_days]
|
|
exit_bar = store.get_bar(event["symbol"], exit_date)
|
|
if not exit_bar or not exit_bar.get("close"):
|
|
continue
|
|
|
|
entries[entry_date].append(
|
|
{
|
|
"symbol": event["symbol"],
|
|
"exit_date": exit_date,
|
|
"score": (owner_count, float(event["total_value"])),
|
|
"owner_count": owner_count,
|
|
"total_value": float(event["total_value"]),
|
|
}
|
|
)
|
|
return entries
|
|
|
|
|
|
def _simulate_equal_weight_curve(
|
|
*,
|
|
store: Any,
|
|
start_date: dt.date,
|
|
end_date: dt.date,
|
|
entries_by_date: dict[dt.date, list[dict[str, Any]]],
|
|
spec: InsiderSignalSpec,
|
|
capital: float,
|
|
) -> tuple[list[DailyPortfolioState], int]:
|
|
trading_days = [
|
|
date for date in store.all_trading_days() if start_date <= date <= end_date
|
|
]
|
|
equity = capital
|
|
peak = capital
|
|
curve: list[DailyPortfolioState] = []
|
|
open_positions: list[dict[str, Any]] = []
|
|
trade_count = 0
|
|
|
|
for date in trading_days:
|
|
if date in entries_by_date and len(open_positions) < spec.max_positions:
|
|
existing_symbols = {position["symbol"] for position in open_positions}
|
|
ranked = sorted(
|
|
entries_by_date[date],
|
|
key=lambda row: (row["score"][0], row["score"][1]),
|
|
reverse=True,
|
|
)
|
|
added = 0
|
|
for row in ranked:
|
|
if row["symbol"] in existing_symbols:
|
|
continue
|
|
bar = store.get_bar(row["symbol"], date)
|
|
if not bar or not bar.get("open"):
|
|
continue
|
|
open_positions.append(
|
|
{
|
|
"symbol": row["symbol"],
|
|
"exit_date": row["exit_date"],
|
|
"prev_price": float(bar["open"]),
|
|
}
|
|
)
|
|
existing_symbols.add(row["symbol"])
|
|
trade_count += 1
|
|
added += 1
|
|
if added >= spec.max_new_per_day or len(open_positions) >= spec.max_positions:
|
|
break
|
|
|
|
if open_positions:
|
|
daily_returns: list[float] = []
|
|
updated_positions: list[dict[str, Any]] = []
|
|
for position in open_positions:
|
|
bar = store.get_bar(position["symbol"], date)
|
|
if not bar or not bar.get("close"):
|
|
continue
|
|
close_price = float(bar["close"])
|
|
prev_price = float(position["prev_price"])
|
|
if prev_price <= 0:
|
|
continue
|
|
daily_returns.append(close_price / prev_price - 1.0)
|
|
updated_positions.append(
|
|
{
|
|
"symbol": position["symbol"],
|
|
"exit_date": position["exit_date"],
|
|
"prev_price": close_price,
|
|
}
|
|
)
|
|
if daily_returns:
|
|
equity *= 1.0 + sum(daily_returns) / len(daily_returns)
|
|
open_positions = [
|
|
position for position in updated_positions if position["exit_date"] > date
|
|
]
|
|
|
|
peak = max(peak, equity)
|
|
curve.append(
|
|
DailyPortfolioState(
|
|
date=date,
|
|
equity=equity,
|
|
sizing_equity=equity,
|
|
cash_available=equity,
|
|
gross_exposure=float(len(open_positions)) * 10.0,
|
|
net_exposure=float(len(open_positions)) * 10.0,
|
|
reserved_risk_budget=0.0,
|
|
unrealized_pnl=0.0,
|
|
realized_pnl=0.0,
|
|
open_positions=[position["symbol"] for position in open_positions],
|
|
daily_new_risk_used=0.0,
|
|
peak_equity=peak,
|
|
current_drawdown_pct=((peak - equity) / peak * 100.0) if peak > 0 else 0.0,
|
|
)
|
|
)
|
|
return curve, trade_count
|
|
|
|
|
|
def _yearly_return_map(curve: list[DailyPortfolioState]) -> dict[str, float]:
|
|
if not curve:
|
|
return {}
|
|
year_start: dict[int, float] = {}
|
|
year_end: dict[int, float] = {}
|
|
for state in curve:
|
|
year_start.setdefault(state.date.year, state.equity)
|
|
year_end[state.date.year] = state.equity
|
|
return {
|
|
str(year): round((year_end[year] / year_start[year] - 1.0) * 100.0, 2)
|
|
for year in sorted(year_start)
|
|
if year_start[year] > 0
|
|
}
|
|
|
|
|
|
def _default_specs() -> list[InsiderSignalSpec]:
|
|
return [
|
|
InsiderSignalSpec("owners2_value500k_hold5", 2, 500_000.0, 5),
|
|
InsiderSignalSpec("owners2_value1m_hold5", 2, 1_000_000.0, 5),
|
|
InsiderSignalSpec("owners2_value500k_hold10", 2, 500_000.0, 10),
|
|
InsiderSignalSpec("owners2_value500k_hold5_offdir", 2, 500_000.0, 5, True),
|
|
InsiderSignalSpec("owners1_value1m_hold5", 1, 1_000_000.0, 5),
|
|
]
|
|
|
|
|
|
def _run_probe(args: argparse.Namespace) -> list[dict[str, Any]]:
|
|
start_date = _parse_date(args.start)
|
|
end_date = _parse_date(args.end, is_end=True)
|
|
rows: list[dict[str, Any]] = []
|
|
|
|
for config_path in args.config:
|
|
manifest = load_manifest(config_path)
|
|
config = resolve_config(manifest)
|
|
store = _build_merged_snapshot_store(
|
|
manifest,
|
|
config,
|
|
snapshot_dir_override=None,
|
|
).slice_by_date_range(start_date, end_date)
|
|
allowed_symbols = {str(symbol).upper() for symbol in store._bars.keys()}
|
|
events = _load_form4_cluster_events(
|
|
cache_dir=Path(args.cache_dir),
|
|
start_date=start_date,
|
|
end_date=end_date,
|
|
allowed_symbols=allowed_symbols,
|
|
user_agent=args.user_agent,
|
|
)
|
|
specs = _default_specs()
|
|
for spec in specs:
|
|
entries = _build_entries_for_spec(store=store, events=events, spec=spec)
|
|
curve, trade_count = _simulate_equal_weight_curve(
|
|
store=store,
|
|
start_date=start_date,
|
|
end_date=end_date,
|
|
entries_by_date=entries,
|
|
spec=spec,
|
|
capital=args.capital,
|
|
)
|
|
rows.append(
|
|
{
|
|
"config": config_path,
|
|
"spec": spec.name,
|
|
"signals": int(sum(len(value) for value in entries.values())),
|
|
"trades": trade_count,
|
|
"total_return_pct": round(compute_total_return_pct(curve) or 0.0, 2),
|
|
"max_drawdown_pct": round(compute_max_drawdown_pct(curve) or 0.0, 2),
|
|
"sharpe_ratio": round(compute_sharpe_ratio(curve) or 0.0, 3),
|
|
"yearly_return_pct": _yearly_return_map(curve),
|
|
}
|
|
)
|
|
return rows
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description="Probe SEC Form 4 insider-buy strategies")
|
|
parser.add_argument("--config", action="append", required=True)
|
|
parser.add_argument("--start", required=True)
|
|
parser.add_argument("--end", required=True)
|
|
parser.add_argument("--capital", type=float, default=10_000.0)
|
|
parser.add_argument("--cache-dir", default="data/cache/sec_form345")
|
|
parser.add_argument("--user-agent", default=_DEFAULT_USER_AGENT)
|
|
parser.add_argument("--json", action="store_true")
|
|
args = parser.parse_args()
|
|
configure_logging("WARNING")
|
|
rows = _run_probe(args)
|
|
if args.json:
|
|
print(json.dumps(rows, indent=2))
|
|
else:
|
|
for row in rows:
|
|
print(row)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|