You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1005 lines
39 KiB
Python
1005 lines
39 KiB
Python
"""Standalone research probe for SEC Form 4 idle-alpha overlays.
|
|
|
|
This tool keeps all writes inside apps/tools and does not touch shared
|
|
strategy manifests. It:
|
|
|
|
1. Replays a baseline manifest over a fixed date window.
|
|
2. Downloads SEC quarterly Form 4 flat files for the covered window.
|
|
3. Builds richer insider-buy cluster features than the original probe.
|
|
4. Scans standalone Form 4 specs, then tests the best candidates as an
|
|
additive overlay versus the baseline's existing idle-alpha sleeve.
|
|
|
|
The overlay model is intentionally conservative:
|
|
- it only spends residual buying power already left idle by the baseline
|
|
- it treats the displaced benchmark as the baseline parking sleeve
|
|
- it skips unpublished SEC quarters instead of failing hard
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import csv
|
|
import datetime as dt
|
|
import io
|
|
import json
|
|
import math
|
|
import urllib.error
|
|
import urllib.request
|
|
import zipfile
|
|
from collections import defaultdict
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from apps.backtester.run import BacktestRunner, _build_merged_snapshot_store, load_manifest, resolve_config
|
|
from libs.backtest.domain import DailyPortfolioState
|
|
from libs.backtest.metrics import (
|
|
compute_max_drawdown_pct,
|
|
compute_sharpe_ratio,
|
|
compute_total_return_pct,
|
|
)
|
|
from libs.common.logging import configure_logging
|
|
|
|
|
|
_DEFAULT_USER_AGENT = "fithia2-form4-idle-alpha/1.0 (local research; contact: dev@example.com)"
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class RawForm4Transaction:
|
|
symbol: str
|
|
filing_date: dt.date
|
|
transaction_date: dt.date
|
|
owner_cik: str
|
|
owner_relationship: str
|
|
owner_title: str
|
|
shares: float
|
|
price: float
|
|
total_value: float
|
|
shares_owned_following: float
|
|
purchase_pct_of_holding: float
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Form4DailyEvent:
|
|
symbol: str
|
|
filing_date: dt.date
|
|
total_value: float
|
|
owner_count: int
|
|
transaction_count: int
|
|
event_day_count: int
|
|
max_purchase_pct: float
|
|
median_purchase_pct: float
|
|
weighted_purchase_pct: float
|
|
max_lag_days: int | None
|
|
min_lag_days: int | None
|
|
has_officer_or_director: bool
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Form4Spec:
|
|
name: str
|
|
cluster_window_days: int
|
|
min_owner_count: int
|
|
min_total_value: float
|
|
min_event_day_count: int
|
|
min_purchase_pct: float
|
|
max_lag_days: int | None
|
|
hold_days: int
|
|
max_positions: int = 6
|
|
max_new_per_day: int = 2
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class BaselineContext:
|
|
manifest_path: str
|
|
start_date: dt.date
|
|
signal_end_date: dt.date
|
|
evaluation_end_date: dt.date
|
|
initial_equity: float
|
|
store: Any
|
|
trading_days: list[dt.date]
|
|
baseline_curve: list[DailyPortfolioState]
|
|
baseline_equity_by_date: dict[dt.date, float]
|
|
baseline_cash_by_date: dict[dt.date, float]
|
|
baseline_metrics: dict[str, float]
|
|
parking_symbol_by_date: dict[dt.date, str]
|
|
|
|
|
|
def _parse_date(value: str, *, is_end: bool = False) -> dt.date:
|
|
parts = value.split("-")
|
|
if len(parts) == 1 and len(value) == 4 and value.isdigit():
|
|
year = int(value)
|
|
return dt.date(year, 12, 31) if is_end else dt.date(year, 1, 1)
|
|
if len(parts) == 2 and all(part.isdigit() for part in parts):
|
|
year = int(parts[0])
|
|
month = int(parts[1])
|
|
if is_end:
|
|
next_month = dt.date(year + (month // 12), (month % 12) + 1, 1)
|
|
return next_month - dt.timedelta(days=1)
|
|
return dt.date(year, month, 1)
|
|
return dt.date.fromisoformat(value)
|
|
|
|
|
|
def _quarter_range(start_date: dt.date, end_date: dt.date) -> list[tuple[int, int]]:
|
|
year = start_date.year
|
|
quarter = (start_date.month - 1) // 3 + 1
|
|
end_key = (end_date.year, (end_date.month - 1) // 3 + 1)
|
|
quarters: list[tuple[int, int]] = []
|
|
while (year, quarter) <= end_key:
|
|
quarters.append((year, quarter))
|
|
quarter += 1
|
|
if quarter == 5:
|
|
year += 1
|
|
quarter = 1
|
|
return quarters
|
|
|
|
|
|
def _quarter_zip_path(cache_dir: Path, year: int, quarter: int) -> Path:
|
|
return cache_dir / f"{year}q{quarter}_form345.zip"
|
|
|
|
|
|
def _quarter_zip_url(year: int, quarter: int) -> str:
|
|
return (
|
|
"https://www.sec.gov/files/structureddata/data/"
|
|
f"insider-transactions-data-sets/{year}q{quarter}_form345.zip"
|
|
)
|
|
|
|
|
|
def _ensure_quarter_zip(
|
|
cache_dir: Path,
|
|
year: int,
|
|
quarter: int,
|
|
*,
|
|
user_agent: str,
|
|
) -> Path | None:
|
|
cache_dir.mkdir(parents=True, exist_ok=True)
|
|
path = _quarter_zip_path(cache_dir, year, quarter)
|
|
if path.exists() and path.stat().st_size > 0:
|
|
return path
|
|
request = urllib.request.Request(
|
|
_quarter_zip_url(year, quarter),
|
|
headers={"User-Agent": user_agent},
|
|
)
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=60) as response:
|
|
data = response.read()
|
|
except urllib.error.HTTPError as exc:
|
|
if exc.code == 404:
|
|
return None
|
|
raise
|
|
path.write_bytes(data)
|
|
return path
|
|
|
|
|
|
def _parse_sec_date(value: str | None) -> dt.date | None:
|
|
if not value:
|
|
return None
|
|
try:
|
|
return dt.datetime.strptime(value, "%d-%b-%Y").date()
|
|
except ValueError:
|
|
return None
|
|
|
|
|
|
def _coerce_float(value: Any) -> float | None:
|
|
try:
|
|
result = float(value)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
if math.isnan(result) or math.isinf(result):
|
|
return None
|
|
return result
|
|
|
|
|
|
def _normalize_title(text: str) -> str:
|
|
normalized = text.strip().lower()
|
|
return " ".join(normalized.split())
|
|
|
|
|
|
def _is_officer_or_director(relationship: str, title: str) -> bool:
|
|
combined = f"{relationship} {title}".strip().lower()
|
|
tokens = (
|
|
"director",
|
|
"officer",
|
|
"chief",
|
|
"ceo",
|
|
"cfo",
|
|
"coo",
|
|
"president",
|
|
"chair",
|
|
)
|
|
return any(token in combined for token in tokens)
|
|
|
|
|
|
def _load_form4_transactions(
|
|
*,
|
|
cache_dir: Path,
|
|
start_date: dt.date,
|
|
end_date: dt.date,
|
|
allowed_symbols: set[str],
|
|
user_agent: str,
|
|
) -> tuple[list[RawForm4Transaction], list[str]]:
|
|
transactions: list[RawForm4Transaction] = []
|
|
skipped_quarters: list[str] = []
|
|
|
|
for year, quarter in _quarter_range(start_date, end_date):
|
|
path = _ensure_quarter_zip(cache_dir, year, quarter, user_agent=user_agent)
|
|
if path is None:
|
|
skipped_quarters.append(f"{year}Q{quarter}")
|
|
continue
|
|
with zipfile.ZipFile(path) as zf:
|
|
submissions: dict[str, dict[str, Any]] = {}
|
|
with zf.open("SUBMISSION.tsv") as handle:
|
|
reader = csv.DictReader(
|
|
io.TextIOWrapper(handle, encoding="utf-8", newline=""),
|
|
delimiter="\t",
|
|
)
|
|
for row in reader:
|
|
symbol = str(row.get("ISSUERTRADINGSYMBOL") or "").strip().upper()
|
|
if not symbol or symbol not in allowed_symbols:
|
|
continue
|
|
if str(row.get("DOCUMENT_TYPE") or "").strip().upper() != "4":
|
|
continue
|
|
filing_date = _parse_sec_date(row.get("FILING_DATE"))
|
|
if filing_date is None or filing_date < start_date or filing_date > end_date:
|
|
continue
|
|
submissions[str(row["ACCESSION_NUMBER"])] = {
|
|
"symbol": symbol,
|
|
"filing_date": filing_date,
|
|
}
|
|
|
|
if not submissions:
|
|
continue
|
|
|
|
relationships: dict[str, list[tuple[str, str, str]]] = defaultdict(list)
|
|
with zf.open("REPORTINGOWNER.tsv") as handle:
|
|
reader = csv.DictReader(
|
|
io.TextIOWrapper(handle, encoding="utf-8", newline=""),
|
|
delimiter="\t",
|
|
)
|
|
for row in reader:
|
|
accession = str(row["ACCESSION_NUMBER"])
|
|
if accession not in submissions:
|
|
continue
|
|
owner_cik = str(row.get("RPTOWNERCIK") or "").strip()
|
|
relationship = str(row.get("RPTOWNER_RELATIONSHIP") or "").strip().lower()
|
|
title = _normalize_title(str(row.get("RPTOWNER_TITLE") or ""))
|
|
relationships[accession].append((owner_cik, relationship, title))
|
|
|
|
with zf.open("NONDERIV_TRANS.tsv") as handle:
|
|
reader = csv.DictReader(
|
|
io.TextIOWrapper(handle, encoding="utf-8", newline=""),
|
|
delimiter="\t",
|
|
)
|
|
for row in reader:
|
|
accession = str(row["ACCESSION_NUMBER"])
|
|
submission = submissions.get(accession)
|
|
if submission is None:
|
|
continue
|
|
if str(row.get("TRANS_CODE") or "").strip().upper() != "P":
|
|
continue
|
|
if str(row.get("TRANS_ACQUIRED_DISP_CD") or "").strip().upper() != "A":
|
|
continue
|
|
|
|
shares = _coerce_float(row.get("TRANS_SHARES"))
|
|
price = _coerce_float(row.get("TRANS_PRICEPERSHARE"))
|
|
shares_following = _coerce_float(row.get("SHRS_OWND_FOLWNG_TRANS")) or 0.0
|
|
if shares is None or price is None or shares <= 0 or price <= 0:
|
|
continue
|
|
|
|
transaction_date = (
|
|
_parse_sec_date(row.get("TRANS_DATE"))
|
|
or submission["filing_date"]
|
|
)
|
|
purchase_pct = (
|
|
shares / shares_following
|
|
if shares_following > 0
|
|
else 0.0
|
|
)
|
|
owner_rows = relationships.get(accession) or [("", "", "")]
|
|
for owner_cik, relationship, title in owner_rows:
|
|
transactions.append(
|
|
RawForm4Transaction(
|
|
symbol=submission["symbol"],
|
|
filing_date=submission["filing_date"],
|
|
transaction_date=transaction_date,
|
|
owner_cik=owner_cik,
|
|
owner_relationship=relationship,
|
|
owner_title=title,
|
|
shares=shares,
|
|
price=price,
|
|
total_value=shares * price,
|
|
shares_owned_following=shares_following,
|
|
purchase_pct_of_holding=purchase_pct,
|
|
)
|
|
)
|
|
return transactions, skipped_quarters
|
|
|
|
|
|
def _aggregate_daily_events(transactions: list[RawForm4Transaction]) -> list[Form4DailyEvent]:
|
|
grouped: dict[tuple[dt.date, str], dict[str, Any]] = {}
|
|
for row in transactions:
|
|
key = (row.filing_date, row.symbol)
|
|
event = grouped.setdefault(
|
|
key,
|
|
{
|
|
"filing_date": row.filing_date,
|
|
"symbol": row.symbol,
|
|
"total_value": 0.0,
|
|
"transaction_count": 0,
|
|
"owner_ciks": set(),
|
|
"purchase_pcts": [],
|
|
"weighted_num": 0.0,
|
|
"weighted_den": 0.0,
|
|
"lag_days": [],
|
|
"has_officer_or_director": False,
|
|
},
|
|
)
|
|
event["total_value"] += row.total_value
|
|
event["transaction_count"] += 1
|
|
if row.owner_cik:
|
|
event["owner_ciks"].add(row.owner_cik)
|
|
if row.purchase_pct_of_holding > 0:
|
|
event["purchase_pcts"].append(row.purchase_pct_of_holding)
|
|
if row.shares_owned_following > 0:
|
|
event["weighted_num"] += row.shares
|
|
event["weighted_den"] += row.shares_owned_following
|
|
lag_days = (row.filing_date - row.transaction_date).days
|
|
event["lag_days"].append(lag_days)
|
|
if _is_officer_or_director(row.owner_relationship, row.owner_title):
|
|
event["has_officer_or_director"] = True
|
|
|
|
events: list[Form4DailyEvent] = []
|
|
for payload in grouped.values():
|
|
purchase_pcts = payload["purchase_pcts"] or [0.0]
|
|
weighted_purchase_pct = (
|
|
payload["weighted_num"] / payload["weighted_den"]
|
|
if payload["weighted_den"] > 0
|
|
else 0.0
|
|
)
|
|
lag_days = payload["lag_days"]
|
|
events.append(
|
|
Form4DailyEvent(
|
|
symbol=str(payload["symbol"]),
|
|
filing_date=payload["filing_date"],
|
|
total_value=float(payload["total_value"]),
|
|
owner_count=len(payload["owner_ciks"]),
|
|
transaction_count=int(payload["transaction_count"]),
|
|
event_day_count=1,
|
|
max_purchase_pct=max(purchase_pcts),
|
|
median_purchase_pct=sorted(purchase_pcts)[len(purchase_pcts) // 2],
|
|
weighted_purchase_pct=weighted_purchase_pct,
|
|
max_lag_days=max(lag_days) if lag_days else None,
|
|
min_lag_days=min(lag_days) if lag_days else None,
|
|
has_officer_or_director=bool(payload["has_officer_or_director"]),
|
|
)
|
|
)
|
|
events.sort(key=lambda row: (row.symbol, row.filing_date))
|
|
return events
|
|
|
|
|
|
def _build_cluster_events(
|
|
daily_events: list[Form4DailyEvent],
|
|
*,
|
|
window_days: int,
|
|
) -> list[Form4DailyEvent]:
|
|
if window_days <= 0:
|
|
return daily_events
|
|
|
|
by_symbol: dict[str, list[Form4DailyEvent]] = defaultdict(list)
|
|
for event in daily_events:
|
|
by_symbol[event.symbol].append(event)
|
|
|
|
clusters: list[Form4DailyEvent] = []
|
|
for symbol, events in by_symbol.items():
|
|
events.sort(key=lambda row: row.filing_date)
|
|
left = 0
|
|
owner_counts: dict[int, int] = defaultdict(int)
|
|
rolling_total_value = 0.0
|
|
rolling_transactions = 0
|
|
rolling_has_role = False
|
|
for right, event in enumerate(events):
|
|
cutoff = event.filing_date - dt.timedelta(days=window_days)
|
|
while left <= right and events[left].filing_date < cutoff:
|
|
old = events[left]
|
|
rolling_total_value -= old.total_value
|
|
rolling_transactions -= old.transaction_count
|
|
left += 1
|
|
|
|
rolling_total_value += event.total_value
|
|
rolling_transactions += event.transaction_count
|
|
cluster_items = events[left : right + 1]
|
|
rolling_has_role = any(item.has_officer_or_director for item in cluster_items)
|
|
purchase_values = [item.max_purchase_pct for item in cluster_items]
|
|
median_values = [item.median_purchase_pct for item in cluster_items]
|
|
weighted_values = [item.weighted_purchase_pct for item in cluster_items]
|
|
lag_values = [item.max_lag_days for item in cluster_items if item.max_lag_days is not None]
|
|
|
|
owner_set: set[str] = set()
|
|
for item in cluster_items:
|
|
# owner_count is already deduped within the filing day, but we need
|
|
# a proxy across days without carrying the full owner identity.
|
|
# Use the observed owner_count sum capped by transaction_count when
|
|
# cross-day owner identity is unavailable after aggregation.
|
|
owner_set.update({f"{item.symbol}:{item.filing_date}:{idx}" for idx in range(item.owner_count)})
|
|
|
|
clusters.append(
|
|
Form4DailyEvent(
|
|
symbol=symbol,
|
|
filing_date=event.filing_date,
|
|
total_value=rolling_total_value,
|
|
owner_count=len(owner_set),
|
|
transaction_count=rolling_transactions,
|
|
event_day_count=len(cluster_items),
|
|
max_purchase_pct=max(purchase_values) if purchase_values else 0.0,
|
|
median_purchase_pct=sorted(median_values)[len(median_values) // 2] if median_values else 0.0,
|
|
weighted_purchase_pct=max(weighted_values) if weighted_values else 0.0,
|
|
max_lag_days=max(lag_values) if lag_values else None,
|
|
min_lag_days=min(lag_values) if lag_values else None,
|
|
has_officer_or_director=rolling_has_role,
|
|
)
|
|
)
|
|
clusters.sort(key=lambda row: (row.symbol, row.filing_date))
|
|
return clusters
|
|
|
|
|
|
def _build_entries_for_spec(
|
|
*,
|
|
store: Any,
|
|
events: list[Form4DailyEvent],
|
|
spec: Form4Spec,
|
|
) -> dict[dt.date, list[dict[str, Any]]]:
|
|
trading_days = store.all_trading_days()
|
|
trading_index = {date: idx for idx, date in enumerate(trading_days)}
|
|
entries: dict[dt.date, list[dict[str, Any]]] = defaultdict(list)
|
|
|
|
for event in events:
|
|
if event.owner_count < spec.min_owner_count:
|
|
continue
|
|
if event.total_value < spec.min_total_value:
|
|
continue
|
|
if event.event_day_count < spec.min_event_day_count:
|
|
continue
|
|
if event.weighted_purchase_pct < spec.min_purchase_pct:
|
|
continue
|
|
if spec.max_lag_days is not None and event.max_lag_days is not None and event.max_lag_days > spec.max_lag_days:
|
|
continue
|
|
|
|
entry_date = next((date for date in trading_days if date > event.filing_date), None)
|
|
if entry_date is None:
|
|
continue
|
|
entry_bar = store.get_bar(event.symbol, entry_date)
|
|
if not entry_bar or not entry_bar.get("open") or not entry_bar.get("close"):
|
|
continue
|
|
entry_index = trading_index.get(entry_date)
|
|
if entry_index is None or entry_index + spec.hold_days >= len(trading_days):
|
|
continue
|
|
exit_date = trading_days[entry_index + spec.hold_days]
|
|
exit_bar = store.get_bar(event.symbol, exit_date)
|
|
if not exit_bar or not exit_bar.get("close"):
|
|
continue
|
|
|
|
entries[entry_date].append(
|
|
{
|
|
"symbol": event.symbol,
|
|
"exit_date": exit_date,
|
|
"score": (
|
|
event.owner_count,
|
|
event.event_day_count,
|
|
round(event.weighted_purchase_pct, 6),
|
|
round(event.total_value, 2),
|
|
),
|
|
"owner_count": event.owner_count,
|
|
"event_day_count": event.event_day_count,
|
|
"total_value": event.total_value,
|
|
"purchase_pct": event.weighted_purchase_pct,
|
|
}
|
|
)
|
|
return entries
|
|
|
|
|
|
def _dedupe_curve_by_date(curve: list[DailyPortfolioState]) -> list[DailyPortfolioState]:
|
|
by_date: dict[dt.date, DailyPortfolioState] = {}
|
|
for state in curve:
|
|
by_date[state.date] = state
|
|
return [by_date[date] for date in sorted(by_date)]
|
|
|
|
|
|
def _build_curve_metrics(curve: list[DailyPortfolioState]) -> dict[str, float]:
|
|
return {
|
|
"total_return_pct": round(compute_total_return_pct(curve) or 0.0, 2),
|
|
"max_drawdown_pct": round(compute_max_drawdown_pct(curve) or 0.0, 2),
|
|
"sharpe_ratio": round(compute_sharpe_ratio(curve) or 0.0, 3),
|
|
}
|
|
|
|
|
|
def _yearly_return_map(curve: list[DailyPortfolioState]) -> dict[str, float]:
|
|
if not curve:
|
|
return {}
|
|
year_start: dict[int, float] = {}
|
|
year_end: dict[int, float] = {}
|
|
for state in curve:
|
|
year_start.setdefault(state.date.year, state.equity)
|
|
year_end[state.date.year] = state.equity
|
|
return {
|
|
str(year): round((year_end[year] / year_start[year] - 1.0) * 100.0, 2)
|
|
for year in sorted(year_start)
|
|
if year_start[year] > 0
|
|
}
|
|
|
|
|
|
def _simulate_equal_weight_curve(
|
|
*,
|
|
store: Any,
|
|
start_date: dt.date,
|
|
end_date: dt.date,
|
|
entries_by_date: dict[dt.date, list[dict[str, Any]]],
|
|
spec: Form4Spec,
|
|
capital: float,
|
|
) -> tuple[list[DailyPortfolioState], int]:
|
|
trading_days = [
|
|
date for date in store.all_trading_days() if start_date <= date <= end_date
|
|
]
|
|
equity = capital
|
|
peak = capital
|
|
curve: list[DailyPortfolioState] = []
|
|
open_positions: list[dict[str, Any]] = []
|
|
trade_count = 0
|
|
|
|
for date in trading_days:
|
|
if date in entries_by_date and len(open_positions) < spec.max_positions:
|
|
existing_symbols = {position["symbol"] for position in open_positions}
|
|
ranked = sorted(entries_by_date[date], key=lambda row: row["score"], reverse=True)
|
|
added = 0
|
|
for row in ranked:
|
|
if row["symbol"] in existing_symbols:
|
|
continue
|
|
bar = store.get_bar(row["symbol"], date)
|
|
if not bar or not bar.get("open"):
|
|
continue
|
|
open_positions.append(
|
|
{
|
|
"symbol": row["symbol"],
|
|
"exit_date": row["exit_date"],
|
|
"prev_price": float(bar["open"]),
|
|
}
|
|
)
|
|
existing_symbols.add(row["symbol"])
|
|
trade_count += 1
|
|
added += 1
|
|
if added >= spec.max_new_per_day or len(open_positions) >= spec.max_positions:
|
|
break
|
|
|
|
if open_positions:
|
|
daily_returns: list[float] = []
|
|
updated_positions: list[dict[str, Any]] = []
|
|
for position in open_positions:
|
|
bar = store.get_bar(position["symbol"], date)
|
|
if not bar or not bar.get("close"):
|
|
continue
|
|
close_price = float(bar["close"])
|
|
prev_price = float(position["prev_price"])
|
|
if prev_price <= 0:
|
|
continue
|
|
daily_returns.append(close_price / prev_price - 1.0)
|
|
updated_positions.append(
|
|
{
|
|
"symbol": position["symbol"],
|
|
"exit_date": position["exit_date"],
|
|
"prev_price": close_price,
|
|
}
|
|
)
|
|
if daily_returns:
|
|
equity *= 1.0 + sum(daily_returns) / len(daily_returns)
|
|
open_positions = [
|
|
position for position in updated_positions if position["exit_date"] > date
|
|
]
|
|
|
|
peak = max(peak, equity)
|
|
curve.append(
|
|
DailyPortfolioState(
|
|
date=date,
|
|
equity=equity,
|
|
sizing_equity=equity,
|
|
cash_available=equity,
|
|
gross_exposure=float(len(open_positions)) * 10.0,
|
|
net_exposure=float(len(open_positions)) * 10.0,
|
|
reserved_risk_budget=0.0,
|
|
unrealized_pnl=0.0,
|
|
realized_pnl=0.0,
|
|
open_positions=[position["symbol"] for position in open_positions],
|
|
daily_new_risk_used=0.0,
|
|
peak_equity=peak,
|
|
current_drawdown_pct=((peak - equity) / peak * 100.0) if peak > 0 else 0.0,
|
|
)
|
|
)
|
|
return curve, trade_count
|
|
|
|
|
|
def _build_parking_symbol_by_date(context: BaselineContext, config: Any) -> dict[dt.date, str]:
|
|
store = context.store
|
|
runner = BacktestRunner(
|
|
manifest=load_manifest(context.manifest_path),
|
|
config=config,
|
|
store=store,
|
|
initial_equity=context.initial_equity,
|
|
enable_engine_analysis=False,
|
|
)
|
|
trading_days = context.trading_days
|
|
if not trading_days:
|
|
return {}
|
|
|
|
held_symbol = runner._evaluate_parking_target(trading_days[0]) or "sgov"
|
|
runner._commit_parking_target(held_symbol)
|
|
parking_symbol_by_date: dict[dt.date, str] = {trading_days[0]: held_symbol}
|
|
for idx in range(1, len(trading_days)):
|
|
date = trading_days[idx]
|
|
parking_symbol_by_date[date] = held_symbol
|
|
next_symbol = runner._evaluate_parking_target(date) or held_symbol
|
|
runner._commit_parking_target(next_symbol)
|
|
held_symbol = next_symbol
|
|
return parking_symbol_by_date
|
|
|
|
|
|
def _build_baseline_context(
|
|
*,
|
|
manifest_path: str,
|
|
start_date: dt.date,
|
|
end_date: dt.date,
|
|
initial_equity: float,
|
|
) -> BaselineContext:
|
|
manifest = load_manifest(manifest_path)
|
|
config = resolve_config(manifest)
|
|
store = _build_merged_snapshot_store(
|
|
manifest,
|
|
config,
|
|
snapshot_dir_override=None,
|
|
).slice_by_date_range(start_date, end_date)
|
|
runner = BacktestRunner(
|
|
manifest=manifest,
|
|
config=config,
|
|
store=store,
|
|
initial_equity=initial_equity,
|
|
enable_engine_analysis=False,
|
|
)
|
|
result = runner.run(output_root=None)
|
|
baseline_curve = _dedupe_curve_by_date(runner._equity_curve)
|
|
trading_days = [state.date for state in baseline_curve]
|
|
baseline_equity_by_date = {state.date: float(state.equity) for state in baseline_curve}
|
|
baseline_cash_by_date = {state.date: float(state.cash_available) for state in baseline_curve}
|
|
baseline_metrics = _build_curve_metrics(baseline_curve)
|
|
baseline_context = BaselineContext(
|
|
manifest_path=manifest_path,
|
|
start_date=start_date,
|
|
signal_end_date=end_date,
|
|
evaluation_end_date=trading_days[-1] if trading_days else end_date,
|
|
initial_equity=initial_equity,
|
|
store=store,
|
|
trading_days=trading_days,
|
|
baseline_curve=baseline_curve,
|
|
baseline_equity_by_date=baseline_equity_by_date,
|
|
baseline_cash_by_date=baseline_cash_by_date,
|
|
baseline_metrics=baseline_metrics,
|
|
parking_symbol_by_date={},
|
|
)
|
|
parking_symbol_by_date = _build_parking_symbol_by_date(baseline_context, config)
|
|
return BaselineContext(
|
|
manifest_path=manifest_path,
|
|
start_date=start_date,
|
|
signal_end_date=end_date,
|
|
evaluation_end_date=trading_days[-1] if trading_days else end_date,
|
|
initial_equity=initial_equity,
|
|
store=store,
|
|
trading_days=trading_days,
|
|
baseline_curve=baseline_curve,
|
|
baseline_equity_by_date=baseline_equity_by_date,
|
|
baseline_cash_by_date=baseline_cash_by_date,
|
|
baseline_metrics=baseline_metrics,
|
|
parking_symbol_by_date=parking_symbol_by_date,
|
|
)
|
|
|
|
|
|
def _simulate_overlay_curve(
|
|
*,
|
|
context: BaselineContext,
|
|
entries_by_date: dict[dt.date, list[dict[str, Any]]],
|
|
spec: Form4Spec,
|
|
deploy_idle_fraction: float,
|
|
) -> tuple[list[DailyPortfolioState], int]:
|
|
store = context.store
|
|
trading_days = context.trading_days
|
|
open_positions: list[dict[str, Any]] = []
|
|
realized_excess = 0.0
|
|
trade_count = 0
|
|
curve: list[DailyPortfolioState] = []
|
|
peak = context.initial_equity
|
|
|
|
for date_index, date in enumerate(trading_days):
|
|
benchmark_symbol = context.parking_symbol_by_date.get(date, "sgov").upper()
|
|
benchmark_bar = store.get_bar(benchmark_symbol, date)
|
|
if benchmark_bar is None or benchmark_bar.get("close") is None:
|
|
benchmark_bar = store.get_bar("SGOV", date)
|
|
|
|
updated_positions: list[dict[str, Any]] = []
|
|
for position in open_positions:
|
|
bar = store.get_bar(position["symbol"], date)
|
|
if not bar or not bar.get("close"):
|
|
continue
|
|
actual_close = float(bar["close"])
|
|
shadow_close = float((benchmark_bar or {}).get("close") or position["shadow_prev_close"])
|
|
actual_value = position["actual_value"]
|
|
shadow_value = position["shadow_value"]
|
|
if position["prev_actual_price"] > 0:
|
|
actual_value *= actual_close / position["prev_actual_price"]
|
|
if position["shadow_prev_close"] > 0:
|
|
shadow_value *= shadow_close / position["shadow_prev_close"]
|
|
next_state = {
|
|
**position,
|
|
"actual_value": actual_value,
|
|
"shadow_value": shadow_value,
|
|
"prev_actual_price": actual_close,
|
|
"shadow_prev_close": shadow_close,
|
|
}
|
|
if position["exit_date"] <= date:
|
|
realized_excess += actual_value - shadow_value
|
|
else:
|
|
updated_positions.append(next_state)
|
|
open_positions = updated_positions
|
|
|
|
baseline_equity = context.baseline_equity_by_date[date]
|
|
baseline_cash = context.baseline_cash_by_date[date]
|
|
shadow_notional_open = sum(position["shadow_value"] for position in open_positions)
|
|
idle_budget = max(0.0, baseline_cash * deploy_idle_fraction - shadow_notional_open)
|
|
|
|
if date in entries_by_date and len(open_positions) < spec.max_positions and idle_budget > 0:
|
|
existing_symbols = {position["symbol"] for position in open_positions}
|
|
ranked = sorted(entries_by_date[date], key=lambda row: row["score"], reverse=True)
|
|
remaining_slots = max(0, spec.max_positions - len(open_positions))
|
|
ranked = [
|
|
row for row in ranked
|
|
if row["symbol"] not in existing_symbols
|
|
][: min(spec.max_new_per_day, remaining_slots)]
|
|
if ranked:
|
|
per_position_budget = idle_budget / len(ranked)
|
|
benchmark_open = float((benchmark_bar or {}).get("open") or (benchmark_bar or {}).get("close") or 0.0)
|
|
benchmark_close = float((benchmark_bar or {}).get("close") or benchmark_open or 0.0)
|
|
for row in ranked:
|
|
bar = store.get_bar(row["symbol"], date)
|
|
if not bar or not bar.get("open") or not bar.get("close"):
|
|
continue
|
|
actual_open = float(bar["open"])
|
|
actual_close = float(bar["close"])
|
|
if per_position_budget <= 0 or actual_open <= 0 or benchmark_open <= 0:
|
|
continue
|
|
open_positions.append(
|
|
{
|
|
"symbol": row["symbol"],
|
|
"exit_date": row["exit_date"],
|
|
"actual_value": per_position_budget * (actual_close / actual_open),
|
|
"shadow_value": per_position_budget * (benchmark_close / benchmark_open),
|
|
"prev_actual_price": actual_close,
|
|
"shadow_prev_close": benchmark_close,
|
|
}
|
|
)
|
|
trade_count += 1
|
|
|
|
open_excess = sum(position["actual_value"] - position["shadow_value"] for position in open_positions)
|
|
equity = baseline_equity + realized_excess + open_excess
|
|
peak = max(peak, equity)
|
|
curve.append(
|
|
DailyPortfolioState(
|
|
date=date,
|
|
equity=equity,
|
|
sizing_equity=equity,
|
|
cash_available=max(0.0, context.baseline_cash_by_date[date] - sum(position["shadow_value"] for position in open_positions)),
|
|
gross_exposure=float(len(open_positions)) * 10.0,
|
|
net_exposure=float(len(open_positions)) * 10.0,
|
|
reserved_risk_budget=0.0,
|
|
unrealized_pnl=open_excess,
|
|
realized_pnl=realized_excess,
|
|
open_positions=[position["symbol"] for position in open_positions],
|
|
daily_new_risk_used=0.0,
|
|
peak_equity=peak,
|
|
current_drawdown_pct=((peak - equity) / peak * 100.0) if peak > 0 else 0.0,
|
|
)
|
|
)
|
|
|
|
return curve, trade_count
|
|
|
|
|
|
def _standalone_score(metrics: dict[str, float]) -> float:
|
|
drawdown = max(metrics["max_drawdown_pct"], 1.0)
|
|
return metrics["total_return_pct"] / drawdown + metrics["sharpe_ratio"] * 25.0
|
|
|
|
|
|
def _overlay_score(metrics: dict[str, float], baseline_metrics: dict[str, float]) -> float:
|
|
delta_return = metrics["total_return_pct"] - baseline_metrics["total_return_pct"]
|
|
delta_drawdown = metrics["max_drawdown_pct"] - baseline_metrics["max_drawdown_pct"]
|
|
return delta_return * 4.0 - max(delta_drawdown, 0.0) * 35.0 + metrics["sharpe_ratio"] * 10.0
|
|
|
|
|
|
def _generate_specs() -> list[Form4Spec]:
|
|
specs: list[Form4Spec] = []
|
|
for cluster_window_days in (0, 3, 5, 10):
|
|
for min_owner_count in (2, 3):
|
|
for min_total_value in (1_000_000.0, 2_000_000.0, 5_000_000.0):
|
|
for min_event_day_count in ((1,) if cluster_window_days == 0 else (1, 2)):
|
|
for min_purchase_pct in (0.0, 0.03, 0.05):
|
|
for max_lag_days in (None, 2, 4):
|
|
for hold_days in (5, 10, 20):
|
|
name = (
|
|
f"w{cluster_window_days}_o{min_owner_count}"
|
|
f"_v{int(min_total_value/1_000_000)}m"
|
|
f"_d{min_event_day_count}"
|
|
f"_p{int(min_purchase_pct*100):02d}"
|
|
f"_lag{max_lag_days if max_lag_days is not None else 'na'}"
|
|
f"_h{hold_days}"
|
|
)
|
|
specs.append(
|
|
Form4Spec(
|
|
name=name,
|
|
cluster_window_days=cluster_window_days,
|
|
min_owner_count=min_owner_count,
|
|
min_total_value=min_total_value,
|
|
min_event_day_count=min_event_day_count,
|
|
min_purchase_pct=min_purchase_pct,
|
|
max_lag_days=max_lag_days,
|
|
hold_days=hold_days,
|
|
)
|
|
)
|
|
return specs
|
|
|
|
|
|
def _run_probe(args: argparse.Namespace) -> dict[str, Any]:
|
|
start_date = _parse_date(args.start)
|
|
end_date = _parse_date(args.end, is_end=True)
|
|
baseline = _build_baseline_context(
|
|
manifest_path=args.config,
|
|
start_date=start_date,
|
|
end_date=end_date,
|
|
initial_equity=args.capital,
|
|
)
|
|
|
|
allowed_symbols = {str(symbol).upper() for symbol in baseline.store._bars.keys()}
|
|
transactions, skipped_quarters = _load_form4_transactions(
|
|
cache_dir=Path(args.cache_dir),
|
|
start_date=start_date,
|
|
end_date=end_date,
|
|
allowed_symbols=allowed_symbols,
|
|
user_agent=args.user_agent,
|
|
)
|
|
daily_events = _aggregate_daily_events(transactions)
|
|
|
|
events_by_window: dict[int, list[Form4DailyEvent]] = {}
|
|
specs = _generate_specs()
|
|
standalone_rows: list[dict[str, Any]] = []
|
|
for spec in specs:
|
|
if spec.cluster_window_days not in events_by_window:
|
|
events_by_window[spec.cluster_window_days] = _build_cluster_events(
|
|
daily_events,
|
|
window_days=spec.cluster_window_days,
|
|
)
|
|
entries = _build_entries_for_spec(
|
|
store=baseline.store,
|
|
events=events_by_window[spec.cluster_window_days],
|
|
spec=spec,
|
|
)
|
|
curve, trade_count = _simulate_equal_weight_curve(
|
|
store=baseline.store,
|
|
start_date=start_date,
|
|
end_date=end_date,
|
|
entries_by_date=entries,
|
|
spec=spec,
|
|
capital=args.capital,
|
|
)
|
|
metrics = _build_curve_metrics(curve)
|
|
standalone_rows.append(
|
|
{
|
|
"spec": spec.name,
|
|
"cluster_window_days": spec.cluster_window_days,
|
|
"signals": int(sum(len(value) for value in entries.values())),
|
|
"trades": trade_count,
|
|
**metrics,
|
|
"standalone_score": round(_standalone_score(metrics), 3),
|
|
"yearly_return_pct": _yearly_return_map(curve),
|
|
}
|
|
)
|
|
|
|
standalone_rows.sort(key=lambda row: row["standalone_score"], reverse=True)
|
|
spec_lookup = {spec.name: spec for spec in specs}
|
|
|
|
overlay_rows: list[dict[str, Any]] = []
|
|
deploy_idle_fractions = (0.03, 0.04, 0.05, 0.06, 0.08, 0.1)
|
|
for spec_name, spec in spec_lookup.items():
|
|
entries = _build_entries_for_spec(
|
|
store=baseline.store,
|
|
events=events_by_window[spec.cluster_window_days],
|
|
spec=spec,
|
|
)
|
|
for deploy_idle_fraction in deploy_idle_fractions:
|
|
curve, trade_count = _simulate_overlay_curve(
|
|
context=baseline,
|
|
entries_by_date=entries,
|
|
spec=spec,
|
|
deploy_idle_fraction=deploy_idle_fraction,
|
|
)
|
|
metrics = _build_curve_metrics(curve)
|
|
overlay_rows.append(
|
|
{
|
|
"spec": spec_name,
|
|
"deploy_idle_fraction": deploy_idle_fraction,
|
|
"signals": int(sum(len(value) for value in entries.values())),
|
|
"trades": trade_count,
|
|
**metrics,
|
|
"delta_return_pct": round(metrics["total_return_pct"] - baseline.baseline_metrics["total_return_pct"], 2),
|
|
"delta_drawdown_pct": round(metrics["max_drawdown_pct"] - baseline.baseline_metrics["max_drawdown_pct"], 2),
|
|
"overlay_score": round(_overlay_score(metrics, baseline.baseline_metrics), 3),
|
|
"yearly_return_pct": _yearly_return_map(curve),
|
|
}
|
|
)
|
|
|
|
overlay_rows.sort(key=lambda row: row["overlay_score"], reverse=True)
|
|
low_dd_overlay_rows = [
|
|
row for row in overlay_rows
|
|
if row["delta_drawdown_pct"] <= 0.5
|
|
]
|
|
low_dd_overlay_rows.sort(
|
|
key=lambda row: (row["delta_return_pct"], row["sharpe_ratio"]),
|
|
reverse=True,
|
|
)
|
|
best_overlay = low_dd_overlay_rows[0] if low_dd_overlay_rows else (overlay_rows[0] if overlay_rows else None)
|
|
verdict = "not_tested"
|
|
if best_overlay is not None:
|
|
if best_overlay["delta_return_pct"] >= 25.0 and best_overlay["delta_drawdown_pct"] <= 0.5:
|
|
verdict = "worth_integrating"
|
|
elif best_overlay["delta_return_pct"] > 0:
|
|
verdict = "maybe_research_more"
|
|
else:
|
|
verdict = "not_worth_integrating"
|
|
|
|
return {
|
|
"window": {
|
|
"start_date": start_date.isoformat(),
|
|
"signal_end_date": end_date.isoformat(),
|
|
"evaluation_end_date": baseline.evaluation_end_date.isoformat(),
|
|
"skipped_quarters": skipped_quarters,
|
|
},
|
|
"baseline": {
|
|
"manifest": args.config,
|
|
**baseline.baseline_metrics,
|
|
"yearly_return_pct": _yearly_return_map(baseline.baseline_curve),
|
|
},
|
|
"form4_data": {
|
|
"transactions": len(transactions),
|
|
"daily_events": len(daily_events),
|
|
"cluster_windows_tested": sorted(events_by_window),
|
|
},
|
|
"standalone_top": standalone_rows[: args.report_top_n],
|
|
"overlay_top": overlay_rows[: args.report_top_n],
|
|
"overlay_low_dd_top": low_dd_overlay_rows[: args.report_top_n],
|
|
"verdict": verdict,
|
|
}
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description="Probe SEC Form 4 idle-alpha overlays")
|
|
parser.add_argument("--config", required=True, help="Baseline experiment manifest JSON")
|
|
parser.add_argument("--start", default="2022-03-03")
|
|
parser.add_argument("--end", default="2025-12-31")
|
|
parser.add_argument("--capital", type=float, default=10_000.0)
|
|
parser.add_argument("--cache-dir", default="data/cache/sec_form345")
|
|
parser.add_argument("--user-agent", default=_DEFAULT_USER_AGENT)
|
|
parser.add_argument("--overlay-top-k", type=int, default=12)
|
|
parser.add_argument("--report-top-n", type=int, default=8)
|
|
parser.add_argument("--json", action="store_true")
|
|
args = parser.parse_args()
|
|
configure_logging("WARNING")
|
|
result = _run_probe(args)
|
|
if args.json:
|
|
print(json.dumps(result, indent=2))
|
|
else:
|
|
print(result)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|