feat: overhaul strategy — document quality > price momentum
Flip scoring weights so event/document quality is primary signal (55%) and market confirmation is secondary (35%). Add research mode with kill-switch cooldown/reset, veto gates for bad events, reduced portfolio risk, and 4 diagnostic analysis scripts. Phase A: Research mode kill-switch reset, risk reduction (0.5%/trade, max 4 positions), bullish-only direction for all event types. Phase B: 2 new sub-scorers (parse_confidence, direction_clarity), 4 veto gates (oneoff risk, parse confidence, unknown/bearish direction). Phase C: signal_quality, event_type_decomposition, kill_switch_impact, concurrent_position analysis scripts. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>main
I Luk Kim
5 months ago
parent
cdf6ae3493
commit
4d0e773ba0
@ -0,0 +1,169 @@
|
|||||||
|
"""Concurrent Position Analysis: Position clustering and correlation.
|
||||||
|
|
||||||
|
Are simultaneous positions correlated? Analyzes trade blotter to find
|
||||||
|
overlapping position groups and their sector distribution.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
python -m dev.analysis.concurrent_position_analysis \
|
||||||
|
--run-dir runs/run_20260313_xyz [--output-dir ./data/analysis]
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import csv
|
||||||
|
import datetime as dt
|
||||||
|
from collections import Counter
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import pyarrow.parquet as pq
|
||||||
|
|
||||||
|
|
||||||
|
def _load_blotter(run_dir: Path) -> list[dict[str, Any]]:
|
||||||
|
"""Load trade blotter from parquet."""
|
||||||
|
blotter_path = run_dir / "trade_blotter.parquet"
|
||||||
|
if not blotter_path.exists():
|
||||||
|
# Try CSV fallback
|
||||||
|
csv_path = run_dir / "trade_blotter.csv"
|
||||||
|
if csv_path.exists():
|
||||||
|
rows = []
|
||||||
|
with open(csv_path) as f:
|
||||||
|
reader = csv.DictReader(f)
|
||||||
|
for row in reader:
|
||||||
|
rows.append(row)
|
||||||
|
return rows
|
||||||
|
raise FileNotFoundError(f"No trade blotter found in {run_dir}")
|
||||||
|
|
||||||
|
table = pq.read_table(blotter_path)
|
||||||
|
return table.to_pylist()
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_date(val: Any) -> dt.date | None:
|
||||||
|
"""Parse a date value from various formats."""
|
||||||
|
if val is None:
|
||||||
|
return None
|
||||||
|
if isinstance(val, dt.date):
|
||||||
|
return val
|
||||||
|
if isinstance(val, dt.datetime):
|
||||||
|
return val.date()
|
||||||
|
try:
|
||||||
|
return dt.date.fromisoformat(str(val)[:10])
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def analyze(trades: list[dict[str, Any]], output_dir: Path) -> None:
|
||||||
|
output_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
# Parse entry/exit dates
|
||||||
|
positions: list[dict[str, Any]] = []
|
||||||
|
for t in trades:
|
||||||
|
entry = _parse_date(t.get("entry_date"))
|
||||||
|
exit_ = _parse_date(t.get("exit_date"))
|
||||||
|
if entry and exit_:
|
||||||
|
positions.append({
|
||||||
|
"trade_id": t.get("trade_id", ""),
|
||||||
|
"symbol": t.get("symbol", ""),
|
||||||
|
"sector": t.get("sector", "UNKNOWN"),
|
||||||
|
"entry_date": entry,
|
||||||
|
"exit_date": exit_,
|
||||||
|
})
|
||||||
|
|
||||||
|
if not positions:
|
||||||
|
print("No valid trades found.")
|
||||||
|
return
|
||||||
|
|
||||||
|
# Build daily position timeline
|
||||||
|
all_dates = set()
|
||||||
|
for p in positions:
|
||||||
|
d = p["entry_date"]
|
||||||
|
while d <= p["exit_date"]:
|
||||||
|
all_dates.add(d)
|
||||||
|
d += dt.timedelta(days=1)
|
||||||
|
|
||||||
|
daily_positions: dict[dt.date, list[dict[str, Any]]] = {}
|
||||||
|
for d in sorted(all_dates):
|
||||||
|
active = [p for p in positions if p["entry_date"] <= d <= p["exit_date"]]
|
||||||
|
if active:
|
||||||
|
daily_positions[d] = active
|
||||||
|
|
||||||
|
# Compute concurrent position stats
|
||||||
|
concurrent_counts = [len(v) for v in daily_positions.values()]
|
||||||
|
max_concurrent = max(concurrent_counts) if concurrent_counts else 0
|
||||||
|
avg_concurrent = sum(concurrent_counts) / len(concurrent_counts) if concurrent_counts else 0
|
||||||
|
|
||||||
|
# Find overlap groups (days with >1 position)
|
||||||
|
overlap_days = {d: ps for d, ps in daily_positions.items() if len(ps) > 1}
|
||||||
|
|
||||||
|
# Sector distribution in overlap groups
|
||||||
|
overlap_sector_counts: Counter[str] = Counter()
|
||||||
|
overlap_symbol_counts: Counter[str] = Counter()
|
||||||
|
for d, ps in overlap_days.items():
|
||||||
|
for p in ps:
|
||||||
|
overlap_sector_counts[p["sector"]] += 1
|
||||||
|
overlap_symbol_counts[p["symbol"]] += 1
|
||||||
|
|
||||||
|
# Peak overlap days
|
||||||
|
peak_days = sorted(overlap_days.items(), key=lambda x: len(x[1]), reverse=True)[:10]
|
||||||
|
|
||||||
|
# CSV: daily concurrent position counts
|
||||||
|
csv_rows: list[dict[str, Any]] = []
|
||||||
|
for d in sorted(daily_positions):
|
||||||
|
ps = daily_positions[d]
|
||||||
|
sectors = Counter(p["sector"] for p in ps)
|
||||||
|
csv_rows.append({
|
||||||
|
"date": str(d),
|
||||||
|
"concurrent_positions": len(ps),
|
||||||
|
"unique_sectors": len(sectors),
|
||||||
|
"sectors": "|".join(f"{s}:{c}" for s, c in sectors.most_common()),
|
||||||
|
"symbols": "|".join(p["symbol"] for p in ps),
|
||||||
|
})
|
||||||
|
|
||||||
|
csv_path = output_dir / "concurrent_position_analysis.csv"
|
||||||
|
if csv_rows:
|
||||||
|
fieldnames = list(csv_rows[0].keys())
|
||||||
|
with open(csv_path, "w", newline="") as f:
|
||||||
|
writer = csv.DictWriter(f, fieldnames=fieldnames)
|
||||||
|
writer.writeheader()
|
||||||
|
writer.writerows(csv_rows)
|
||||||
|
|
||||||
|
# Console output
|
||||||
|
print(f"\n{'='*70}")
|
||||||
|
print(f"Concurrent Position Analysis — {len(positions)} trades")
|
||||||
|
print(f"{'='*70}")
|
||||||
|
print(f"\n Total trading days with positions: {len(daily_positions)}")
|
||||||
|
print(f" Days with >1 concurrent position: {len(overlap_days)}")
|
||||||
|
print(f" Max concurrent positions: {max_concurrent}")
|
||||||
|
print(f" Avg concurrent positions: {avg_concurrent:.1f}")
|
||||||
|
|
||||||
|
if overlap_sector_counts:
|
||||||
|
print(f"\n Sector distribution in overlap periods:")
|
||||||
|
for sector, count in overlap_sector_counts.most_common():
|
||||||
|
print(f" {sector:<20} {count:>5} position-days")
|
||||||
|
|
||||||
|
if peak_days:
|
||||||
|
print(f"\n Top 10 peak overlap days:")
|
||||||
|
for d, ps in peak_days:
|
||||||
|
symbols = ", ".join(p["symbol"] for p in ps)
|
||||||
|
sectors = set(p["sector"] for p in ps)
|
||||||
|
print(f" {d} {len(ps)} positions sectors={len(sectors)} [{symbols}]")
|
||||||
|
|
||||||
|
print(f"\nCSV written to: {csv_path}")
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
parser = argparse.ArgumentParser(description="Concurrent Position Analysis")
|
||||||
|
parser.add_argument("--run-dir", required=True, help="Path to backtest run directory")
|
||||||
|
parser.add_argument("--output-dir", default="./data/analysis", help="Output directory")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
trades = _load_blotter(Path(args.run_dir))
|
||||||
|
if not trades:
|
||||||
|
print("No trades found in blotter.")
|
||||||
|
return
|
||||||
|
|
||||||
|
analyze(trades, Path(args.output_dir))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@ -0,0 +1,122 @@
|
|||||||
|
"""Event Type Decomposition: Per-event-type performance analysis.
|
||||||
|
|
||||||
|
Which event types contribute alpha?
|
||||||
|
|
||||||
|
Groups snapshot data by event_type and computes forward return statistics,
|
||||||
|
win rates, and average composite scores per type.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
python -m dev.analysis.event_type_decomposition \
|
||||||
|
--snapshot-dir data/datasets/snapshots/snapshot_2026_03_20 \
|
||||||
|
--split train [--output-dir ./data/analysis]
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import csv
|
||||||
|
import statistics
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import pyarrow.parquet as pq
|
||||||
|
|
||||||
|
from libs.backtest.scoring import compute_entry_score
|
||||||
|
|
||||||
|
|
||||||
|
def _load_rows(snapshot_dir: Path, split: str) -> list[dict[str, Any]]:
|
||||||
|
parquet_path = snapshot_dir / f"{split}.parquet"
|
||||||
|
table = pq.read_table(parquet_path)
|
||||||
|
return table.to_pylist()
|
||||||
|
|
||||||
|
|
||||||
|
FWD_HORIZONS = ["fwd_return_1d", "fwd_return_3d", "fwd_return_5d", "fwd_return_10d"]
|
||||||
|
|
||||||
|
|
||||||
|
def analyze(rows: list[dict[str, Any]], output_dir: Path) -> None:
|
||||||
|
output_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
# Compute scores
|
||||||
|
for row in rows:
|
||||||
|
row["_composite_score"] = compute_entry_score(row)
|
||||||
|
|
||||||
|
# Group by event_type
|
||||||
|
by_type: dict[str, list[dict[str, Any]]] = {}
|
||||||
|
for row in rows:
|
||||||
|
et = row.get("event_type", "unknown")
|
||||||
|
by_type.setdefault(et, []).append(row)
|
||||||
|
|
||||||
|
csv_rows: list[dict[str, Any]] = []
|
||||||
|
for event_type in sorted(by_type):
|
||||||
|
items = by_type[event_type]
|
||||||
|
n = len(items)
|
||||||
|
row_out: dict[str, Any] = {"event_type": event_type, "count": n}
|
||||||
|
|
||||||
|
# Average composite score
|
||||||
|
scores = [r["_composite_score"] for r in items]
|
||||||
|
row_out["avg_score"] = statistics.mean(scores) if scores else None
|
||||||
|
|
||||||
|
for horizon in FWD_HORIZONS:
|
||||||
|
vals = [float(r[horizon]) for r in items if r.get(horizon) is not None]
|
||||||
|
if vals:
|
||||||
|
row_out[f"{horizon}_mean"] = statistics.mean(vals)
|
||||||
|
row_out[f"{horizon}_win_rate"] = sum(1 for v in vals if v > 0) / len(vals)
|
||||||
|
row_out[f"{horizon}_std"] = statistics.stdev(vals) if len(vals) > 1 else 0.0
|
||||||
|
else:
|
||||||
|
row_out[f"{horizon}_mean"] = None
|
||||||
|
row_out[f"{horizon}_win_rate"] = None
|
||||||
|
row_out[f"{horizon}_std"] = None
|
||||||
|
|
||||||
|
csv_rows.append(row_out)
|
||||||
|
|
||||||
|
# Write CSV
|
||||||
|
csv_path = output_dir / "event_type_decomposition.csv"
|
||||||
|
if csv_rows:
|
||||||
|
fieldnames = list(csv_rows[0].keys())
|
||||||
|
with open(csv_path, "w", newline="") as f:
|
||||||
|
writer = csv.DictWriter(f, fieldnames=fieldnames)
|
||||||
|
writer.writeheader()
|
||||||
|
writer.writerows(csv_rows)
|
||||||
|
|
||||||
|
# Console output
|
||||||
|
print(f"\n{'='*90}")
|
||||||
|
print(f"Event Type Decomposition — {len(rows)} events, {len(by_type)} types")
|
||||||
|
print(f"{'='*90}")
|
||||||
|
print(f"\n{'Type':<22} {'N':>5} {'AvgScore':>8}", end="")
|
||||||
|
for h in FWD_HORIZONS:
|
||||||
|
label = h.replace("fwd_return_", "")
|
||||||
|
print(f" {label+'_mean':>9} {label+'_wr':>7}", end="")
|
||||||
|
print()
|
||||||
|
print("-" * 90)
|
||||||
|
|
||||||
|
for row_out in csv_rows:
|
||||||
|
avg_s = row_out["avg_score"]
|
||||||
|
print(f"{row_out['event_type']:<22} {row_out['count']:>5} {avg_s:.3f}" if avg_s else
|
||||||
|
f"{row_out['event_type']:<22} {row_out['count']:>5} N/A", end="")
|
||||||
|
for h in FWD_HORIZONS:
|
||||||
|
mean_val = row_out[f"{h}_mean"]
|
||||||
|
wr_val = row_out[f"{h}_win_rate"]
|
||||||
|
mean_str = f"{mean_val:+.4f}" if mean_val is not None else " N/A"
|
||||||
|
wr_str = f"{wr_val:.1%}" if wr_val is not None else " N/A"
|
||||||
|
print(f" {mean_str:>9} {wr_str:>7}", end="")
|
||||||
|
print()
|
||||||
|
|
||||||
|
print(f"\nCSV written to: {csv_path}")
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
parser = argparse.ArgumentParser(description="Event Type Decomposition Analysis")
|
||||||
|
parser.add_argument("--snapshot-dir", required=True, help="Path to snapshot directory")
|
||||||
|
parser.add_argument("--split", default="train", help="Split name (train/valid/test)")
|
||||||
|
parser.add_argument("--output-dir", default="./data/analysis", help="Output directory")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
rows = _load_rows(Path(args.snapshot_dir), args.split)
|
||||||
|
if not rows:
|
||||||
|
print("No data found.")
|
||||||
|
return
|
||||||
|
|
||||||
|
analyze(rows, Path(args.output_dir))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@ -0,0 +1,122 @@
|
|||||||
|
"""Kill Switch Impact Analysis: Compare live vs research mode.
|
||||||
|
|
||||||
|
How much does the permanent kill switch hide?
|
||||||
|
|
||||||
|
Runs the backtest twice (live mode and research mode) and compares
|
||||||
|
key metrics side-by-side.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
python -m dev.analysis.kill_switch_impact_analysis \
|
||||||
|
--manifest configs/experiments/phase5_v1.json \
|
||||||
|
--snapshot-dir data/datasets/snapshots \
|
||||||
|
--split train [--output-root ./runs]
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from libs.backtest.manifests import load_manifest, resolve_config
|
||||||
|
from libs.backtest.domain import MetricsBundle
|
||||||
|
|
||||||
|
|
||||||
|
def _run_backtest(
|
||||||
|
manifest_path: str,
|
||||||
|
snapshot_dir: str | None,
|
||||||
|
split: str,
|
||||||
|
mode: str,
|
||||||
|
output_root: str,
|
||||||
|
initial_equity: float,
|
||||||
|
config_root: str,
|
||||||
|
) -> tuple[str, MetricsBundle, int]:
|
||||||
|
"""Run a single backtest and return (run_id, metrics, trade_count)."""
|
||||||
|
from apps.backtester.run import BacktestRunner, _build_store
|
||||||
|
|
||||||
|
manifest = load_manifest(manifest_path)
|
||||||
|
config = resolve_config(manifest, config_root=config_root)
|
||||||
|
config.risk.backtest_mode = mode
|
||||||
|
|
||||||
|
store = _build_store(manifest, config, split, snapshot_dir_override=snapshot_dir)
|
||||||
|
runner = BacktestRunner(
|
||||||
|
manifest=manifest,
|
||||||
|
config=config,
|
||||||
|
store=store,
|
||||||
|
initial_equity=initial_equity,
|
||||||
|
)
|
||||||
|
result = runner.run(output_root=output_root)
|
||||||
|
return result.run_id, result.metrics, result.metrics.trade_count
|
||||||
|
|
||||||
|
|
||||||
|
def _fmt(val: float | None, fmt: str = ".2f") -> str:
|
||||||
|
if val is None:
|
||||||
|
return "N/A"
|
||||||
|
return f"{val:{fmt}}"
|
||||||
|
|
||||||
|
|
||||||
|
def _fmt_pct(val: float | None) -> str:
|
||||||
|
if val is None:
|
||||||
|
return "N/A"
|
||||||
|
return f"{val:.2f}%"
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
parser = argparse.ArgumentParser(description="Kill Switch Impact Analysis")
|
||||||
|
parser.add_argument("--manifest", required=True, help="Path to experiment manifest JSON")
|
||||||
|
parser.add_argument("--snapshot-dir", help="Override snapshot root directory")
|
||||||
|
parser.add_argument("--split", default="train", help="Split name (train/valid/test)")
|
||||||
|
parser.add_argument("--output-root", default="./runs", help="Output root directory")
|
||||||
|
parser.add_argument("--initial-equity", type=float, default=100_000.0)
|
||||||
|
parser.add_argument("--config-root", default=".", help="Root dir for resolving config paths")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
print("Running LIVE mode backtest...")
|
||||||
|
live_id, live_m, live_trades = _run_backtest(
|
||||||
|
args.manifest, args.snapshot_dir, args.split, "live",
|
||||||
|
args.output_root, args.initial_equity, args.config_root,
|
||||||
|
)
|
||||||
|
|
||||||
|
print("Running RESEARCH mode backtest...")
|
||||||
|
research_id, research_m, research_trades = _run_backtest(
|
||||||
|
args.manifest, args.snapshot_dir, args.split, "research",
|
||||||
|
args.output_root, args.initial_equity, args.config_root,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Side-by-side comparison
|
||||||
|
print(f"\n{'='*60}")
|
||||||
|
print(f"Kill Switch Impact Analysis — split={args.split}")
|
||||||
|
print(f"{'='*60}")
|
||||||
|
print(f"\n{'Metric':<28} {'Live':>14} {'Research':>14}")
|
||||||
|
print("-" * 60)
|
||||||
|
|
||||||
|
rows = [
|
||||||
|
("Run ID", live_id[:16], research_id[:16]),
|
||||||
|
("Total Trades", str(live_trades), str(research_trades)),
|
||||||
|
("Win Rate", _fmt_pct(live_m.win_rate and live_m.win_rate * 100),
|
||||||
|
_fmt_pct(research_m.win_rate and research_m.win_rate * 100)),
|
||||||
|
("Total Return", _fmt_pct(live_m.total_return_pct),
|
||||||
|
_fmt_pct(research_m.total_return_pct)),
|
||||||
|
("Max Drawdown", _fmt_pct(live_m.max_drawdown_pct),
|
||||||
|
_fmt_pct(research_m.max_drawdown_pct)),
|
||||||
|
("Sharpe Ratio", _fmt(live_m.sharpe_ratio), _fmt(research_m.sharpe_ratio)),
|
||||||
|
("Sortino Ratio", _fmt(live_m.sortino_ratio), _fmt(research_m.sortino_ratio)),
|
||||||
|
("Profit Factor", _fmt(live_m.profit_factor), _fmt(research_m.profit_factor)),
|
||||||
|
("Expectancy (R)", _fmt(live_m.expectancy_r), _fmt(research_m.expectancy_r)),
|
||||||
|
("Avg Holding Days", _fmt(live_m.avg_holding_days, ".1f"),
|
||||||
|
_fmt(research_m.avg_holding_days, ".1f")),
|
||||||
|
("Avg Positions Held", _fmt(live_m.avg_positions_held, ".1f"),
|
||||||
|
_fmt(research_m.avg_positions_held, ".1f")),
|
||||||
|
]
|
||||||
|
|
||||||
|
for label, live_val, research_val in rows:
|
||||||
|
print(f"{label:<28} {live_val:>14} {research_val:>14}")
|
||||||
|
|
||||||
|
# Delta summary
|
||||||
|
if live_m.total_return_pct is not None and research_m.total_return_pct is not None:
|
||||||
|
delta_return = research_m.total_return_pct - live_m.total_return_pct
|
||||||
|
delta_trades = research_trades - live_trades
|
||||||
|
print(f"\n Delta trades: {delta_trades:+d}")
|
||||||
|
print(f" Delta return: {delta_return:+.2f}%")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@ -0,0 +1,170 @@
|
|||||||
|
"""Signal Quality Analysis: Score-bucket monotonicity check.
|
||||||
|
|
||||||
|
Does higher composite score predict better forward returns?
|
||||||
|
|
||||||
|
Reads snapshot parquet, computes compute_entry_score for each row,
|
||||||
|
buckets into quintiles, and checks Spearman rank correlation between
|
||||||
|
score bucket and mean forward return.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
python -m dev.analysis.signal_quality_analysis \
|
||||||
|
--snapshot-dir data/datasets/snapshots/snapshot_2026_03_20 \
|
||||||
|
--split train [--output-dir ./data/analysis]
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import csv
|
||||||
|
import statistics
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import pyarrow.parquet as pq
|
||||||
|
|
||||||
|
from libs.backtest.scoring import compute_entry_score
|
||||||
|
|
||||||
|
|
||||||
|
def _load_rows(snapshot_dir: Path, split: str) -> list[dict[str, Any]]:
|
||||||
|
"""Load parquet snapshot into list of dicts."""
|
||||||
|
parquet_path = snapshot_dir / f"{split}.parquet"
|
||||||
|
table = pq.read_table(parquet_path)
|
||||||
|
return table.to_pylist()
|
||||||
|
|
||||||
|
|
||||||
|
def _bucket_label(score: float) -> str:
|
||||||
|
"""Assign score to a quintile bucket."""
|
||||||
|
if score < 0.2:
|
||||||
|
return "0.0-0.2"
|
||||||
|
if score < 0.4:
|
||||||
|
return "0.2-0.4"
|
||||||
|
if score < 0.6:
|
||||||
|
return "0.4-0.6"
|
||||||
|
if score < 0.8:
|
||||||
|
return "0.6-0.8"
|
||||||
|
return "0.8-1.0"
|
||||||
|
|
||||||
|
|
||||||
|
BUCKET_ORDER = ["0.0-0.2", "0.2-0.4", "0.4-0.6", "0.6-0.8", "0.8-1.0"]
|
||||||
|
FWD_HORIZONS = ["fwd_return_1d", "fwd_return_3d", "fwd_return_5d", "fwd_return_10d"]
|
||||||
|
|
||||||
|
|
||||||
|
def analyze(rows: list[dict[str, Any]], output_dir: Path) -> None:
|
||||||
|
"""Score all rows, bucket, compute statistics, write output."""
|
||||||
|
output_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
# Score each row and assign bucket
|
||||||
|
scored: list[dict[str, Any]] = []
|
||||||
|
for row in rows:
|
||||||
|
score = compute_entry_score(row)
|
||||||
|
bucket = _bucket_label(score)
|
||||||
|
scored.append({"score": score, "bucket": bucket, **row})
|
||||||
|
|
||||||
|
# Group by bucket
|
||||||
|
by_bucket: dict[str, list[dict[str, Any]]] = {b: [] for b in BUCKET_ORDER}
|
||||||
|
for s in scored:
|
||||||
|
by_bucket[s["bucket"]].append(s)
|
||||||
|
|
||||||
|
# Compute per-bucket stats
|
||||||
|
csv_rows: list[dict[str, Any]] = []
|
||||||
|
for bucket in BUCKET_ORDER:
|
||||||
|
items = by_bucket[bucket]
|
||||||
|
n = len(items)
|
||||||
|
row_out: dict[str, Any] = {"bucket": bucket, "count": n}
|
||||||
|
|
||||||
|
for horizon in FWD_HORIZONS:
|
||||||
|
vals = [float(r[horizon]) for r in items if r.get(horizon) is not None]
|
||||||
|
if vals:
|
||||||
|
row_out[f"{horizon}_mean"] = statistics.mean(vals)
|
||||||
|
row_out[f"{horizon}_win_rate"] = sum(1 for v in vals if v > 0) / len(vals)
|
||||||
|
else:
|
||||||
|
row_out[f"{horizon}_mean"] = None
|
||||||
|
row_out[f"{horizon}_win_rate"] = None
|
||||||
|
|
||||||
|
csv_rows.append(row_out)
|
||||||
|
|
||||||
|
# Spearman rank correlation (bucket rank vs mean return)
|
||||||
|
spearman_results: dict[str, float | None] = {}
|
||||||
|
for horizon in FWD_HORIZONS:
|
||||||
|
means = [r[f"{horizon}_mean"] for r in csv_rows if r[f"{horizon}_mean"] is not None]
|
||||||
|
if len(means) >= 3:
|
||||||
|
# Rank correlation: do the means increase with bucket rank?
|
||||||
|
ranks_score = list(range(len(means)))
|
||||||
|
ranks_return = _rank(means)
|
||||||
|
spearman_results[horizon] = _spearman(ranks_score, ranks_return)
|
||||||
|
else:
|
||||||
|
spearman_results[horizon] = None
|
||||||
|
|
||||||
|
# Write CSV
|
||||||
|
csv_path = output_dir / "signal_quality_analysis.csv"
|
||||||
|
if csv_rows:
|
||||||
|
fieldnames = list(csv_rows[0].keys())
|
||||||
|
with open(csv_path, "w", newline="") as f:
|
||||||
|
writer = csv.DictWriter(f, fieldnames=fieldnames)
|
||||||
|
writer.writeheader()
|
||||||
|
writer.writerows(csv_rows)
|
||||||
|
|
||||||
|
# Console output
|
||||||
|
print(f"\n{'='*80}")
|
||||||
|
print(f"Signal Quality Analysis — {len(scored)} events")
|
||||||
|
print(f"{'='*80}")
|
||||||
|
print(f"\n{'Bucket':<12} {'Count':>6}", end="")
|
||||||
|
for h in FWD_HORIZONS:
|
||||||
|
label = h.replace("fwd_return_", "")
|
||||||
|
print(f" {label+'_mean':>10} {label+'_wr':>8}", end="")
|
||||||
|
print()
|
||||||
|
print("-" * 80)
|
||||||
|
|
||||||
|
for row_out in csv_rows:
|
||||||
|
print(f"{row_out['bucket']:<12} {row_out['count']:>6}", end="")
|
||||||
|
for h in FWD_HORIZONS:
|
||||||
|
mean_val = row_out[f"{h}_mean"]
|
||||||
|
wr_val = row_out[f"{h}_win_rate"]
|
||||||
|
mean_str = f"{mean_val:+.4f}" if mean_val is not None else " N/A"
|
||||||
|
wr_str = f"{wr_val:.1%}" if wr_val is not None else " N/A"
|
||||||
|
print(f" {mean_str:>10} {wr_str:>8}", end="")
|
||||||
|
print()
|
||||||
|
|
||||||
|
print(f"\nSpearman rank correlation (bucket vs mean return):")
|
||||||
|
for h in FWD_HORIZONS:
|
||||||
|
label = h.replace("fwd_return_", "")
|
||||||
|
val = spearman_results.get(h)
|
||||||
|
print(f" {label}: {val:+.3f}" if val is not None else f" {label}: N/A")
|
||||||
|
|
||||||
|
print(f"\nCSV written to: {csv_path}")
|
||||||
|
|
||||||
|
|
||||||
|
def _rank(values: list[float]) -> list[float]:
|
||||||
|
"""Compute ranks (0-indexed) for a list of values."""
|
||||||
|
indexed = sorted(enumerate(values), key=lambda x: x[1])
|
||||||
|
ranks = [0.0] * len(values)
|
||||||
|
for rank, (idx, _) in enumerate(indexed):
|
||||||
|
ranks[idx] = float(rank)
|
||||||
|
return ranks
|
||||||
|
|
||||||
|
|
||||||
|
def _spearman(x: list[float], y: list[float]) -> float:
|
||||||
|
"""Compute Spearman rank correlation coefficient."""
|
||||||
|
n = len(x)
|
||||||
|
if n < 2:
|
||||||
|
return 0.0
|
||||||
|
d_sq = sum((xi - yi) ** 2 for xi, yi in zip(x, y))
|
||||||
|
return 1 - (6 * d_sq) / (n * (n**2 - 1))
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
parser = argparse.ArgumentParser(description="Signal Quality Analysis")
|
||||||
|
parser.add_argument("--snapshot-dir", required=True, help="Path to snapshot directory")
|
||||||
|
parser.add_argument("--split", default="train", help="Split name (train/valid/test)")
|
||||||
|
parser.add_argument("--output-dir", default="./data/analysis", help="Output directory")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
rows = _load_rows(Path(args.snapshot_dir), args.split)
|
||||||
|
if not rows:
|
||||||
|
print("No data found.")
|
||||||
|
return
|
||||||
|
|
||||||
|
analyze(rows, Path(args.output_dir))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Loading…
Reference in New Issue