feat: overhaul strategy — document quality > price momentum
Flip scoring weights so event/document quality is primary signal (55%) and market confirmation is secondary (35%). Add research mode with kill-switch cooldown/reset, veto gates for bad events, reduced portfolio risk, and 4 diagnostic analysis scripts. Phase A: Research mode kill-switch reset, risk reduction (0.5%/trade, max 4 positions), bullish-only direction for all event types. Phase B: 2 new sub-scorers (parse_confidence, direction_clarity), 4 veto gates (oneoff risk, parse confidence, unknown/bearish direction). Phase C: signal_quality, event_type_decomposition, kill_switch_impact, concurrent_position analysis scripts. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>main
I Luk Kim
5 months ago
parent
cdf6ae3493
commit
4d0e773ba0
@ -0,0 +1,169 @@
|
||||
"""Concurrent Position Analysis: Position clustering and correlation.
|
||||
|
||||
Are simultaneous positions correlated? Analyzes trade blotter to find
|
||||
overlapping position groups and their sector distribution.
|
||||
|
||||
Usage:
|
||||
python -m dev.analysis.concurrent_position_analysis \
|
||||
--run-dir runs/run_20260313_xyz [--output-dir ./data/analysis]
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import datetime as dt
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pyarrow.parquet as pq
|
||||
|
||||
|
||||
def _load_blotter(run_dir: Path) -> list[dict[str, Any]]:
|
||||
"""Load trade blotter from parquet."""
|
||||
blotter_path = run_dir / "trade_blotter.parquet"
|
||||
if not blotter_path.exists():
|
||||
# Try CSV fallback
|
||||
csv_path = run_dir / "trade_blotter.csv"
|
||||
if csv_path.exists():
|
||||
rows = []
|
||||
with open(csv_path) as f:
|
||||
reader = csv.DictReader(f)
|
||||
for row in reader:
|
||||
rows.append(row)
|
||||
return rows
|
||||
raise FileNotFoundError(f"No trade blotter found in {run_dir}")
|
||||
|
||||
table = pq.read_table(blotter_path)
|
||||
return table.to_pylist()
|
||||
|
||||
|
||||
def _parse_date(val: Any) -> dt.date | None:
|
||||
"""Parse a date value from various formats."""
|
||||
if val is None:
|
||||
return None
|
||||
if isinstance(val, dt.date):
|
||||
return val
|
||||
if isinstance(val, dt.datetime):
|
||||
return val.date()
|
||||
try:
|
||||
return dt.date.fromisoformat(str(val)[:10])
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def analyze(trades: list[dict[str, Any]], output_dir: Path) -> None:
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Parse entry/exit dates
|
||||
positions: list[dict[str, Any]] = []
|
||||
for t in trades:
|
||||
entry = _parse_date(t.get("entry_date"))
|
||||
exit_ = _parse_date(t.get("exit_date"))
|
||||
if entry and exit_:
|
||||
positions.append({
|
||||
"trade_id": t.get("trade_id", ""),
|
||||
"symbol": t.get("symbol", ""),
|
||||
"sector": t.get("sector", "UNKNOWN"),
|
||||
"entry_date": entry,
|
||||
"exit_date": exit_,
|
||||
})
|
||||
|
||||
if not positions:
|
||||
print("No valid trades found.")
|
||||
return
|
||||
|
||||
# Build daily position timeline
|
||||
all_dates = set()
|
||||
for p in positions:
|
||||
d = p["entry_date"]
|
||||
while d <= p["exit_date"]:
|
||||
all_dates.add(d)
|
||||
d += dt.timedelta(days=1)
|
||||
|
||||
daily_positions: dict[dt.date, list[dict[str, Any]]] = {}
|
||||
for d in sorted(all_dates):
|
||||
active = [p for p in positions if p["entry_date"] <= d <= p["exit_date"]]
|
||||
if active:
|
||||
daily_positions[d] = active
|
||||
|
||||
# Compute concurrent position stats
|
||||
concurrent_counts = [len(v) for v in daily_positions.values()]
|
||||
max_concurrent = max(concurrent_counts) if concurrent_counts else 0
|
||||
avg_concurrent = sum(concurrent_counts) / len(concurrent_counts) if concurrent_counts else 0
|
||||
|
||||
# Find overlap groups (days with >1 position)
|
||||
overlap_days = {d: ps for d, ps in daily_positions.items() if len(ps) > 1}
|
||||
|
||||
# Sector distribution in overlap groups
|
||||
overlap_sector_counts: Counter[str] = Counter()
|
||||
overlap_symbol_counts: Counter[str] = Counter()
|
||||
for d, ps in overlap_days.items():
|
||||
for p in ps:
|
||||
overlap_sector_counts[p["sector"]] += 1
|
||||
overlap_symbol_counts[p["symbol"]] += 1
|
||||
|
||||
# Peak overlap days
|
||||
peak_days = sorted(overlap_days.items(), key=lambda x: len(x[1]), reverse=True)[:10]
|
||||
|
||||
# CSV: daily concurrent position counts
|
||||
csv_rows: list[dict[str, Any]] = []
|
||||
for d in sorted(daily_positions):
|
||||
ps = daily_positions[d]
|
||||
sectors = Counter(p["sector"] for p in ps)
|
||||
csv_rows.append({
|
||||
"date": str(d),
|
||||
"concurrent_positions": len(ps),
|
||||
"unique_sectors": len(sectors),
|
||||
"sectors": "|".join(f"{s}:{c}" for s, c in sectors.most_common()),
|
||||
"symbols": "|".join(p["symbol"] for p in ps),
|
||||
})
|
||||
|
||||
csv_path = output_dir / "concurrent_position_analysis.csv"
|
||||
if csv_rows:
|
||||
fieldnames = list(csv_rows[0].keys())
|
||||
with open(csv_path, "w", newline="") as f:
|
||||
writer = csv.DictWriter(f, fieldnames=fieldnames)
|
||||
writer.writeheader()
|
||||
writer.writerows(csv_rows)
|
||||
|
||||
# Console output
|
||||
print(f"\n{'='*70}")
|
||||
print(f"Concurrent Position Analysis — {len(positions)} trades")
|
||||
print(f"{'='*70}")
|
||||
print(f"\n Total trading days with positions: {len(daily_positions)}")
|
||||
print(f" Days with >1 concurrent position: {len(overlap_days)}")
|
||||
print(f" Max concurrent positions: {max_concurrent}")
|
||||
print(f" Avg concurrent positions: {avg_concurrent:.1f}")
|
||||
|
||||
if overlap_sector_counts:
|
||||
print(f"\n Sector distribution in overlap periods:")
|
||||
for sector, count in overlap_sector_counts.most_common():
|
||||
print(f" {sector:<20} {count:>5} position-days")
|
||||
|
||||
if peak_days:
|
||||
print(f"\n Top 10 peak overlap days:")
|
||||
for d, ps in peak_days:
|
||||
symbols = ", ".join(p["symbol"] for p in ps)
|
||||
sectors = set(p["sector"] for p in ps)
|
||||
print(f" {d} {len(ps)} positions sectors={len(sectors)} [{symbols}]")
|
||||
|
||||
print(f"\nCSV written to: {csv_path}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Concurrent Position Analysis")
|
||||
parser.add_argument("--run-dir", required=True, help="Path to backtest run directory")
|
||||
parser.add_argument("--output-dir", default="./data/analysis", help="Output directory")
|
||||
args = parser.parse_args()
|
||||
|
||||
trades = _load_blotter(Path(args.run_dir))
|
||||
if not trades:
|
||||
print("No trades found in blotter.")
|
||||
return
|
||||
|
||||
analyze(trades, Path(args.output_dir))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -0,0 +1,122 @@
|
||||
"""Event Type Decomposition: Per-event-type performance analysis.
|
||||
|
||||
Which event types contribute alpha?
|
||||
|
||||
Groups snapshot data by event_type and computes forward return statistics,
|
||||
win rates, and average composite scores per type.
|
||||
|
||||
Usage:
|
||||
python -m dev.analysis.event_type_decomposition \
|
||||
--snapshot-dir data/datasets/snapshots/snapshot_2026_03_20 \
|
||||
--split train [--output-dir ./data/analysis]
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import statistics
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pyarrow.parquet as pq
|
||||
|
||||
from libs.backtest.scoring import compute_entry_score
|
||||
|
||||
|
||||
def _load_rows(snapshot_dir: Path, split: str) -> list[dict[str, Any]]:
|
||||
parquet_path = snapshot_dir / f"{split}.parquet"
|
||||
table = pq.read_table(parquet_path)
|
||||
return table.to_pylist()
|
||||
|
||||
|
||||
FWD_HORIZONS = ["fwd_return_1d", "fwd_return_3d", "fwd_return_5d", "fwd_return_10d"]
|
||||
|
||||
|
||||
def analyze(rows: list[dict[str, Any]], output_dir: Path) -> None:
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Compute scores
|
||||
for row in rows:
|
||||
row["_composite_score"] = compute_entry_score(row)
|
||||
|
||||
# Group by event_type
|
||||
by_type: dict[str, list[dict[str, Any]]] = {}
|
||||
for row in rows:
|
||||
et = row.get("event_type", "unknown")
|
||||
by_type.setdefault(et, []).append(row)
|
||||
|
||||
csv_rows: list[dict[str, Any]] = []
|
||||
for event_type in sorted(by_type):
|
||||
items = by_type[event_type]
|
||||
n = len(items)
|
||||
row_out: dict[str, Any] = {"event_type": event_type, "count": n}
|
||||
|
||||
# Average composite score
|
||||
scores = [r["_composite_score"] for r in items]
|
||||
row_out["avg_score"] = statistics.mean(scores) if scores else None
|
||||
|
||||
for horizon in FWD_HORIZONS:
|
||||
vals = [float(r[horizon]) for r in items if r.get(horizon) is not None]
|
||||
if vals:
|
||||
row_out[f"{horizon}_mean"] = statistics.mean(vals)
|
||||
row_out[f"{horizon}_win_rate"] = sum(1 for v in vals if v > 0) / len(vals)
|
||||
row_out[f"{horizon}_std"] = statistics.stdev(vals) if len(vals) > 1 else 0.0
|
||||
else:
|
||||
row_out[f"{horizon}_mean"] = None
|
||||
row_out[f"{horizon}_win_rate"] = None
|
||||
row_out[f"{horizon}_std"] = None
|
||||
|
||||
csv_rows.append(row_out)
|
||||
|
||||
# Write CSV
|
||||
csv_path = output_dir / "event_type_decomposition.csv"
|
||||
if csv_rows:
|
||||
fieldnames = list(csv_rows[0].keys())
|
||||
with open(csv_path, "w", newline="") as f:
|
||||
writer = csv.DictWriter(f, fieldnames=fieldnames)
|
||||
writer.writeheader()
|
||||
writer.writerows(csv_rows)
|
||||
|
||||
# Console output
|
||||
print(f"\n{'='*90}")
|
||||
print(f"Event Type Decomposition — {len(rows)} events, {len(by_type)} types")
|
||||
print(f"{'='*90}")
|
||||
print(f"\n{'Type':<22} {'N':>5} {'AvgScore':>8}", end="")
|
||||
for h in FWD_HORIZONS:
|
||||
label = h.replace("fwd_return_", "")
|
||||
print(f" {label+'_mean':>9} {label+'_wr':>7}", end="")
|
||||
print()
|
||||
print("-" * 90)
|
||||
|
||||
for row_out in csv_rows:
|
||||
avg_s = row_out["avg_score"]
|
||||
print(f"{row_out['event_type']:<22} {row_out['count']:>5} {avg_s:.3f}" if avg_s else
|
||||
f"{row_out['event_type']:<22} {row_out['count']:>5} N/A", end="")
|
||||
for h in FWD_HORIZONS:
|
||||
mean_val = row_out[f"{h}_mean"]
|
||||
wr_val = row_out[f"{h}_win_rate"]
|
||||
mean_str = f"{mean_val:+.4f}" if mean_val is not None else " N/A"
|
||||
wr_str = f"{wr_val:.1%}" if wr_val is not None else " N/A"
|
||||
print(f" {mean_str:>9} {wr_str:>7}", end="")
|
||||
print()
|
||||
|
||||
print(f"\nCSV written to: {csv_path}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Event Type Decomposition Analysis")
|
||||
parser.add_argument("--snapshot-dir", required=True, help="Path to snapshot directory")
|
||||
parser.add_argument("--split", default="train", help="Split name (train/valid/test)")
|
||||
parser.add_argument("--output-dir", default="./data/analysis", help="Output directory")
|
||||
args = parser.parse_args()
|
||||
|
||||
rows = _load_rows(Path(args.snapshot_dir), args.split)
|
||||
if not rows:
|
||||
print("No data found.")
|
||||
return
|
||||
|
||||
analyze(rows, Path(args.output_dir))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -0,0 +1,122 @@
|
||||
"""Kill Switch Impact Analysis: Compare live vs research mode.
|
||||
|
||||
How much does the permanent kill switch hide?
|
||||
|
||||
Runs the backtest twice (live mode and research mode) and compares
|
||||
key metrics side-by-side.
|
||||
|
||||
Usage:
|
||||
python -m dev.analysis.kill_switch_impact_analysis \
|
||||
--manifest configs/experiments/phase5_v1.json \
|
||||
--snapshot-dir data/datasets/snapshots \
|
||||
--split train [--output-root ./runs]
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
from libs.backtest.manifests import load_manifest, resolve_config
|
||||
from libs.backtest.domain import MetricsBundle
|
||||
|
||||
|
||||
def _run_backtest(
|
||||
manifest_path: str,
|
||||
snapshot_dir: str | None,
|
||||
split: str,
|
||||
mode: str,
|
||||
output_root: str,
|
||||
initial_equity: float,
|
||||
config_root: str,
|
||||
) -> tuple[str, MetricsBundle, int]:
|
||||
"""Run a single backtest and return (run_id, metrics, trade_count)."""
|
||||
from apps.backtester.run import BacktestRunner, _build_store
|
||||
|
||||
manifest = load_manifest(manifest_path)
|
||||
config = resolve_config(manifest, config_root=config_root)
|
||||
config.risk.backtest_mode = mode
|
||||
|
||||
store = _build_store(manifest, config, split, snapshot_dir_override=snapshot_dir)
|
||||
runner = BacktestRunner(
|
||||
manifest=manifest,
|
||||
config=config,
|
||||
store=store,
|
||||
initial_equity=initial_equity,
|
||||
)
|
||||
result = runner.run(output_root=output_root)
|
||||
return result.run_id, result.metrics, result.metrics.trade_count
|
||||
|
||||
|
||||
def _fmt(val: float | None, fmt: str = ".2f") -> str:
|
||||
if val is None:
|
||||
return "N/A"
|
||||
return f"{val:{fmt}}"
|
||||
|
||||
|
||||
def _fmt_pct(val: float | None) -> str:
|
||||
if val is None:
|
||||
return "N/A"
|
||||
return f"{val:.2f}%"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Kill Switch Impact Analysis")
|
||||
parser.add_argument("--manifest", required=True, help="Path to experiment manifest JSON")
|
||||
parser.add_argument("--snapshot-dir", help="Override snapshot root directory")
|
||||
parser.add_argument("--split", default="train", help="Split name (train/valid/test)")
|
||||
parser.add_argument("--output-root", default="./runs", help="Output root directory")
|
||||
parser.add_argument("--initial-equity", type=float, default=100_000.0)
|
||||
parser.add_argument("--config-root", default=".", help="Root dir for resolving config paths")
|
||||
args = parser.parse_args()
|
||||
|
||||
print("Running LIVE mode backtest...")
|
||||
live_id, live_m, live_trades = _run_backtest(
|
||||
args.manifest, args.snapshot_dir, args.split, "live",
|
||||
args.output_root, args.initial_equity, args.config_root,
|
||||
)
|
||||
|
||||
print("Running RESEARCH mode backtest...")
|
||||
research_id, research_m, research_trades = _run_backtest(
|
||||
args.manifest, args.snapshot_dir, args.split, "research",
|
||||
args.output_root, args.initial_equity, args.config_root,
|
||||
)
|
||||
|
||||
# Side-by-side comparison
|
||||
print(f"\n{'='*60}")
|
||||
print(f"Kill Switch Impact Analysis — split={args.split}")
|
||||
print(f"{'='*60}")
|
||||
print(f"\n{'Metric':<28} {'Live':>14} {'Research':>14}")
|
||||
print("-" * 60)
|
||||
|
||||
rows = [
|
||||
("Run ID", live_id[:16], research_id[:16]),
|
||||
("Total Trades", str(live_trades), str(research_trades)),
|
||||
("Win Rate", _fmt_pct(live_m.win_rate and live_m.win_rate * 100),
|
||||
_fmt_pct(research_m.win_rate and research_m.win_rate * 100)),
|
||||
("Total Return", _fmt_pct(live_m.total_return_pct),
|
||||
_fmt_pct(research_m.total_return_pct)),
|
||||
("Max Drawdown", _fmt_pct(live_m.max_drawdown_pct),
|
||||
_fmt_pct(research_m.max_drawdown_pct)),
|
||||
("Sharpe Ratio", _fmt(live_m.sharpe_ratio), _fmt(research_m.sharpe_ratio)),
|
||||
("Sortino Ratio", _fmt(live_m.sortino_ratio), _fmt(research_m.sortino_ratio)),
|
||||
("Profit Factor", _fmt(live_m.profit_factor), _fmt(research_m.profit_factor)),
|
||||
("Expectancy (R)", _fmt(live_m.expectancy_r), _fmt(research_m.expectancy_r)),
|
||||
("Avg Holding Days", _fmt(live_m.avg_holding_days, ".1f"),
|
||||
_fmt(research_m.avg_holding_days, ".1f")),
|
||||
("Avg Positions Held", _fmt(live_m.avg_positions_held, ".1f"),
|
||||
_fmt(research_m.avg_positions_held, ".1f")),
|
||||
]
|
||||
|
||||
for label, live_val, research_val in rows:
|
||||
print(f"{label:<28} {live_val:>14} {research_val:>14}")
|
||||
|
||||
# Delta summary
|
||||
if live_m.total_return_pct is not None and research_m.total_return_pct is not None:
|
||||
delta_return = research_m.total_return_pct - live_m.total_return_pct
|
||||
delta_trades = research_trades - live_trades
|
||||
print(f"\n Delta trades: {delta_trades:+d}")
|
||||
print(f" Delta return: {delta_return:+.2f}%")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -0,0 +1,170 @@
|
||||
"""Signal Quality Analysis: Score-bucket monotonicity check.
|
||||
|
||||
Does higher composite score predict better forward returns?
|
||||
|
||||
Reads snapshot parquet, computes compute_entry_score for each row,
|
||||
buckets into quintiles, and checks Spearman rank correlation between
|
||||
score bucket and mean forward return.
|
||||
|
||||
Usage:
|
||||
python -m dev.analysis.signal_quality_analysis \
|
||||
--snapshot-dir data/datasets/snapshots/snapshot_2026_03_20 \
|
||||
--split train [--output-dir ./data/analysis]
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import statistics
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pyarrow.parquet as pq
|
||||
|
||||
from libs.backtest.scoring import compute_entry_score
|
||||
|
||||
|
||||
def _load_rows(snapshot_dir: Path, split: str) -> list[dict[str, Any]]:
|
||||
"""Load parquet snapshot into list of dicts."""
|
||||
parquet_path = snapshot_dir / f"{split}.parquet"
|
||||
table = pq.read_table(parquet_path)
|
||||
return table.to_pylist()
|
||||
|
||||
|
||||
def _bucket_label(score: float) -> str:
|
||||
"""Assign score to a quintile bucket."""
|
||||
if score < 0.2:
|
||||
return "0.0-0.2"
|
||||
if score < 0.4:
|
||||
return "0.2-0.4"
|
||||
if score < 0.6:
|
||||
return "0.4-0.6"
|
||||
if score < 0.8:
|
||||
return "0.6-0.8"
|
||||
return "0.8-1.0"
|
||||
|
||||
|
||||
BUCKET_ORDER = ["0.0-0.2", "0.2-0.4", "0.4-0.6", "0.6-0.8", "0.8-1.0"]
|
||||
FWD_HORIZONS = ["fwd_return_1d", "fwd_return_3d", "fwd_return_5d", "fwd_return_10d"]
|
||||
|
||||
|
||||
def analyze(rows: list[dict[str, Any]], output_dir: Path) -> None:
|
||||
"""Score all rows, bucket, compute statistics, write output."""
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Score each row and assign bucket
|
||||
scored: list[dict[str, Any]] = []
|
||||
for row in rows:
|
||||
score = compute_entry_score(row)
|
||||
bucket = _bucket_label(score)
|
||||
scored.append({"score": score, "bucket": bucket, **row})
|
||||
|
||||
# Group by bucket
|
||||
by_bucket: dict[str, list[dict[str, Any]]] = {b: [] for b in BUCKET_ORDER}
|
||||
for s in scored:
|
||||
by_bucket[s["bucket"]].append(s)
|
||||
|
||||
# Compute per-bucket stats
|
||||
csv_rows: list[dict[str, Any]] = []
|
||||
for bucket in BUCKET_ORDER:
|
||||
items = by_bucket[bucket]
|
||||
n = len(items)
|
||||
row_out: dict[str, Any] = {"bucket": bucket, "count": n}
|
||||
|
||||
for horizon in FWD_HORIZONS:
|
||||
vals = [float(r[horizon]) for r in items if r.get(horizon) is not None]
|
||||
if vals:
|
||||
row_out[f"{horizon}_mean"] = statistics.mean(vals)
|
||||
row_out[f"{horizon}_win_rate"] = sum(1 for v in vals if v > 0) / len(vals)
|
||||
else:
|
||||
row_out[f"{horizon}_mean"] = None
|
||||
row_out[f"{horizon}_win_rate"] = None
|
||||
|
||||
csv_rows.append(row_out)
|
||||
|
||||
# Spearman rank correlation (bucket rank vs mean return)
|
||||
spearman_results: dict[str, float | None] = {}
|
||||
for horizon in FWD_HORIZONS:
|
||||
means = [r[f"{horizon}_mean"] for r in csv_rows if r[f"{horizon}_mean"] is not None]
|
||||
if len(means) >= 3:
|
||||
# Rank correlation: do the means increase with bucket rank?
|
||||
ranks_score = list(range(len(means)))
|
||||
ranks_return = _rank(means)
|
||||
spearman_results[horizon] = _spearman(ranks_score, ranks_return)
|
||||
else:
|
||||
spearman_results[horizon] = None
|
||||
|
||||
# Write CSV
|
||||
csv_path = output_dir / "signal_quality_analysis.csv"
|
||||
if csv_rows:
|
||||
fieldnames = list(csv_rows[0].keys())
|
||||
with open(csv_path, "w", newline="") as f:
|
||||
writer = csv.DictWriter(f, fieldnames=fieldnames)
|
||||
writer.writeheader()
|
||||
writer.writerows(csv_rows)
|
||||
|
||||
# Console output
|
||||
print(f"\n{'='*80}")
|
||||
print(f"Signal Quality Analysis — {len(scored)} events")
|
||||
print(f"{'='*80}")
|
||||
print(f"\n{'Bucket':<12} {'Count':>6}", end="")
|
||||
for h in FWD_HORIZONS:
|
||||
label = h.replace("fwd_return_", "")
|
||||
print(f" {label+'_mean':>10} {label+'_wr':>8}", end="")
|
||||
print()
|
||||
print("-" * 80)
|
||||
|
||||
for row_out in csv_rows:
|
||||
print(f"{row_out['bucket']:<12} {row_out['count']:>6}", end="")
|
||||
for h in FWD_HORIZONS:
|
||||
mean_val = row_out[f"{h}_mean"]
|
||||
wr_val = row_out[f"{h}_win_rate"]
|
||||
mean_str = f"{mean_val:+.4f}" if mean_val is not None else " N/A"
|
||||
wr_str = f"{wr_val:.1%}" if wr_val is not None else " N/A"
|
||||
print(f" {mean_str:>10} {wr_str:>8}", end="")
|
||||
print()
|
||||
|
||||
print(f"\nSpearman rank correlation (bucket vs mean return):")
|
||||
for h in FWD_HORIZONS:
|
||||
label = h.replace("fwd_return_", "")
|
||||
val = spearman_results.get(h)
|
||||
print(f" {label}: {val:+.3f}" if val is not None else f" {label}: N/A")
|
||||
|
||||
print(f"\nCSV written to: {csv_path}")
|
||||
|
||||
|
||||
def _rank(values: list[float]) -> list[float]:
|
||||
"""Compute ranks (0-indexed) for a list of values."""
|
||||
indexed = sorted(enumerate(values), key=lambda x: x[1])
|
||||
ranks = [0.0] * len(values)
|
||||
for rank, (idx, _) in enumerate(indexed):
|
||||
ranks[idx] = float(rank)
|
||||
return ranks
|
||||
|
||||
|
||||
def _spearman(x: list[float], y: list[float]) -> float:
|
||||
"""Compute Spearman rank correlation coefficient."""
|
||||
n = len(x)
|
||||
if n < 2:
|
||||
return 0.0
|
||||
d_sq = sum((xi - yi) ** 2 for xi, yi in zip(x, y))
|
||||
return 1 - (6 * d_sq) / (n * (n**2 - 1))
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Signal Quality Analysis")
|
||||
parser.add_argument("--snapshot-dir", required=True, help="Path to snapshot directory")
|
||||
parser.add_argument("--split", default="train", help="Split name (train/valid/test)")
|
||||
parser.add_argument("--output-dir", default="./data/analysis", help="Output directory")
|
||||
args = parser.parse_args()
|
||||
|
||||
rows = _load_rows(Path(args.snapshot_dir), args.split)
|
||||
if not rows:
|
||||
print("No data found.")
|
||||
return
|
||||
|
||||
analyze(rows, Path(args.output_dir))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Reference in New Issue