You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
123 lines
4.5 KiB
Python
123 lines
4.5 KiB
Python
"""Kill Switch Impact Analysis: Compare live vs research mode.
|
|
|
|
How much does the permanent kill switch hide?
|
|
|
|
Runs the backtest twice (live mode and research mode) and compares
|
|
key metrics side-by-side.
|
|
|
|
Usage:
|
|
python -m dev.analysis.kill_switch_impact_analysis \
|
|
--manifest configs/experiments/phase5_v1.json \
|
|
--snapshot-dir data/datasets/snapshots \
|
|
--split train [--output-root ./runs]
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from pathlib import Path
|
|
|
|
from libs.backtest.manifests import load_manifest, resolve_config
|
|
from libs.backtest.domain import MetricsBundle
|
|
|
|
|
|
def _run_backtest(
|
|
manifest_path: str,
|
|
snapshot_dir: str | None,
|
|
split: str,
|
|
mode: str,
|
|
output_root: str,
|
|
initial_equity: float,
|
|
config_root: str,
|
|
) -> tuple[str, MetricsBundle, int]:
|
|
"""Run a single backtest and return (run_id, metrics, trade_count)."""
|
|
from apps.backtester.run import BacktestRunner, _build_store
|
|
|
|
manifest = load_manifest(manifest_path)
|
|
config = resolve_config(manifest, config_root=config_root)
|
|
config.risk.backtest_mode = mode
|
|
|
|
store = _build_store(manifest, config, split, snapshot_dir_override=snapshot_dir)
|
|
runner = BacktestRunner(
|
|
manifest=manifest,
|
|
config=config,
|
|
store=store,
|
|
initial_equity=initial_equity,
|
|
)
|
|
result = runner.run(output_root=output_root)
|
|
return result.run_id, result.metrics, result.metrics.trade_count
|
|
|
|
|
|
def _fmt(val: float | None, fmt: str = ".2f") -> str:
|
|
if val is None:
|
|
return "N/A"
|
|
return f"{val:{fmt}}"
|
|
|
|
|
|
def _fmt_pct(val: float | None) -> str:
|
|
if val is None:
|
|
return "N/A"
|
|
return f"{val:.2f}%"
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description="Kill Switch Impact Analysis")
|
|
parser.add_argument("--manifest", required=True, help="Path to experiment manifest JSON")
|
|
parser.add_argument("--snapshot-dir", help="Override snapshot root directory")
|
|
parser.add_argument("--split", default="train", help="Split name (train/valid/test)")
|
|
parser.add_argument("--output-root", default="./runs", help="Output root directory")
|
|
parser.add_argument("--initial-equity", type=float, default=100_000.0)
|
|
parser.add_argument("--config-root", default=".", help="Root dir for resolving config paths")
|
|
args = parser.parse_args()
|
|
|
|
print("Running LIVE mode backtest...")
|
|
live_id, live_m, live_trades = _run_backtest(
|
|
args.manifest, args.snapshot_dir, args.split, "live",
|
|
args.output_root, args.initial_equity, args.config_root,
|
|
)
|
|
|
|
print("Running RESEARCH mode backtest...")
|
|
research_id, research_m, research_trades = _run_backtest(
|
|
args.manifest, args.snapshot_dir, args.split, "research",
|
|
args.output_root, args.initial_equity, args.config_root,
|
|
)
|
|
|
|
# Side-by-side comparison
|
|
print(f"\n{'='*60}")
|
|
print(f"Kill Switch Impact Analysis — split={args.split}")
|
|
print(f"{'='*60}")
|
|
print(f"\n{'Metric':<28} {'Live':>14} {'Research':>14}")
|
|
print("-" * 60)
|
|
|
|
rows = [
|
|
("Run ID", live_id[:16], research_id[:16]),
|
|
("Total Trades", str(live_trades), str(research_trades)),
|
|
("Win Rate", _fmt_pct(live_m.win_rate and live_m.win_rate * 100),
|
|
_fmt_pct(research_m.win_rate and research_m.win_rate * 100)),
|
|
("Total Return", _fmt_pct(live_m.total_return_pct),
|
|
_fmt_pct(research_m.total_return_pct)),
|
|
("Max Drawdown", _fmt_pct(live_m.max_drawdown_pct),
|
|
_fmt_pct(research_m.max_drawdown_pct)),
|
|
("Sharpe Ratio", _fmt(live_m.sharpe_ratio), _fmt(research_m.sharpe_ratio)),
|
|
("Sortino Ratio", _fmt(live_m.sortino_ratio), _fmt(research_m.sortino_ratio)),
|
|
("Profit Factor", _fmt(live_m.profit_factor), _fmt(research_m.profit_factor)),
|
|
("Expectancy (R)", _fmt(live_m.expectancy_r), _fmt(research_m.expectancy_r)),
|
|
("Avg Holding Days", _fmt(live_m.avg_holding_days, ".1f"),
|
|
_fmt(research_m.avg_holding_days, ".1f")),
|
|
("Avg Positions Held", _fmt(live_m.avg_positions_held, ".1f"),
|
|
_fmt(research_m.avg_positions_held, ".1f")),
|
|
]
|
|
|
|
for label, live_val, research_val in rows:
|
|
print(f"{label:<28} {live_val:>14} {research_val:>14}")
|
|
|
|
# Delta summary
|
|
if live_m.total_return_pct is not None and research_m.total_return_pct is not None:
|
|
delta_return = research_m.total_return_pct - live_m.total_return_pct
|
|
delta_trades = research_trades - live_trades
|
|
print(f"\n Delta trades: {delta_trades:+d}")
|
|
print(f" Delta return: {delta_return:+.2f}%")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|