"""Kill Switch Impact Analysis: Compare live vs research mode. How much does the permanent kill switch hide? Runs the backtest twice (live mode and research mode) and compares key metrics side-by-side. Usage: python -m apps.tools.kill_switch_impact_analysis \ --manifest configs/experiments/return_max_long_v1.1.json \ --snapshot-dir data/datasets/snapshots \ --split train [--output-root ./runs] """ from __future__ import annotations import argparse from pathlib import Path from libs.backtest.manifests import load_manifest, resolve_config from libs.backtest.domain import MetricsBundle def _run_backtest( manifest_path: str, snapshot_dir: str | None, split: str, mode: str, output_root: str, initial_equity: float, config_root: str, ) -> tuple[str, MetricsBundle, int]: """Run a single backtest and return (run_id, metrics, trade_count).""" from apps.backtester.run import BacktestRunner, _build_store manifest = load_manifest(manifest_path) config = resolve_config(manifest, config_root=config_root) config.risk.backtest_mode = mode store = _build_store(manifest, config, split, snapshot_dir_override=snapshot_dir) runner = BacktestRunner( manifest=manifest, config=config, store=store, initial_equity=initial_equity, ) result = runner.run(output_root=output_root) return result.run_id, result.metrics, result.metrics.trade_count def _fmt(val: float | None, fmt: str = ".2f") -> str: if val is None: return "N/A" return f"{val:{fmt}}" def _fmt_pct(val: float | None) -> str: if val is None: return "N/A" return f"{val:.2f}%" def main() -> None: parser = argparse.ArgumentParser(description="Kill Switch Impact Analysis") parser.add_argument("--manifest", required=True, help="Path to experiment manifest JSON") parser.add_argument("--snapshot-dir", help="Override snapshot root directory") parser.add_argument("--split", default="train", help="Split name (train/valid/test)") parser.add_argument("--output-root", default="./runs", help="Output root directory") parser.add_argument("--initial-equity", type=float, default=100_000.0) parser.add_argument("--config-root", default=".", help="Root dir for resolving config paths") args = parser.parse_args() print("Running LIVE mode backtest...") live_id, live_m, live_trades = _run_backtest( args.manifest, args.snapshot_dir, args.split, "live", args.output_root, args.initial_equity, args.config_root, ) print("Running RESEARCH mode backtest...") research_id, research_m, research_trades = _run_backtest( args.manifest, args.snapshot_dir, args.split, "research", args.output_root, args.initial_equity, args.config_root, ) # Side-by-side comparison print(f"\n{'='*60}") print(f"Kill Switch Impact Analysis — split={args.split}") print(f"{'='*60}") print(f"\n{'Metric':<28} {'Live':>14} {'Research':>14}") print("-" * 60) rows = [ ("Run ID", live_id[:16], research_id[:16]), ("Total Trades", str(live_trades), str(research_trades)), ("Win Rate", _fmt_pct(live_m.win_rate and live_m.win_rate * 100), _fmt_pct(research_m.win_rate and research_m.win_rate * 100)), ("Total Return", _fmt_pct(live_m.total_return_pct), _fmt_pct(research_m.total_return_pct)), ("Max Drawdown", _fmt_pct(live_m.max_drawdown_pct), _fmt_pct(research_m.max_drawdown_pct)), ("Sharpe Ratio", _fmt(live_m.sharpe_ratio), _fmt(research_m.sharpe_ratio)), ("Sortino Ratio", _fmt(live_m.sortino_ratio), _fmt(research_m.sortino_ratio)), ("Profit Factor", _fmt(live_m.profit_factor), _fmt(research_m.profit_factor)), ("Expectancy (R)", _fmt(live_m.expectancy_r), _fmt(research_m.expectancy_r)), ("Avg Holding Days", _fmt(live_m.avg_holding_days, ".1f"), _fmt(research_m.avg_holding_days, ".1f")), ("Avg Positions Held", _fmt(live_m.avg_positions_held, ".1f"), _fmt(research_m.avg_positions_held, ".1f")), ] for label, live_val, research_val in rows: print(f"{label:<28} {live_val:>14} {research_val:>14}") # Delta summary if live_m.total_return_pct is not None and research_m.total_return_pct is not None: delta_return = research_m.total_return_pct - live_m.total_return_pct delta_trades = research_trades - live_trades print(f"\n Delta trades: {delta_trades:+d}") print(f" Delta return: {delta_return:+.2f}%") if __name__ == "__main__": main()