#!/usr/bin/env python3 """Composite performance evaluator: run one or more primary strategies with idle-alpha sleeve + cash parking presets and compare results side-by-side. Usage: python -m apps.tools.run_composite_eval \\ --manifests configs/experiments/return_max_long_v7.70.json \\ configs/experiments/return_max_long_v18.250.json \\ --parking qqqm_low_dd \\ --idle-alpha micro_event_alpha_plus_event_plus \\ --output-root /tmp/composite_eval Compares composite CW, Sharpe, DD, idle fraction, and amplification factor. """ from __future__ import annotations import argparse import json import subprocess import sys from pathlib import Path REPO_ROOT = Path(__file__).resolve().parents[2] DEFAULT_SNAPSHOT_DIR = "" DEFAULT_OUTPUT_ROOT = "/tmp/composite_eval" def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( description="Run multiple experiments with IA+parking presets and compare composite results." ) parser.add_argument( "--manifests", nargs="+", required=True, help="Experiment manifest paths (can use experiment names like 'return_max_long_v7.70').", ) parser.add_argument( "--parking", default="qqqm_low_dd", help="Cash parking preset name (default: qqqm_low_dd).", ) parser.add_argument( "--idle-alpha", default="micro_event_alpha_plus_event_plus", help="Idle alpha sleeve preset name (default: micro_event_alpha_plus_event_plus).", ) parser.add_argument( "--no-presets", action="store_true", help="Run without any presets (primary-only mode for baseline comparison).", ) parser.add_argument( "--split", default="all", help="Backtest split (default: 'all' = full period merging train+valid+test).", ) parser.add_argument( "--snapshot-dir", default=DEFAULT_SNAPSHOT_DIR, help=f"Snapshot root directory (default: {DEFAULT_SNAPSHOT_DIR}).", ) parser.add_argument( "--output-root", default=DEFAULT_OUTPUT_ROOT, help=f"Root directory for run artifacts (default: {DEFAULT_OUTPUT_ROOT}).", ) parser.add_argument( "--sort", choices=["cw", "sharpe", "dd", "amp", "idle"], default="cw", help="Sort results by this column (default: cw).", ) parser.add_argument( "--idle-alpha-dedup", default=None, choices=["skip", "rename"], help="IA dedup mode: 'rename' injects IA engines even when engine_id conflicts with primary (adds __ia_sleeve suffix).", ) return parser.parse_args() def _resolve_manifest_path(manifest: str) -> str: """Resolve experiment name or path to a JSON file path.""" p = Path(manifest) if p.exists(): return str(p) # Try configs/experiments/return_max_long_.json and bare name candidates = [ REPO_ROOT / "configs" / "experiments" / f"{manifest}.json", REPO_ROOT / "configs" / "experiments" / f"return_max_long_{manifest}.json", ] for c in candidates: if c.exists(): return str(c) return manifest # pass through; run.py will error if not found def run_composite( manifest: str, parking_preset: str | None, idle_alpha_preset: str | None, split: str | None, output_root: Path, snapshot_dir: str, idle_alpha_dedup: str | None = None, ) -> dict[str, object]: """Run a single backtest with presets and return key metrics.""" exp_name = Path(manifest).stem run_output = output_root / exp_name run_output.mkdir(parents=True, exist_ok=True) cmd = [ sys.executable, "-m", "apps.backtester.run", "--manifest", manifest, "--output-root", str(run_output), ] if snapshot_dir: cmd += ["--snapshot-dir", snapshot_dir] if split: cmd += ["--split", split] if parking_preset: cmd += ["--parking", parking_preset] if idle_alpha_preset: cmd += ["--idle-alpha", idle_alpha_preset] if idle_alpha_dedup: cmd += ["--idle-alpha-dedup", idle_alpha_dedup] print(f" Running {exp_name}...", flush=True) result = subprocess.run( cmd, cwd=str(REPO_ROOT), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, check=False, ) # Print last few lines of output lines = result.stdout.splitlines() if result.stdout else [] for line in lines[-6:]: print(f" {line}") if result.returncode != 0: raise RuntimeError(f"{exp_name} failed (returncode={result.returncode})") # Find the most recently created run directory run_dirs = sorted(run_output.glob("bt_*"), key=lambda p: p.stat().st_mtime) if not run_dirs: raise RuntimeError(f"No run directory found under {run_output}") latest = run_dirs[-1] # Read metrics metrics_path = latest / "metrics" / "metrics_summary.json" sleeve_path = latest / "metrics" / "sleeve_decomposition.json" summary = json.loads(metrics_path.read_text()) if metrics_path.exists() else {} sleeve = json.loads(sleeve_path.read_text()) if sleeve_path.exists() else {} ia_injected = sleeve.get("ia_engines_injected", []) ia_blocked = sleeve.get("ia_engines_blocked", []) return { "experiment": exp_name, "run_dir": str(latest), "trade_count": summary.get("trade_count", 0), "total_return_pct": summary.get("total_return_pct"), "sharpe_ratio": summary.get("sharpe_ratio"), "max_drawdown_pct": summary.get("max_drawdown_pct"), "composite_amplification": sleeve.get("composite_amplification"), "avg_idle_fraction_pct": sleeve.get("avg_idle_fraction_pct"), "core_contribution_pct": sleeve.get("core", {}).get("contribution_pct"), "ia_contribution_pct": sleeve.get("idle_alpha", {}).get("contribution_pct"), "parking_contribution_pct": sleeve.get("parking", {}).get("contribution_pct"), "ia_engines_injected_count": len(ia_injected), "ia_engines_blocked_count": len(ia_blocked), "sleeve_file": str(sleeve_path) if sleeve_path.exists() else None, } def _fmt(v: object, decimals: int = 2) -> str: if v is None: return "N/A" try: return f"{float(v):.{decimals}f}" # type: ignore[arg-type] except (TypeError, ValueError): return str(v) def print_table(rows: list[dict[str, object]], sort_key: str) -> None: sort_map = { "cw": "total_return_pct", "sharpe": "sharpe_ratio", "dd": "max_drawdown_pct", "amp": "composite_amplification", "idle": "avg_idle_fraction_pct", } field = sort_map.get(sort_key, "total_return_pct") reverse = sort_key != "dd" # lower DD is better sorted_rows = sorted( rows, key=lambda r: (r.get(field) is not None, r.get(field) or 0.0), reverse=reverse, ) header = ( f"{'Experiment':<45} {'CW%':>8} {'Sharpe':>7} {'DD%':>6} " f"{'Amp':>6} {'Idle%':>6} {'Core%':>6} {'IA%':>5} {'Park%':>6} {'IAInj':>6} {'IABlk':>6} {'N':>5}" ) sep = "-" * len(header) print(f"\n{sep}") print(header) print(sep) for r in sorted_rows: name = str(r["experiment"])[:44] ia_inj = r.get("ia_engines_injected_count") ia_blk = r.get("ia_engines_blocked_count") print( f"{name:<45} " f"{_fmt(r['total_return_pct']):>8} " f"{_fmt(r['sharpe_ratio']):>7} " f"{_fmt(r['max_drawdown_pct']):>6} " f"{_fmt(r['composite_amplification']):>6} " f"{_fmt(r['avg_idle_fraction_pct']):>6} " f"{_fmt(r['core_contribution_pct']):>6} " f"{_fmt(r['ia_contribution_pct']):>5} " f"{_fmt(r['parking_contribution_pct']):>6} " f"{str(ia_inj) if ia_inj is not None else 'N/A':>6} " f"{str(ia_blk) if ia_blk is not None else 'N/A':>6} " f"{int(r['trade_count'] or 0):>5}" ) print(sep) def main() -> None: args = parse_args() output_root = Path(args.output_root) output_root.mkdir(parents=True, exist_ok=True) parking = None if args.no_presets else args.parking idle_alpha = None if args.no_presets else args.idle_alpha preset_label = "no presets" if args.no_presets else f"parking={parking} ia={idle_alpha}" print(f"Composite eval: {len(args.manifests)} experiment(s) [{preset_label}]") rows: list[dict[str, object]] = [] errors: list[str] = [] for manifest in args.manifests: resolved = _resolve_manifest_path(manifest) try: row = run_composite( manifest=resolved, parking_preset=parking, idle_alpha_preset=idle_alpha, split=args.split, output_root=output_root, snapshot_dir=args.snapshot_dir, idle_alpha_dedup=args.idle_alpha_dedup, ) rows.append(row) except Exception as exc: exp_name = Path(manifest).stem print(f" ERROR {exp_name}: {exc}") errors.append(exp_name) if rows: print_table(rows, sort_key=args.sort) # Save summary JSON summary_path = output_root / "composite_eval_summary.json" summary_path.write_text(json.dumps(rows, indent=2) + "\n") print(f"\nSaved summary to: {summary_path}") if errors: print(f"\nFailed experiments: {', '.join(errors)}") sys.exit(1) if __name__ == "__main__": main()