fithia2/apps/tools/run_composite_eval.py

#!/usr/bin/env python3
"""Composite performance evaluator: run one or more primary strategies with
idle-alpha sleeve + cash parking presets and compare results side-by-side.

Usage:
    python -m apps.tools.run_composite_eval \\
        --manifests configs/experiments/return_max_long_v7.70.json \\
                    configs/experiments/return_max_long_v18.250.json \\
        --parking qqqm_low_dd \\
        --idle-alpha micro_event_alpha_plus_event_plus \\
        --output-root /tmp/composite_eval

Compares composite CW, Sharpe, DD, idle fraction, and amplification factor.
"""
from __future__ import annotations

import argparse
import json
import subprocess
import sys
from pathlib import Path


REPO_ROOT = Path(__file__).resolve().parents[2]
DEFAULT_SNAPSHOT_DIR = ""
DEFAULT_OUTPUT_ROOT = "/tmp/composite_eval"


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(
        description="Run multiple experiments with IA+parking presets and compare composite results."
    )
    parser.add_argument(
        "--manifests", nargs="+", required=True,
        help="Experiment manifest paths (can use experiment names like 'return_max_long_v7.70').",
    )
    parser.add_argument(
        "--parking", default="qqqm_low_dd",
        help="Cash parking preset name (default: qqqm_low_dd).",
    )
    parser.add_argument(
        "--idle-alpha", default="micro_event_alpha_plus_event_plus",
        help="Idle alpha sleeve preset name (default: micro_event_alpha_plus_event_plus).",
    )
    parser.add_argument(
        "--no-presets", action="store_true",
        help="Run without any presets (primary-only mode for baseline comparison).",
    )
    parser.add_argument(
        "--split", default="all",
        help="Backtest split (default: 'all' = full period merging train+valid+test).",
    )
    parser.add_argument(
        "--snapshot-dir", default=DEFAULT_SNAPSHOT_DIR,
        help=f"Snapshot root directory (default: {DEFAULT_SNAPSHOT_DIR}).",
    )
    parser.add_argument(
        "--output-root", default=DEFAULT_OUTPUT_ROOT,
        help=f"Root directory for run artifacts (default: {DEFAULT_OUTPUT_ROOT}).",
    )
    parser.add_argument(
        "--sort", choices=["cw", "sharpe", "dd", "amp", "idle"], default="cw",
        help="Sort results by this column (default: cw).",
    )
    parser.add_argument(
        "--idle-alpha-dedup", default=None, choices=["skip", "rename"],
        help="IA dedup mode: 'rename' injects IA engines even when engine_id conflicts with primary (adds __ia_sleeve suffix).",
    )
    return parser.parse_args()


def _resolve_manifest_path(manifest: str) -> str:
    """Resolve experiment name or path to a JSON file path."""
    p = Path(manifest)
    if p.exists():
        return str(p)
    # Try configs/experiments/return_max_long_<name>.json and bare name
    candidates = [
        REPO_ROOT / "configs" / "experiments" / f"{manifest}.json",
        REPO_ROOT / "configs" / "experiments" / f"return_max_long_{manifest}.json",
    ]
    for c in candidates:
        if c.exists():
            return str(c)
    return manifest  # pass through; run.py will error if not found


def run_composite(
    manifest: str,
    parking_preset: str | None,
    idle_alpha_preset: str | None,
    split: str | None,
    output_root: Path,
    snapshot_dir: str,
    idle_alpha_dedup: str | None = None,
) -> dict[str, object]:
    """Run a single backtest with presets and return key metrics."""
    exp_name = Path(manifest).stem
    run_output = output_root / exp_name
    run_output.mkdir(parents=True, exist_ok=True)

    cmd = [
        sys.executable, "-m", "apps.backtester.run",
        "--manifest", manifest,
        "--output-root", str(run_output),
    ]
    if snapshot_dir:
        cmd += ["--snapshot-dir", snapshot_dir]
    if split:
        cmd += ["--split", split]
    if parking_preset:
        cmd += ["--parking", parking_preset]
    if idle_alpha_preset:
        cmd += ["--idle-alpha", idle_alpha_preset]
    if idle_alpha_dedup:
        cmd += ["--idle-alpha-dedup", idle_alpha_dedup]

    print(f"  Running {exp_name}...", flush=True)
    result = subprocess.run(
        cmd,
        cwd=str(REPO_ROOT),
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT,
        text=True,
        check=False,
    )
    # Print last few lines of output
    lines = result.stdout.splitlines() if result.stdout else []
    for line in lines[-6:]:
        print(f"    {line}")
    if result.returncode != 0:
        raise RuntimeError(f"{exp_name} failed (returncode={result.returncode})")

    # Find the most recently created run directory
    run_dirs = sorted(run_output.glob("bt_*"), key=lambda p: p.stat().st_mtime)
    if not run_dirs:
        raise RuntimeError(f"No run directory found under {run_output}")
    latest = run_dirs[-1]

    # Read metrics
    metrics_path = latest / "metrics" / "metrics_summary.json"
    sleeve_path = latest / "metrics" / "sleeve_decomposition.json"

    summary = json.loads(metrics_path.read_text()) if metrics_path.exists() else {}
    sleeve = json.loads(sleeve_path.read_text()) if sleeve_path.exists() else {}

    ia_injected = sleeve.get("ia_engines_injected", [])
    ia_blocked = sleeve.get("ia_engines_blocked", [])
    return {
        "experiment": exp_name,
        "run_dir": str(latest),
        "trade_count": summary.get("trade_count", 0),
        "total_return_pct": summary.get("total_return_pct"),
        "sharpe_ratio": summary.get("sharpe_ratio"),
        "max_drawdown_pct": summary.get("max_drawdown_pct"),
        "composite_amplification": sleeve.get("composite_amplification"),
        "avg_idle_fraction_pct": sleeve.get("avg_idle_fraction_pct"),
        "core_contribution_pct": sleeve.get("core", {}).get("contribution_pct"),
        "ia_contribution_pct": sleeve.get("idle_alpha", {}).get("contribution_pct"),
        "parking_contribution_pct": sleeve.get("parking", {}).get("contribution_pct"),
        "ia_engines_injected_count": len(ia_injected),
        "ia_engines_blocked_count": len(ia_blocked),
        "sleeve_file": str(sleeve_path) if sleeve_path.exists() else None,
    }


def _fmt(v: object, decimals: int = 2) -> str:
    if v is None:
        return "N/A"
    try:
        return f"{float(v):.{decimals}f}"  # type: ignore[arg-type]
    except (TypeError, ValueError):
        return str(v)


def print_table(rows: list[dict[str, object]], sort_key: str) -> None:
    sort_map = {
        "cw": "total_return_pct",
        "sharpe": "sharpe_ratio",
        "dd": "max_drawdown_pct",
        "amp": "composite_amplification",
        "idle": "avg_idle_fraction_pct",
    }
    field = sort_map.get(sort_key, "total_return_pct")
    reverse = sort_key != "dd"  # lower DD is better
    sorted_rows = sorted(
        rows,
        key=lambda r: (r.get(field) is not None, r.get(field) or 0.0),
        reverse=reverse,
    )

    header = (
        f"{'Experiment':<45} {'CW%':>8} {'Sharpe':>7} {'DD%':>6} "
        f"{'Amp':>6} {'Idle%':>6} {'Core%':>6} {'IA%':>5} {'Park%':>6} {'IAInj':>6} {'IABlk':>6} {'N':>5}"
    )
    sep = "-" * len(header)
    print(f"\n{sep}")
    print(header)
    print(sep)
    for r in sorted_rows:
        name = str(r["experiment"])[:44]
        ia_inj = r.get("ia_engines_injected_count")
        ia_blk = r.get("ia_engines_blocked_count")
        print(
            f"{name:<45} "
            f"{_fmt(r['total_return_pct']):>8} "
            f"{_fmt(r['sharpe_ratio']):>7} "
            f"{_fmt(r['max_drawdown_pct']):>6} "
            f"{_fmt(r['composite_amplification']):>6} "
            f"{_fmt(r['avg_idle_fraction_pct']):>6} "
            f"{_fmt(r['core_contribution_pct']):>6} "
            f"{_fmt(r['ia_contribution_pct']):>5} "
            f"{_fmt(r['parking_contribution_pct']):>6} "
            f"{str(ia_inj) if ia_inj is not None else 'N/A':>6} "
            f"{str(ia_blk) if ia_blk is not None else 'N/A':>6} "
            f"{int(r['trade_count'] or 0):>5}"
        )
    print(sep)


def main() -> None:
    args = parse_args()
    output_root = Path(args.output_root)
    output_root.mkdir(parents=True, exist_ok=True)

    parking = None if args.no_presets else args.parking
    idle_alpha = None if args.no_presets else args.idle_alpha
    preset_label = "no presets" if args.no_presets else f"parking={parking} ia={idle_alpha}"
    print(f"Composite eval: {len(args.manifests)} experiment(s) [{preset_label}]")

    rows: list[dict[str, object]] = []
    errors: list[str] = []
    for manifest in args.manifests:
        resolved = _resolve_manifest_path(manifest)
        try:
            row = run_composite(
                manifest=resolved,
                parking_preset=parking,
                idle_alpha_preset=idle_alpha,
                split=args.split,
                output_root=output_root,
                snapshot_dir=args.snapshot_dir,
                idle_alpha_dedup=args.idle_alpha_dedup,
            )
            rows.append(row)
        except Exception as exc:
            exp_name = Path(manifest).stem
            print(f"  ERROR {exp_name}: {exc}")
            errors.append(exp_name)

    if rows:
        print_table(rows, sort_key=args.sort)

    # Save summary JSON
    summary_path = output_root / "composite_eval_summary.json"
    summary_path.write_text(json.dumps(rows, indent=2) + "\n")
    print(f"\nSaved summary to: {summary_path}")

    if errors:
        print(f"\nFailed experiments: {', '.join(errors)}")
        sys.exit(1)


if __name__ == "__main__":
    main()