"""Replay runner: execute backtest twice and assert identical output. Usage: python -m apps.backtester.replay --manifest configs/experiments/baseline_v1.json \ --snapshot-id snapshot_2026_03_20 --output-root ./runs """ from __future__ import annotations import argparse import json import sys from pathlib import Path from libs.backtest.manifests import load_manifest, resolve_config from libs.common.logging import get_logger logger = get_logger(__name__) def _run_once( manifest_path: str, snapshot_id: str | None, split: str, output_root: str, config_root: str, initial_equity: float, ) -> dict: """Execute one backtest run and return a dict of key determinism metrics.""" from apps.backtester.run import BacktestRunner, _build_store from libs.backtest.manifests import load_manifest, resolve_config manifest = load_manifest(manifest_path) config = resolve_config(manifest, config_root=config_root, snapshot_id_override=snapshot_id) store = _build_store(manifest, config, split) runner = BacktestRunner(manifest=manifest, config=config, store=store, initial_equity=initial_equity) result = runner.run(output_root=output_root) # Build a determinism fingerprint (exclude timestamps and run_id) return { "trade_count": result.metrics.trade_count, "total_return_pct": result.metrics.total_return_pct, "max_drawdown_pct": result.metrics.max_drawdown_pct, "win_rate": result.metrics.win_rate, "avg_r_multiple": result.metrics.avg_r_multiple, "total_trading_days": result.total_trading_days, "total_candidates_seen": result.total_candidates_seen, "total_orders_rejected": result.total_orders_rejected, } def run_replay( manifest_path: str, snapshot_id: str | None, split: str, output_root: str, config_root: str, initial_equity: float, ) -> bool: """Run twice and compare. Returns True if identical, False otherwise.""" logger.info("replay_run_1_start") run1 = _run_once(manifest_path, snapshot_id, split, output_root, config_root, initial_equity) logger.info("replay_run_1_complete", metrics=run1) logger.info("replay_run_2_start") run2 = _run_once(manifest_path, snapshot_id, split, output_root, config_root, initial_equity) logger.info("replay_run_2_complete", metrics=run2) if run1 == run2: print("REPLAY PASS: runs are identical") print(json.dumps(run1, indent=2)) return True else: print("REPLAY FAIL: runs differ!") print("Run 1:") print(json.dumps(run1, indent=2)) print("Run 2:") print(json.dumps(run2, indent=2)) diff = {k: (run1.get(k), run2.get(k)) for k in set(run1) | set(run2) if run1.get(k) != run2.get(k)} print("Differences:") print(json.dumps(diff, indent=2)) return False def main() -> None: parser = argparse.ArgumentParser(description="ACE-F Backtest Replay / Determinism Check") parser.add_argument("--manifest", required=True, help="Path to experiment manifest JSON") parser.add_argument("--snapshot-id", help="Override dataset_snapshot_id") parser.add_argument("--split", default="train") parser.add_argument("--output-root", default="./runs") parser.add_argument("--config-root", default=".") parser.add_argument("--initial-equity", type=float, default=100_000.0) args = parser.parse_args() ok = run_replay( manifest_path=args.manifest, snapshot_id=args.snapshot_id, split=args.split, output_root=args.output_root, config_root=args.config_root, initial_equity=args.initial_equity, ) sys.exit(0 if ok else 1) if __name__ == "__main__": main()