You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
108 lines
3.7 KiB
Python
108 lines
3.7 KiB
Python
"""Replay runner: execute backtest twice and assert identical output.
|
|
|
|
Usage:
|
|
python -m apps.backtester.replay --manifest configs/experiments/baseline_v1.json \
|
|
--snapshot-id snapshot_2026_03_20 --output-root ./runs
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
from libs.backtest.manifests import load_manifest, resolve_config
|
|
from libs.common.logging import get_logger
|
|
|
|
logger = get_logger(__name__)
|
|
|
|
|
|
def _run_once(
|
|
manifest_path: str,
|
|
snapshot_id: str | None,
|
|
split: str,
|
|
output_root: str,
|
|
config_root: str,
|
|
initial_equity: float,
|
|
) -> dict:
|
|
"""Execute one backtest run and return a dict of key determinism metrics."""
|
|
from apps.backtester.run import BacktestRunner, _build_store
|
|
from libs.backtest.manifests import load_manifest, resolve_config
|
|
|
|
manifest = load_manifest(manifest_path)
|
|
config = resolve_config(manifest, config_root=config_root, snapshot_id_override=snapshot_id)
|
|
store = _build_store(manifest, config, split)
|
|
|
|
runner = BacktestRunner(manifest=manifest, config=config, store=store, initial_equity=initial_equity)
|
|
result = runner.run(output_root=output_root)
|
|
|
|
# Build a determinism fingerprint (exclude timestamps and run_id)
|
|
return {
|
|
"trade_count": result.metrics.trade_count,
|
|
"total_return_pct": result.metrics.total_return_pct,
|
|
"max_drawdown_pct": result.metrics.max_drawdown_pct,
|
|
"win_rate": result.metrics.win_rate,
|
|
"avg_r_multiple": result.metrics.avg_r_multiple,
|
|
"total_trading_days": result.total_trading_days,
|
|
"total_candidates_seen": result.total_candidates_seen,
|
|
"total_orders_rejected": result.total_orders_rejected,
|
|
}
|
|
|
|
|
|
def run_replay(
|
|
manifest_path: str,
|
|
snapshot_id: str | None,
|
|
split: str,
|
|
output_root: str,
|
|
config_root: str,
|
|
initial_equity: float,
|
|
) -> bool:
|
|
"""Run twice and compare. Returns True if identical, False otherwise."""
|
|
logger.info("replay_run_1_start")
|
|
run1 = _run_once(manifest_path, snapshot_id, split, output_root, config_root, initial_equity)
|
|
logger.info("replay_run_1_complete", metrics=run1)
|
|
|
|
logger.info("replay_run_2_start")
|
|
run2 = _run_once(manifest_path, snapshot_id, split, output_root, config_root, initial_equity)
|
|
logger.info("replay_run_2_complete", metrics=run2)
|
|
|
|
if run1 == run2:
|
|
print("REPLAY PASS: runs are identical")
|
|
print(json.dumps(run1, indent=2))
|
|
return True
|
|
else:
|
|
print("REPLAY FAIL: runs differ!")
|
|
print("Run 1:")
|
|
print(json.dumps(run1, indent=2))
|
|
print("Run 2:")
|
|
print(json.dumps(run2, indent=2))
|
|
diff = {k: (run1.get(k), run2.get(k)) for k in set(run1) | set(run2) if run1.get(k) != run2.get(k)}
|
|
print("Differences:")
|
|
print(json.dumps(diff, indent=2))
|
|
return False
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description="ACE-F Backtest Replay / Determinism Check")
|
|
parser.add_argument("--manifest", required=True, help="Path to experiment manifest JSON")
|
|
parser.add_argument("--snapshot-id", help="Override dataset_snapshot_id")
|
|
parser.add_argument("--split", default="train")
|
|
parser.add_argument("--output-root", default="./runs")
|
|
parser.add_argument("--config-root", default=".")
|
|
parser.add_argument("--initial-equity", type=float, default=100_000.0)
|
|
args = parser.parse_args()
|
|
|
|
ok = run_replay(
|
|
manifest_path=args.manifest,
|
|
snapshot_id=args.snapshot_id,
|
|
split=args.split,
|
|
output_root=args.output_root,
|
|
config_root=args.config_root,
|
|
initial_equity=args.initial_equity,
|
|
)
|
|
sys.exit(0 if ok else 1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|