You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
90 lines
3.0 KiB
Python
90 lines
3.0 KiB
Python
#!/usr/bin/env python3
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description="Run train/valid/test backtests for multiple manifests.")
|
|
parser.add_argument("--manifest", action="append", required=True, help="Experiment manifest path. Repeat for multiple manifests.")
|
|
parser.add_argument("--snapshot-dir", default="data/datasets/snapshots", help="Snapshot root directory.")
|
|
parser.add_argument("--output-root", required=True, help="Root directory for run artifacts.")
|
|
parser.add_argument("--summary-path", required=True, help="Path to write summary JSON.")
|
|
return parser.parse_args()
|
|
|
|
|
|
def run_split(root: Path, manifest: str, split: str, output_root: Path, snapshot_dir: str) -> dict[str, object]:
|
|
cmd = [
|
|
sys.executable,
|
|
"apps/backtester/run.py",
|
|
"--manifest",
|
|
manifest,
|
|
"--snapshot-dir",
|
|
snapshot_dir,
|
|
"--split",
|
|
split,
|
|
"--output-root",
|
|
str(output_root),
|
|
]
|
|
result = subprocess.run(
|
|
cmd,
|
|
cwd=root,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT,
|
|
text=True,
|
|
check=False,
|
|
)
|
|
tail = result.stdout.splitlines()[-5:] if result.stdout else []
|
|
for line in tail:
|
|
print(line)
|
|
if result.returncode != 0:
|
|
raise RuntimeError(f"{Path(manifest).stem} {split} failed:\n{result.stdout}")
|
|
|
|
latest = max(output_root.glob("bt_*"), key=lambda path: path.stat().st_mtime)
|
|
summary = json.loads((latest / "metrics" / "metrics_summary.json").read_text())
|
|
return {
|
|
"run_id": latest.name,
|
|
"ret": summary["total_return_pct"],
|
|
"dd": summary["max_drawdown_pct"],
|
|
"n": summary["trade_count"],
|
|
"pf": summary["profit_factor"],
|
|
}
|
|
|
|
|
|
def main() -> None:
|
|
args = parse_args()
|
|
root = Path(__file__).resolve().parents[2]
|
|
output_root = Path(args.output_root)
|
|
output_root.mkdir(parents=True, exist_ok=True)
|
|
|
|
rows: list[dict[str, object]] = []
|
|
for manifest in args.manifest:
|
|
exp_name = Path(manifest).stem
|
|
print(f"RUN {exp_name}", flush=True)
|
|
exp_root = output_root / exp_name
|
|
exp_root.mkdir(parents=True, exist_ok=True)
|
|
metrics = {
|
|
split: run_split(root, manifest, split, exp_root, args.snapshot_dir)
|
|
for split in ("train", "valid", "test")
|
|
}
|
|
rows.append({"experiment": exp_name, "metrics": metrics})
|
|
|
|
summary_path = Path(args.summary_path)
|
|
summary_path.parent.mkdir(parents=True, exist_ok=True)
|
|
summary_path.write_text(json.dumps(rows, indent=2) + "\n")
|
|
print(f"WROTE {summary_path}")
|
|
for row in rows:
|
|
metrics = row["metrics"]
|
|
print(
|
|
row["experiment"],
|
|
f"tr={metrics['train']['ret']:.2f} v={metrics['valid']['ret']:.2f} t={metrics['test']['ret']:.2f}",
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|