You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
129 lines
4.3 KiB
Python
129 lines
4.3 KiB
Python
#!/usr/bin/env python3
|
|
"""DeepEval Phase 1: Run WFV comparison matrix (clean lineage).
|
|
|
|
Usage:
|
|
python -m apps.tools.run_deepeval_phase1 [--output-root runs/deepeval_phase1]
|
|
|
|
Runs walk-forward validation for:
|
|
1.1 Clean lineage baseline
|
|
1.2 Clean lineage replay of legacy v0.326
|
|
|
|
Then prints a comparison matrix.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
from concurrent.futures import ProcessPoolExecutor, as_completed
|
|
|
|
|
|
CONFIGS = {
|
|
"v1.1_clean_baseline": "configs/experiments/return_max_long_v1.1.json",
|
|
"v1.2_clean_replay": "configs/experiments/return_max_long_v1.2.json",
|
|
}
|
|
|
|
WF_TRAIN_DAYS = 504
|
|
WF_TEST_DAYS = 63
|
|
|
|
|
|
def run_single_wfv(label: str, manifest_path: str, output_root: str) -> dict:
|
|
"""Run a single WFV and return the summary dict."""
|
|
out_dir = str(Path(output_root) / label)
|
|
cmd = [
|
|
sys.executable, "-m", "apps.backtester.run",
|
|
"--manifest", manifest_path,
|
|
"--walk-forward",
|
|
"--wf-train-days", str(WF_TRAIN_DAYS),
|
|
"--wf-test-days", str(WF_TEST_DAYS),
|
|
"--output-root", out_dir,
|
|
"--initial-equity", "100000",
|
|
"--snapshot-dir", "data/datasets/snapshots",
|
|
]
|
|
print(f"[{label}] Starting WFV...")
|
|
result = subprocess.run(cmd, capture_output=True, text=True)
|
|
if result.returncode != 0:
|
|
print(f"[{label}] FAILED:\n{result.stderr[-500:]}")
|
|
return {"label": label, "error": result.stderr[-500:]}
|
|
|
|
summary_path = Path(out_dir) / "walk_forward" / "walk_forward_summary.json"
|
|
if not summary_path.exists():
|
|
print(f"[{label}] WARNING: summary not found at {summary_path}")
|
|
return {"label": label, "error": "summary not found"}
|
|
|
|
with open(summary_path) as f:
|
|
summary = json.load(f)
|
|
print(f"[{label}] Done. Test mean return: {summary['test_aggregate']['mean_return_pct']:.1f}%")
|
|
return {"label": label, "summary": summary}
|
|
|
|
|
|
def print_comparison_matrix(results: list[dict]) -> None:
|
|
"""Print a formatted comparison matrix."""
|
|
print("\n" + "=" * 90)
|
|
print("DeepEval Phase 1 — WFV Comparison Matrix")
|
|
print("=" * 90)
|
|
|
|
header = f"{'Config':<25} {'Test Mean%':>10} {'Test Med%':>10} {'Test Worst%':>11} {'Train-Test Gap':>14} {'Test Sharpe':>11}"
|
|
print(header)
|
|
print("-" * 90)
|
|
|
|
for r in results:
|
|
label = r["label"]
|
|
if "error" in r:
|
|
print(f"{label:<25} {'ERROR':>10}")
|
|
continue
|
|
s = r["summary"]
|
|
ta = s["test_aggregate"]
|
|
gap = s["gap_stats"]
|
|
print(
|
|
f"{label:<25} "
|
|
f"{ta['mean_return_pct']:>10.1f} "
|
|
f"{ta['median_return_pct']:>10.1f} "
|
|
f"{ta['worst_return_pct']:>11.1f} "
|
|
f"{gap['mean_train_test_return_gap_pct']:>14.1f} "
|
|
f"{ta.get('mean_profit_factor', 0):>11.1f}"
|
|
)
|
|
|
|
print("=" * 90)
|
|
print("\nKey interpretation:")
|
|
print(" - v1.1 is the clean-lineage baseline")
|
|
print(" - v1.2 is the clean-lineage replay of legacy v0.326 non-exact core")
|
|
print(" - Train-Test Gap: lower is better (less overfitting)")
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description="DeepEval Phase 1: WFV Comparison Matrix")
|
|
parser.add_argument("--output-root", default="runs/deepeval_phase1", help="Output directory")
|
|
parser.add_argument("--parallel", type=int, default=2, help="Max parallel WFV runs")
|
|
args = parser.parse_args()
|
|
|
|
results = []
|
|
if args.parallel > 1:
|
|
with ProcessPoolExecutor(max_workers=args.parallel) as executor:
|
|
futures = {
|
|
executor.submit(run_single_wfv, label, path, args.output_root): label
|
|
for label, path in CONFIGS.items()
|
|
}
|
|
for future in as_completed(futures):
|
|
results.append(future.result())
|
|
else:
|
|
for label, path in CONFIGS.items():
|
|
results.append(run_single_wfv(label, path, args.output_root))
|
|
|
|
# Sort by label for consistent output
|
|
results.sort(key=lambda r: r["label"])
|
|
print_comparison_matrix(results)
|
|
|
|
# Save raw results
|
|
out_path = Path(args.output_root) / "phase1_comparison.json"
|
|
out_path.parent.mkdir(parents=True, exist_ok=True)
|
|
with open(out_path, "w") as f:
|
|
json.dump(results, f, indent=2, default=str)
|
|
print(f"\nRaw results saved to {out_path}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|