You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
179 lines
7.7 KiB
Python
179 lines
7.7 KiB
Python
"""Phase 2: engine LOO+1 (re-enable disabled engines on new snapshot) + risk_budget scaling.
|
|
|
|
The 5 disabled engines were LOO-rejected on the OLD ftb_fix_v2 snapshot.
|
|
On NEW pead_dualconv_ftb_fix_v2_probe snapshot they may behave differently.
|
|
"""
|
|
import json
|
|
import subprocess
|
|
import shutil
|
|
import copy
|
|
from pathlib import Path
|
|
|
|
BASE = "configs/experiments/return_max_long_v8.11_composed_gld_tqqq_bmc30_cl65_vol40_rmin1_cap96.json"
|
|
SWEEP_DIR = Path("configs/experiments/_tune_v811_p2")
|
|
SWEEP_DIR.mkdir(exist_ok=True)
|
|
SNAPSHOT = "pead_dualconv_ftb_fix_v2_probe"
|
|
|
|
|
|
def load_base():
|
|
with open(BASE) as f:
|
|
return json.load(f)
|
|
|
|
|
|
def save(name: str, cfg: dict) -> str:
|
|
cfg["experiment_name"] = name
|
|
path = SWEEP_DIR / f"{name}.json"
|
|
path.write_text(json.dumps(cfg, indent=2))
|
|
return str(path)
|
|
|
|
|
|
def make_loo_plus(eid_to_enable: str) -> dict:
|
|
"""Re-enable a specific disabled engine."""
|
|
cfg = load_base()
|
|
for e in cfg["strategy_engines"]:
|
|
if e.get("engine_id") == eid_to_enable and "_disabled_reason" in e:
|
|
del e["_disabled_reason"]
|
|
return cfg
|
|
raise ValueError(f"Engine {eid_to_enable} not found or not disabled")
|
|
|
|
|
|
def make_loo_minus(eid_to_disable: str) -> dict:
|
|
"""Disable an enabled engine."""
|
|
cfg = load_base()
|
|
for e in cfg["strategy_engines"]:
|
|
if e.get("engine_id") == eid_to_disable and "_disabled_reason" not in e:
|
|
e["_disabled_reason"] = "phase2_loo_minus_test"
|
|
return cfg
|
|
raise ValueError(f"Engine {eid_to_disable} not found or already disabled")
|
|
|
|
|
|
def make_rb_scale(scale_dict: dict) -> dict:
|
|
"""Scale engine_risk_budget_pct for specific engines."""
|
|
cfg = load_base()
|
|
for e in cfg["strategy_engines"]:
|
|
eid = e.get("engine_id")
|
|
if eid in scale_dict and "_disabled_reason" not in e:
|
|
if "engine_risk_budget_pct" in e:
|
|
e["engine_risk_budget_pct"] *= scale_dict[eid]
|
|
return cfg
|
|
|
|
|
|
def run_backtest(config_path: str) -> dict:
|
|
cmd = ["python3", "-m", "apps.backtester.run", "--manifest", config_path,
|
|
"--start", "2022-01-01", "--split", "all", "--initial-equity", "10000",
|
|
"--snapshot-id", SNAPSHOT]
|
|
cache = Path(f"data/parquet/{SNAPSHOT}/.runtime_cache")
|
|
if cache.exists():
|
|
shutil.rmtree(cache)
|
|
proc = subprocess.run(cmd, capture_output=True, text=True, timeout=200)
|
|
tail = proc.stdout.split("\n")[-12:]
|
|
run_id = return_pct = sqs_total = trades = None
|
|
for line in tail:
|
|
if "Run complete:" in line:
|
|
run_id = line.split("Run complete:")[1].strip()
|
|
elif "Total return:" in line:
|
|
try: return_pct = float(line.split(":")[1].strip().rstrip("%"))
|
|
except: pass
|
|
elif "SQS:" in line:
|
|
try: sqs_total = float(line.split("SQS:")[1].strip().split(" ")[0])
|
|
except: pass
|
|
elif "Trades:" in line:
|
|
try: trades = int(line.split(":")[1].strip())
|
|
except: pass
|
|
mdd = sharpe = pf = None
|
|
if run_id:
|
|
m_path = Path(f"runs/{run_id}/metrics/metrics_summary.json")
|
|
if m_path.exists():
|
|
m = json.loads(m_path.read_text())
|
|
mdd = m["max_drawdown_pct"]
|
|
sharpe = m["sharpe_ratio"]
|
|
pf = m["profit_factor"]
|
|
return {"return": return_pct, "trades": trades, "mdd": mdd, "sharpe": sharpe, "pf": pf, "sqs": sqs_total, "run_id": run_id}
|
|
|
|
|
|
DISABLED_ENGINES = [
|
|
"next_open_long_unknown_material_patient",
|
|
"next_open_long_guidance_unknown_orderly",
|
|
"next_open_long_unknown_material_orderly",
|
|
"next_open_long_other_material_mixed_orderly",
|
|
"next_open_long_unknown_ome_orderly",
|
|
]
|
|
|
|
ENABLED_ENGINES = [
|
|
"reaction_close_long_core",
|
|
"reaction_close_long_residual_lowclose_gap_d3",
|
|
"reaction_close_long_residual_smallcap_gap",
|
|
"reaction_close_long_extreme_orderly",
|
|
"next_open_long_unknown_inline_hivol",
|
|
"next_open_long_earnings_mixed_inline_orderly",
|
|
"next_open_long_megacap_material_contract_orderly",
|
|
"next_open_long_bullish_raised_recovery_broad_oneoff",
|
|
]
|
|
|
|
|
|
def main():
|
|
baseline = {"return": 6839, "mdd": 9.78, "sharpe": 3.53, "pf": 8.26, "sqs": 84.8, "trades": 329}
|
|
print(f"BASELINE v8.11: ret={baseline['return']}% MDD={baseline['mdd']}% Sharpe={baseline['sharpe']} PF={baseline['pf']} SQS={baseline['sqs']} trades={baseline['trades']}", flush=True)
|
|
print("=" * 120, flush=True)
|
|
results = []
|
|
|
|
# Phase 2A: re-enable each disabled engine
|
|
print("\n=== Phase 2A: Re-enable disabled engines (LOO+1) ===", flush=True)
|
|
for i, eid in enumerate(DISABLED_ENGINES):
|
|
name = f"L_plus_{i:02d}_{eid[:30]}"
|
|
cfg = make_loo_plus(eid)
|
|
path = save(name, cfg)
|
|
print(f"\n>>> [{i+1}/{len(DISABLED_ENGINES)}] +{eid}", flush=True)
|
|
try:
|
|
r = run_backtest(path)
|
|
r["op"] = f"+{eid}"
|
|
if r["return"] is None:
|
|
print(f" FAILED", flush=True); continue
|
|
d_ret = r["return"] - baseline["return"]
|
|
d_sqs = (r["sqs"] or 0) - baseline["sqs"]
|
|
d_mdd = (r["mdd"] or 0) - baseline["mdd"]
|
|
print(f" ret={r['return']:.0f}% (Δ{d_ret:+.0f}) MDD={r['mdd']:.2f}% (Δ{d_mdd:+.2f}) Sharpe={r['sharpe']:.2f} PF={r['pf']:.2f} SQS={r['sqs']:.1f} (Δ{d_sqs:+.1f}) trades={r['trades']}", flush=True)
|
|
results.append(r)
|
|
except Exception as e:
|
|
print(f" FAILED: {e}", flush=True)
|
|
|
|
# Phase 2B: scale top engine risk budgets
|
|
print("\n=== Phase 2B: Risk budget scaling ===", flush=True)
|
|
rb_tests = [
|
|
("core_x12", {"reaction_close_long_core": 1.2}),
|
|
("core_x08", {"reaction_close_long_core": 0.8}),
|
|
("residuals_x2", {"reaction_close_long_residual_lowclose_gap_d3": 2.0, "reaction_close_long_residual_smallcap_gap": 2.0}),
|
|
("extreme_x15", {"reaction_close_long_extreme_orderly": 1.5}),
|
|
("next_open_x3", {"next_open_long_unknown_inline_hivol": 3.0, "next_open_long_earnings_mixed_inline_orderly": 3.0, "next_open_long_megacap_material_contract_orderly": 3.0, "next_open_long_bullish_raised_recovery_broad_oneoff": 3.0}),
|
|
("hivol_x4", {"next_open_long_unknown_inline_hivol": 4.0}),
|
|
]
|
|
for i, (label, scale_dict) in enumerate(rb_tests):
|
|
name = f"RB_{label}"
|
|
cfg = make_rb_scale(scale_dict)
|
|
path = save(name, cfg)
|
|
print(f"\n>>> RB scale: {label} {scale_dict}", flush=True)
|
|
try:
|
|
r = run_backtest(path)
|
|
r["op"] = f"RB:{label}"
|
|
if r["return"] is None:
|
|
print(f" FAILED", flush=True); continue
|
|
d_ret = r["return"] - baseline["return"]
|
|
d_sqs = (r["sqs"] or 0) - baseline["sqs"]
|
|
d_mdd = (r["mdd"] or 0) - baseline["mdd"]
|
|
print(f" ret={r['return']:.0f}% (Δ{d_ret:+.0f}) MDD={r['mdd']:.2f}% (Δ{d_mdd:+.2f}) Sharpe={r['sharpe']:.2f} PF={r['pf']:.2f} SQS={r['sqs']:.1f} (Δ{d_sqs:+.1f}) trades={r['trades']}", flush=True)
|
|
results.append(r)
|
|
except Exception as e:
|
|
print(f" FAILED: {e}", flush=True)
|
|
|
|
print("\n" + "=" * 120, flush=True)
|
|
print("SUMMARY (sorted by SQS, then return):", flush=True)
|
|
results.sort(key=lambda x: (x.get("sqs") or 0, x.get("return") or 0), reverse=True)
|
|
print(f"{'op':50s} {'return':>10s} {'MDD':>8s} {'Sharpe':>8s} {'PF':>6s} {'SQS':>7s} {'trades':>7s}", flush=True)
|
|
print(f"{'BASELINE':50s} {baseline['return']:>9.0f}% {baseline['mdd']:>7.2f}% {baseline['sharpe']:>8.2f} {baseline['pf']:>6.2f} {baseline['sqs']:>7.1f} {baseline['trades']:>7d}", flush=True)
|
|
for r in results:
|
|
print(f"{r['op'][:48]:50s} {r['return']:>9.0f}% {r['mdd']:>7.2f}% {r['sharpe']:>8.2f} {r['pf']:>6.2f} {r['sqs']:>7.1f} {r['trades']:>7d}", flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|