|
|
"""BacktestRunner: main simulation class and CLI entry point."""
|
|
|
from __future__ import annotations
|
|
|
|
|
|
import argparse
|
|
|
import datetime as dt
|
|
|
import json
|
|
|
import subprocess
|
|
|
import sys
|
|
|
from pathlib import Path
|
|
|
from typing import Any
|
|
|
|
|
|
from libs.backtest.allocator import build_planned_order
|
|
|
from libs.backtest.artifacts import create_run_directory, write_all_artifacts
|
|
|
from libs.backtest.domain import (
|
|
|
BacktestConfig,
|
|
|
Candidate,
|
|
|
DailyPortfolioState,
|
|
|
ExperimentManifest,
|
|
|
ExperimentResult,
|
|
|
FilledTrade,
|
|
|
MetricsBundle,
|
|
|
OpenPosition,
|
|
|
PositionStatus,
|
|
|
)
|
|
|
from libs.backtest.execution import (
|
|
|
simulate_entry,
|
|
|
simulate_exit,
|
|
|
simulate_kill_switch_exit,
|
|
|
update_trailing_stop,
|
|
|
)
|
|
|
from libs.backtest.manifests import generate_run_id, load_manifest, resolve_config
|
|
|
from libs.backtest.metrics import build_metrics_bundle
|
|
|
from libs.backtest.selector import select_candidates
|
|
|
from libs.backtest.snapshot_store import SnapshotStore
|
|
|
from libs.backtest.splits import generate_walk_forward_windows
|
|
|
from libs.common.logging import get_logger
|
|
|
from libs.common.time_utils import utc_now
|
|
|
|
|
|
logger = get_logger(__name__)
|
|
|
|
|
|
_KILL_SWITCH_DRAWDOWN_PCT = 25.0
|
|
|
|
|
|
|
|
|
def _get_git_commit_hash() -> str:
|
|
|
try:
|
|
|
result = subprocess.run(
|
|
|
["git", "rev-parse", "--short", "HEAD"],
|
|
|
capture_output=True, text=True, timeout=5,
|
|
|
)
|
|
|
return result.stdout.strip() or "unknown"
|
|
|
except Exception:
|
|
|
return "unknown"
|
|
|
|
|
|
|
|
|
class BacktestRunner:
|
|
|
"""Event-driven backtester simulation engine."""
|
|
|
|
|
|
def __init__(
|
|
|
self,
|
|
|
manifest: ExperimentManifest,
|
|
|
config: BacktestConfig,
|
|
|
store: SnapshotStore,
|
|
|
initial_equity: float = 100_000.0,
|
|
|
split_name: str | None = None,
|
|
|
) -> None:
|
|
|
self.split_name = split_name
|
|
|
self.manifest = manifest
|
|
|
self.config = config
|
|
|
self.store = store
|
|
|
self.initial_equity = initial_equity
|
|
|
|
|
|
# Simulation state
|
|
|
self._equity = initial_equity
|
|
|
self._cash = initial_equity
|
|
|
self._open_positions: list[OpenPosition] = []
|
|
|
self._closed_trades: list[FilledTrade] = []
|
|
|
self._equity_curve: list[DailyPortfolioState] = []
|
|
|
self._candidate_map: dict[str, Candidate] = {} # trade_id → candidate
|
|
|
|
|
|
# Stats
|
|
|
self._total_candidates_seen = 0
|
|
|
self._total_orders_rejected = 0
|
|
|
self._peak_equity = initial_equity
|
|
|
self._realized_pnl = 0.0
|
|
|
self._daily_new_risk_used = 0.0
|
|
|
self._consecutive_losses = 0
|
|
|
self._cooldown_remaining = 0
|
|
|
self._kill_switch_triggered = False
|
|
|
self._kill_switch_cooldown_remaining = 0
|
|
|
|
|
|
def run(self, output_root: str | Path | None = None) -> ExperimentResult:
|
|
|
"""Execute the full simulation. Returns ExperimentResult."""
|
|
|
started_at = utc_now()
|
|
|
run_id = generate_run_id(self.config)
|
|
|
logger.info("backtest_start", run_id=run_id, strategy=self.config.strategy_name)
|
|
|
|
|
|
exec_dates = self.store.all_execution_dates()
|
|
|
if not exec_dates:
|
|
|
logger.warning("backtest_no_dates", run_id=run_id)
|
|
|
|
|
|
# Iterate ALL trading days (not just candidate days) so stop/target/time
|
|
|
# exits are checked every day, not just on days with new candidates.
|
|
|
all_dates = self.store.all_trading_days()
|
|
|
|
|
|
# Record initial equity state (before any trades)
|
|
|
if all_dates:
|
|
|
self._equity_curve.append(
|
|
|
DailyPortfolioState(
|
|
|
date=all_dates[0],
|
|
|
equity=self.initial_equity,
|
|
|
cash_available=self.initial_equity,
|
|
|
gross_exposure=0.0,
|
|
|
net_exposure=0.0,
|
|
|
reserved_risk_budget=0.0,
|
|
|
unrealized_pnl=0.0,
|
|
|
realized_pnl=0.0,
|
|
|
open_positions=[],
|
|
|
daily_new_risk_used=0.0,
|
|
|
peak_equity=self.initial_equity,
|
|
|
current_drawdown_pct=0.0,
|
|
|
)
|
|
|
)
|
|
|
|
|
|
for date in all_dates:
|
|
|
self._simulate_day(date)
|
|
|
|
|
|
# Force-close any remaining open positions at end of backtest
|
|
|
if self._open_positions:
|
|
|
last_date = all_dates[-1] if all_dates else dt.date.today()
|
|
|
self._force_close_all(last_date, reason="end_of_backtest")
|
|
|
|
|
|
finished_at = utc_now()
|
|
|
metrics = build_metrics_bundle(
|
|
|
self._closed_trades, self._equity_curve, self._candidate_map
|
|
|
)
|
|
|
|
|
|
# Create run directory and write artifacts
|
|
|
run_dir = None
|
|
|
artifact_paths: dict[str, str] = {}
|
|
|
if output_root is not None:
|
|
|
run_dir = create_run_directory(output_root, run_id)
|
|
|
git_hash = _get_git_commit_hash()
|
|
|
artifact_paths = write_all_artifacts(
|
|
|
run_dir=run_dir,
|
|
|
run_id=run_id,
|
|
|
manifest=self.manifest,
|
|
|
config=self.config,
|
|
|
metrics=metrics,
|
|
|
trades=self._closed_trades,
|
|
|
equity_curve=self._equity_curve,
|
|
|
open_positions=self._open_positions,
|
|
|
candidate_map=self._candidate_map,
|
|
|
started_at=started_at,
|
|
|
finished_at=finished_at,
|
|
|
git_hash=git_hash,
|
|
|
total_trading_days=len(self._equity_curve),
|
|
|
total_candidates_seen=self._total_candidates_seen,
|
|
|
total_orders_rejected=self._total_orders_rejected,
|
|
|
split_name=self.split_name,
|
|
|
)
|
|
|
|
|
|
logger.info(
|
|
|
"backtest_complete",
|
|
|
run_id=run_id,
|
|
|
trades=len(self._closed_trades),
|
|
|
days=len(self._equity_curve),
|
|
|
)
|
|
|
|
|
|
return ExperimentResult(
|
|
|
run_id=run_id,
|
|
|
manifest=self.manifest,
|
|
|
resolved_config=self.config,
|
|
|
metrics=metrics,
|
|
|
artifact_paths=artifact_paths,
|
|
|
started_at=started_at,
|
|
|
finished_at=finished_at,
|
|
|
total_trading_days=len(self._equity_curve),
|
|
|
total_candidates_seen=self._total_candidates_seen,
|
|
|
total_orders_rejected=self._total_orders_rejected,
|
|
|
)
|
|
|
|
|
|
def _simulate_day(self, date: dt.date) -> None:
|
|
|
"""Simulate a single trading day."""
|
|
|
# Reset daily risk tracker
|
|
|
self._daily_new_risk_used = 0.0
|
|
|
|
|
|
# Decrement cooldowns
|
|
|
if self._cooldown_remaining > 0:
|
|
|
self._cooldown_remaining -= 1
|
|
|
if self._kill_switch_cooldown_remaining > 0:
|
|
|
self._kill_switch_cooldown_remaining -= 1
|
|
|
|
|
|
# Increment days_held for all open positions
|
|
|
for pos in self._open_positions:
|
|
|
pos.days_held += 1
|
|
|
|
|
|
# --- EXITS FIRST (using today's OHLCV) ---
|
|
|
# Build position → candidate lookup for attribution mapping
|
|
|
pos_to_candidate = {pos.position_id: pos.plan.candidate for pos in self._open_positions}
|
|
|
|
|
|
newly_closed: list[FilledTrade] = []
|
|
|
still_open: list[OpenPosition] = []
|
|
|
|
|
|
for pos in self._open_positions:
|
|
|
bar = self.store.get_bar(pos.plan.candidate.symbol, date)
|
|
|
|
|
|
# Kill switch: force close
|
|
|
if self._kill_switch_triggered:
|
|
|
trade = simulate_kill_switch_exit(pos, bar, date, self.config.execution)
|
|
|
newly_closed.append(trade)
|
|
|
continue
|
|
|
|
|
|
if bar is None:
|
|
|
# Missing bar — hold position (do not impute zero)
|
|
|
still_open.append(pos)
|
|
|
continue
|
|
|
|
|
|
# Update trailing stop if configured
|
|
|
if self.config.execution.trailing_model:
|
|
|
update_trailing_stop(
|
|
|
pos, bar,
|
|
|
self.config.execution.trailing_model,
|
|
|
warmup_days=self.config.execution.trailing_warmup_days,
|
|
|
)
|
|
|
|
|
|
# Build effective execution config with per-event-type overrides
|
|
|
effective_exec = self.config.execution
|
|
|
evt_profile = self.config.get_event_profile(pos.plan.candidate.event_type)
|
|
|
if evt_profile and evt_profile.max_holding_days_override is not None:
|
|
|
effective_exec = self.config.execution.model_copy(
|
|
|
update={"max_holding_days": evt_profile.max_holding_days_override}
|
|
|
)
|
|
|
|
|
|
prev_status = pos.status
|
|
|
trade = simulate_exit(pos, bar, effective_exec, date)
|
|
|
if trade is not None:
|
|
|
newly_closed.append(trade)
|
|
|
# Partial exit: status just changed from ENTERED to PARTIALLY_EXITED
|
|
|
# Keep position open for remaining shares
|
|
|
if prev_status == PositionStatus.ENTERED and pos.status == PositionStatus.PARTIALLY_EXITED:
|
|
|
still_open.append(pos)
|
|
|
else:
|
|
|
still_open.append(pos)
|
|
|
|
|
|
# Process closed trades
|
|
|
for trade in newly_closed:
|
|
|
self._closed_trades.append(trade)
|
|
|
# Map trade to candidate for attribution
|
|
|
cand = pos_to_candidate.get(trade.position_id)
|
|
|
if cand:
|
|
|
self._candidate_map[trade.trade_id] = cand
|
|
|
self._realized_pnl += trade.net_pnl
|
|
|
self._cash += trade.net_pnl + (trade.entry_price * trade.shares)
|
|
|
# Track consecutive losses for cooldown
|
|
|
if trade.net_pnl < 0:
|
|
|
self._consecutive_losses += 1
|
|
|
else:
|
|
|
self._consecutive_losses = 0
|
|
|
if (
|
|
|
self.config.risk.cooldown_after_loss_streak > 0
|
|
|
and self._consecutive_losses >= self.config.risk.cooldown_after_loss_streak
|
|
|
):
|
|
|
self._cooldown_remaining = self.config.risk.cooldown_days
|
|
|
self._consecutive_losses = 0
|
|
|
|
|
|
self._open_positions = still_open
|
|
|
|
|
|
# --- Compute current equity for kill-switch check ---
|
|
|
market_value = self._compute_positions_market_value(date)
|
|
|
unrealized = market_value - sum(
|
|
|
p.entry_price * p.shares_open for p in self._open_positions
|
|
|
)
|
|
|
self._equity = self._cash + market_value
|
|
|
self._peak_equity = max(self._peak_equity, self._equity)
|
|
|
drawdown_pct = (
|
|
|
(self._peak_equity - self._equity) / self._peak_equity * 100.0
|
|
|
if self._peak_equity > 0
|
|
|
else 0.0
|
|
|
)
|
|
|
|
|
|
if drawdown_pct >= _KILL_SWITCH_DRAWDOWN_PCT and not self._kill_switch_triggered:
|
|
|
logger.warning("kill_switch_triggered", date=str(date), drawdown_pct=drawdown_pct)
|
|
|
if self.config.risk.kill_switch_log_only:
|
|
|
logger.info("kill_switch_log_only_mode", date=str(date))
|
|
|
# Don't trigger — just observe
|
|
|
else:
|
|
|
self._kill_switch_triggered = True
|
|
|
if self.config.risk.backtest_mode == "research":
|
|
|
self._kill_switch_cooldown_remaining = self.config.risk.kill_switch_cooldown_days
|
|
|
|
|
|
# Research mode: reset kill switch after cooldown expires
|
|
|
# Reset peak_equity to current equity so drawdown restarts from 0
|
|
|
if (
|
|
|
self._kill_switch_triggered
|
|
|
and self.config.risk.backtest_mode == "research"
|
|
|
and self._kill_switch_cooldown_remaining <= 0
|
|
|
):
|
|
|
self._kill_switch_triggered = False
|
|
|
self._peak_equity = self._equity
|
|
|
drawdown_pct = 0.0
|
|
|
logger.info("kill_switch_reset", date=str(date))
|
|
|
|
|
|
# --- ENTRIES (only if kill switch not triggered) ---
|
|
|
if not self._kill_switch_triggered:
|
|
|
raw_rows = self.store.get_candidates_for_date(date)
|
|
|
self._total_candidates_seen += len(raw_rows)
|
|
|
portfolio_state = self._build_portfolio_state(date, drawdown_pct, unrealized)
|
|
|
candidates = select_candidates(
|
|
|
raw_rows, self.config.universe, self.config.signal,
|
|
|
event_type_profiles=self.config.event_type_profiles or None,
|
|
|
)
|
|
|
|
|
|
macro_data = self.store.get_macro_for_date(date)
|
|
|
|
|
|
for candidate in candidates:
|
|
|
plan = build_planned_order(
|
|
|
candidate=candidate,
|
|
|
portfolio_state=portfolio_state,
|
|
|
open_positions=self._open_positions,
|
|
|
config=self.config,
|
|
|
cooldown_remaining=self._cooldown_remaining,
|
|
|
macro_data=macro_data,
|
|
|
)
|
|
|
|
|
|
if plan.skip_reason is not None:
|
|
|
self._total_orders_rejected += 1
|
|
|
logger.debug(
|
|
|
"order_rejected",
|
|
|
symbol=candidate.symbol,
|
|
|
reason=plan.skip_reason,
|
|
|
date=str(date),
|
|
|
)
|
|
|
continue
|
|
|
|
|
|
bar = self.store.get_bar(candidate.symbol, date)
|
|
|
pos = simulate_entry(plan, bar, self.config.execution)
|
|
|
if pos is not None:
|
|
|
self._open_positions.append(pos)
|
|
|
self._cash -= pos.entry_price * pos.shares_total
|
|
|
self._daily_new_risk_used += plan.risk_dollars
|
|
|
# Update equity and portfolio state for next candidate
|
|
|
mv = self._compute_positions_market_value(date)
|
|
|
self._equity = self._cash + mv
|
|
|
ur = mv - sum(
|
|
|
p.entry_price * p.shares_open
|
|
|
for p in self._open_positions
|
|
|
)
|
|
|
portfolio_state = self._build_portfolio_state(
|
|
|
date, drawdown_pct, ur
|
|
|
)
|
|
|
|
|
|
# --- Record daily equity curve snapshot ---
|
|
|
market_value_final = self._compute_positions_market_value(date)
|
|
|
unrealized_final = market_value_final - sum(
|
|
|
p.entry_price * p.shares_open for p in self._open_positions
|
|
|
)
|
|
|
self._equity = self._cash + market_value_final
|
|
|
self._peak_equity = max(self._peak_equity, self._equity)
|
|
|
final_drawdown = (
|
|
|
(self._peak_equity - self._equity) / self._peak_equity * 100.0
|
|
|
if self._peak_equity > 0
|
|
|
else 0.0
|
|
|
)
|
|
|
self._equity_curve.append(
|
|
|
DailyPortfolioState(
|
|
|
date=date,
|
|
|
equity=self._equity,
|
|
|
cash_available=self._cash,
|
|
|
gross_exposure=sum(
|
|
|
p.entry_price * p.shares_open for p in self._open_positions
|
|
|
),
|
|
|
net_exposure=sum(
|
|
|
p.entry_price * p.shares_open for p in self._open_positions
|
|
|
),
|
|
|
reserved_risk_budget=self._daily_new_risk_used,
|
|
|
unrealized_pnl=unrealized_final,
|
|
|
realized_pnl=self._realized_pnl,
|
|
|
open_positions=[p.position_id for p in self._open_positions],
|
|
|
daily_new_risk_used=self._daily_new_risk_used,
|
|
|
peak_equity=self._peak_equity,
|
|
|
current_drawdown_pct=final_drawdown,
|
|
|
)
|
|
|
)
|
|
|
|
|
|
def _compute_positions_market_value(self, date: dt.date) -> float:
|
|
|
"""Market value of all open positions using today's close.
|
|
|
|
|
|
For long: market_value = close * shares.
|
|
|
For short: market_value = (2 * entry - close) * shares.
|
|
|
This reflects that a short position gains when price falls:
|
|
|
the "value" of a short at entry is entry_price * shares,
|
|
|
and PnL = (entry - close) * shares, so effective value = entry + PnL = (2*entry - close).
|
|
|
|
|
|
Falls back to entry_price when bar is missing (assumes no change
|
|
|
rather than treating the position as worthless).
|
|
|
"""
|
|
|
total = 0.0
|
|
|
for pos in self._open_positions:
|
|
|
bar = self.store.get_bar(pos.plan.candidate.symbol, date)
|
|
|
is_short = pos.plan.candidate.trade_direction == "short"
|
|
|
if bar and bar.get("close"):
|
|
|
close = float(bar["close"])
|
|
|
if is_short:
|
|
|
total += (2.0 * pos.entry_price - close) * pos.shares_open
|
|
|
else:
|
|
|
total += close * pos.shares_open
|
|
|
else:
|
|
|
total += pos.entry_price * pos.shares_open
|
|
|
return total
|
|
|
|
|
|
def _compute_unrealized_pnl(self, date: dt.date) -> float:
|
|
|
"""Unrealized PnL = market_value − cost_basis."""
|
|
|
market_value = self._compute_positions_market_value(date)
|
|
|
cost_basis = sum(p.entry_price * p.shares_open for p in self._open_positions)
|
|
|
return market_value - cost_basis
|
|
|
|
|
|
def _build_portfolio_state(
|
|
|
self,
|
|
|
date: dt.date,
|
|
|
drawdown_pct: float,
|
|
|
unrealized: float,
|
|
|
) -> DailyPortfolioState:
|
|
|
return DailyPortfolioState(
|
|
|
date=date,
|
|
|
equity=self._equity,
|
|
|
cash_available=self._cash,
|
|
|
gross_exposure=sum(
|
|
|
p.entry_price * p.shares_open for p in self._open_positions
|
|
|
),
|
|
|
net_exposure=sum(
|
|
|
p.entry_price * p.shares_open for p in self._open_positions
|
|
|
),
|
|
|
reserved_risk_budget=self._daily_new_risk_used,
|
|
|
unrealized_pnl=unrealized,
|
|
|
realized_pnl=self._realized_pnl,
|
|
|
open_positions=[p.position_id for p in self._open_positions],
|
|
|
daily_new_risk_used=self._daily_new_risk_used,
|
|
|
peak_equity=self._peak_equity,
|
|
|
current_drawdown_pct=drawdown_pct,
|
|
|
)
|
|
|
|
|
|
def _force_close_all(self, date: dt.date, reason: str = "force_close") -> None:
|
|
|
"""Close all open positions (end of backtest or kill switch)."""
|
|
|
for pos in list(self._open_positions):
|
|
|
bar = self.store.get_bar(pos.plan.candidate.symbol, date)
|
|
|
trade = simulate_kill_switch_exit(pos, bar, date, self.config.execution)
|
|
|
self._closed_trades.append(trade)
|
|
|
self._candidate_map[trade.trade_id] = pos.plan.candidate
|
|
|
self._realized_pnl += trade.net_pnl
|
|
|
self._cash += trade.net_pnl + (trade.entry_price * trade.shares)
|
|
|
self._open_positions = []
|
|
|
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
|
# CLI
|
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
|
|
|
def _build_store(
|
|
|
manifest: ExperimentManifest,
|
|
|
config: BacktestConfig,
|
|
|
split_name: str,
|
|
|
snapshot_dir_override: str | None = None,
|
|
|
) -> SnapshotStore:
|
|
|
from libs.common.config import get_settings
|
|
|
|
|
|
s = get_settings()
|
|
|
snapshot_dir = Path(snapshot_dir_override or s.parquet_dir) / config.dataset_snapshot_id
|
|
|
|
|
|
# Resolve scoring function from config
|
|
|
scoring_fn = None
|
|
|
if config.signal.scoring_model == "pead":
|
|
|
from libs.backtest.scoring import compute_pead_score
|
|
|
from functools import partial
|
|
|
|
|
|
scoring_fn = partial(
|
|
|
compute_pead_score,
|
|
|
reaction_threshold=config.signal.pead_reaction_threshold,
|
|
|
volume_threshold=config.signal.pead_volume_threshold,
|
|
|
)
|
|
|
|
|
|
return SnapshotStore.load(
|
|
|
snapshot_dir=snapshot_dir,
|
|
|
split_name=split_name,
|
|
|
oracle_url=s.stock_oracle_url,
|
|
|
db_dsn=s.postgres_dsn,
|
|
|
scoring_fn=scoring_fn,
|
|
|
)
|
|
|
|
|
|
|
|
|
def run_walk_forward(
|
|
|
manifest: ExperimentManifest,
|
|
|
config: BacktestConfig,
|
|
|
snapshot_dir_override: str | None,
|
|
|
initial_equity: float,
|
|
|
output_root: str,
|
|
|
train_days: int = 252,
|
|
|
test_days: int = 63,
|
|
|
step_days: int | None = None,
|
|
|
) -> list[dict[str, Any]]:
|
|
|
"""Run walk-forward cross-validation over all splits.
|
|
|
|
|
|
Loads all three splits (train/valid/test) into one SnapshotStore,
|
|
|
generates walk-forward windows, and runs a separate backtest on each
|
|
|
window's test period. Returns per-fold metrics.
|
|
|
"""
|
|
|
from libs.common.config import get_settings
|
|
|
|
|
|
s = get_settings()
|
|
|
snapshot_root = Path(snapshot_dir_override or s.parquet_dir) / config.dataset_snapshot_id
|
|
|
|
|
|
# Merge all splits into a single SnapshotStore
|
|
|
stores: list[SnapshotStore] = []
|
|
|
for split in ["train", "valid", "test"]:
|
|
|
parquet_path = snapshot_root / f"{split}.parquet"
|
|
|
if parquet_path.exists():
|
|
|
st = SnapshotStore.load(
|
|
|
snapshot_dir=snapshot_root,
|
|
|
split_name=split,
|
|
|
oracle_url=s.stock_oracle_url,
|
|
|
db_dsn=s.postgres_dsn,
|
|
|
)
|
|
|
stores.append(st)
|
|
|
|
|
|
if not stores:
|
|
|
print("No splits found to load.")
|
|
|
return []
|
|
|
|
|
|
# Merge candidates and bars from all stores
|
|
|
merged_candidates: dict[dt.date, list[dict[str, Any]]] = {}
|
|
|
merged_bars: dict[str, dict[dt.date, dict[str, Any]]] = {}
|
|
|
merged_macro: dict[dt.date, dict[str, Any]] = {}
|
|
|
|
|
|
for st in stores:
|
|
|
for d in st.all_execution_dates():
|
|
|
merged_candidates.setdefault(d, []).extend(st.get_candidates_for_date(d))
|
|
|
merged_bars.update(st._bars)
|
|
|
merged_macro.update(st._macro)
|
|
|
|
|
|
merged_store = SnapshotStore(
|
|
|
candidates_by_exec_date=merged_candidates,
|
|
|
bars_by_symbol_date=merged_bars,
|
|
|
macro_by_date=merged_macro,
|
|
|
)
|
|
|
|
|
|
all_dates = merged_store.all_trading_days()
|
|
|
if not all_dates:
|
|
|
print("No trading days found in merged data.")
|
|
|
return []
|
|
|
|
|
|
windows = generate_walk_forward_windows(
|
|
|
all_dates,
|
|
|
train_days=train_days,
|
|
|
test_days=test_days,
|
|
|
step_days=step_days,
|
|
|
)
|
|
|
|
|
|
if not windows:
|
|
|
print(f"Not enough data for walk-forward windows (need {train_days + test_days} days, have {len(all_dates)}).")
|
|
|
# Fall back to a single window using all available data
|
|
|
print("Running single-window backtest on all data instead.")
|
|
|
windows_dates = [(all_dates[0], all_dates[-1])]
|
|
|
else:
|
|
|
windows_dates = [(w.test_start, w.test_end) for w in windows]
|
|
|
|
|
|
fold_results: list[dict[str, Any]] = []
|
|
|
|
|
|
for fold_idx, (test_start, test_end) in enumerate(windows_dates):
|
|
|
# Filter candidates to only those within the test window
|
|
|
filtered_candidates: dict[dt.date, list[dict[str, Any]]] = {}
|
|
|
for d, cands in merged_candidates.items():
|
|
|
if test_start <= d <= test_end:
|
|
|
filtered_candidates[d] = cands
|
|
|
|
|
|
fold_store = SnapshotStore(
|
|
|
candidates_by_exec_date=filtered_candidates,
|
|
|
bars_by_symbol_date=merged_bars,
|
|
|
macro_by_date=merged_macro,
|
|
|
)
|
|
|
|
|
|
runner = BacktestRunner(
|
|
|
manifest=manifest,
|
|
|
config=config,
|
|
|
store=fold_store,
|
|
|
initial_equity=initial_equity,
|
|
|
)
|
|
|
result = runner.run(output_root=output_root)
|
|
|
|
|
|
fold_info = {
|
|
|
"fold": fold_idx,
|
|
|
"test_start": str(test_start),
|
|
|
"test_end": str(test_end),
|
|
|
"trade_count": result.metrics.trade_count,
|
|
|
"total_return_pct": result.metrics.total_return_pct,
|
|
|
"win_rate": result.metrics.win_rate,
|
|
|
"profit_factor": result.metrics.profit_factor,
|
|
|
"max_drawdown_pct": result.metrics.max_drawdown_pct,
|
|
|
"sharpe_ratio": result.metrics.sharpe_ratio,
|
|
|
"run_id": result.run_id,
|
|
|
}
|
|
|
fold_results.append(fold_info)
|
|
|
print(
|
|
|
f"Fold {fold_idx}: {test_start}→{test_end} | "
|
|
|
f"Trades={result.metrics.trade_count} "
|
|
|
f"Return={result.metrics.total_return_pct or 0:.2f}% "
|
|
|
f"WinRate={result.metrics.win_rate or 0:.1%}"
|
|
|
)
|
|
|
|
|
|
# Aggregate summary
|
|
|
total_trades = sum(f["trade_count"] or 0 for f in fold_results)
|
|
|
returns = [f["total_return_pct"] for f in fold_results if f["total_return_pct"] is not None]
|
|
|
win_rates = [f["win_rate"] for f in fold_results if f["win_rate"] is not None]
|
|
|
|
|
|
print(f"\n--- Walk-Forward Summary ({len(fold_results)} folds) ---")
|
|
|
print(f"Total trades: {total_trades}")
|
|
|
if returns:
|
|
|
import statistics
|
|
|
print(f"Mean return: {statistics.mean(returns):.2f}%")
|
|
|
if len(returns) > 1:
|
|
|
print(f"StdDev return: {statistics.stdev(returns):.2f}%")
|
|
|
if win_rates:
|
|
|
import statistics
|
|
|
print(f"Mean win rate: {statistics.mean(win_rates):.1%}")
|
|
|
|
|
|
return fold_results
|
|
|
|
|
|
|
|
|
def main() -> None:
|
|
|
parser = argparse.ArgumentParser(description="ACE-F Backtester")
|
|
|
parser.add_argument("--manifest", required=True, help="Path to experiment manifest JSON")
|
|
|
parser.add_argument("--snapshot-id", help="Override dataset_snapshot_id")
|
|
|
parser.add_argument("--snapshot-dir", help="Override snapshot root directory (default: data/parquet/)")
|
|
|
parser.add_argument("--split", default="train", help="Split name (train/valid/test)")
|
|
|
parser.add_argument("--output-root", default="./runs", help="Output root directory")
|
|
|
parser.add_argument("--initial-equity", type=float, default=100_000.0)
|
|
|
parser.add_argument("--config-root", default=".", help="Root dir for resolving config paths")
|
|
|
parser.add_argument(
|
|
|
"--walk-forward",
|
|
|
action="store_true",
|
|
|
help="Run walk-forward cross-validation instead of single backtest",
|
|
|
)
|
|
|
parser.add_argument("--wf-train-days", type=int, default=252, help="Walk-forward train window (trading days)")
|
|
|
parser.add_argument("--wf-test-days", type=int, default=63, help="Walk-forward test window (trading days)")
|
|
|
parser.add_argument("--mode", choices=["research", "live"], default=None,
|
|
|
help="Backtest mode: research (kill switch resets) or live (permanent)")
|
|
|
args = parser.parse_args()
|
|
|
|
|
|
manifest = load_manifest(args.manifest)
|
|
|
config = resolve_config(manifest, config_root=args.config_root, snapshot_id_override=args.snapshot_id)
|
|
|
|
|
|
if args.mode:
|
|
|
config.risk.backtest_mode = args.mode
|
|
|
|
|
|
if args.walk_forward:
|
|
|
fold_results = run_walk_forward(
|
|
|
manifest=manifest,
|
|
|
config=config,
|
|
|
snapshot_dir_override=args.snapshot_dir,
|
|
|
initial_equity=args.initial_equity,
|
|
|
output_root=args.output_root,
|
|
|
train_days=args.wf_train_days,
|
|
|
test_days=args.wf_test_days,
|
|
|
)
|
|
|
# Write summary JSON
|
|
|
summary_path = Path(args.output_root) / "walk_forward_summary.json"
|
|
|
summary_path.parent.mkdir(parents=True, exist_ok=True)
|
|
|
summary_path.write_text(json.dumps(fold_results, indent=2))
|
|
|
print(f"\nSummary written to: {summary_path}")
|
|
|
else:
|
|
|
store = _build_store(manifest, config, args.split, snapshot_dir_override=args.snapshot_dir)
|
|
|
runner = BacktestRunner(
|
|
|
manifest=manifest,
|
|
|
config=config,
|
|
|
store=store,
|
|
|
initial_equity=args.initial_equity,
|
|
|
split_name=args.split,
|
|
|
)
|
|
|
result = runner.run(output_root=args.output_root)
|
|
|
print(f"Run complete: {result.run_id}")
|
|
|
print(f"Trades: {result.metrics.trade_count}")
|
|
|
if result.metrics.total_return_pct is not None:
|
|
|
print(f"Total return: {result.metrics.total_return_pct:.2f}%")
|
|
|
|
|
|
# Print SQS score
|
|
|
from libs.backtest.tracker import compute_sqs
|
|
|
sqs_score, sqs_breakdown = compute_sqs(result.metrics)
|
|
|
print(f"SQS: {sqs_score} ({', '.join(f'{k}={v}' for k, v in sqs_breakdown.items())})")
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
main()
|