Refactor paper backtest to use BacktestRunner — eliminates engine divergence
Major refactor: `fithia2 paper backtest` now uses the exact same BacktestRunner + SnapshotStore pipeline as `apps/backtester/run.py`. Before: PaperTradingEngine + EventDetector + MockBroker - Different scoring (compute_entry_score vs config scoring_model) - Different data source (DB + Oracle vs Parquet snapshot) - Different feature computation (real-time vs pipeline) → Config gate changes didn't take effect in paper backtest After: BacktestRunner + SnapshotStore (Parquet) - Identical scoring, engine matching, position sizing - Same Parquet data as research backtester - Config changes work identically in both systems Trade output format preserved for reporter.py compatibility. PaperTradingEngine still used for live Alpaca trading (unchanged). Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>main
I Luk Kim
5 months ago
parent
0797535047
commit
d7ecaf97dc
@ -0,0 +1,236 @@
|
||||
"""Paper-trading backtest simulator.
|
||||
|
||||
Refactored to use the SAME BacktestRunner + SnapshotStore as
|
||||
apps/backtester/run.py. This guarantees identical scoring, engine
|
||||
matching, and position sizing between `fithia2 paper backtest` and
|
||||
the research backtester.
|
||||
|
||||
Previous implementation used PaperTradingEngine + EventDetector +
|
||||
MockBroker which had different scoring functions, feature computation,
|
||||
and data sources — causing divergent results.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
import math
|
||||
import statistics
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from libs.common.logging import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
def run_backtest_session_sync(
|
||||
session_name: str,
|
||||
config_path: str,
|
||||
initial_equity: float,
|
||||
start_date: dt.date,
|
||||
end_date: dt.date,
|
||||
) -> dict[str, Any]:
|
||||
"""Run a single strategy using BacktestRunner (same as research backtester).
|
||||
|
||||
Uses the existing Parquet snapshot + BacktestRunner pipeline so results
|
||||
match `python -m apps.backtester.run --manifest <config>` exactly.
|
||||
"""
|
||||
from apps.backtester.run import (
|
||||
BacktestRunner,
|
||||
_build_store,
|
||||
_build_merged_snapshot_store,
|
||||
load_manifest,
|
||||
resolve_config,
|
||||
)
|
||||
|
||||
manifest = load_manifest(config_path)
|
||||
config = resolve_config(manifest)
|
||||
|
||||
# Use merged store (train+valid+test) to cover the full date range
|
||||
store = _build_merged_snapshot_store(manifest, config, snapshot_dir_override=None)
|
||||
|
||||
# Slice to requested date range
|
||||
store = store.slice_by_date_range(start_date, end_date)
|
||||
|
||||
runner = BacktestRunner(
|
||||
manifest=manifest,
|
||||
config=config,
|
||||
store=store,
|
||||
initial_equity=initial_equity,
|
||||
split_name="paper_backtest",
|
||||
)
|
||||
|
||||
# Run with temporary output directory
|
||||
tmp_dir = tempfile.mkdtemp(prefix="paper_bt_")
|
||||
try:
|
||||
result = runner.run(output_root=tmp_dir)
|
||||
except Exception:
|
||||
result = runner.run(output_root=None)
|
||||
|
||||
# Convert to paper backtest format
|
||||
return _convert_from_runner(session_name, config_path, initial_equity, result, tmp_dir)
|
||||
|
||||
|
||||
def _convert_from_runner(
|
||||
session_name: str,
|
||||
config_path: str,
|
||||
initial_equity: float,
|
||||
result: Any,
|
||||
tmp_dir: str,
|
||||
) -> dict[str, Any]:
|
||||
"""Convert BacktestRunner result to paper backtest output format."""
|
||||
equity_curve: list[dict] = []
|
||||
trades: list[dict] = []
|
||||
|
||||
# Load from Parquet artifacts
|
||||
try:
|
||||
import pyarrow.parquet as pq
|
||||
import glob
|
||||
import os
|
||||
|
||||
run_dirs = sorted(glob.glob(os.path.join(tmp_dir, "bt_*")))
|
||||
if run_dirs:
|
||||
run_dir = Path(run_dirs[0])
|
||||
|
||||
# Equity curve
|
||||
eq_path = run_dir / "artifacts" / "daily_equity_curve.parquet"
|
||||
if eq_path.exists():
|
||||
eq_df = pq.read_table(str(eq_path)).to_pandas()
|
||||
for _, row in eq_df.iterrows():
|
||||
d = row.get("date")
|
||||
if isinstance(d, str):
|
||||
d = dt.date.fromisoformat(d[:10])
|
||||
equity_curve.append({
|
||||
"date": d,
|
||||
"equity": float(row.get("equity", initial_equity)),
|
||||
})
|
||||
|
||||
# Trade blotter
|
||||
bl_path = run_dir / "artifacts" / "trade_blotter.parquet"
|
||||
if bl_path.exists():
|
||||
bl_df = pq.read_table(str(bl_path)).to_pandas()
|
||||
for _, row in bl_df.iterrows():
|
||||
entry_px = row.get("entry_price")
|
||||
exit_px = row.get("exit_price")
|
||||
shares = int(row.get("shares", 0))
|
||||
pnl_pct = float(row.get("pnl_pct", 0.0))
|
||||
pnl_dollar = pnl_pct * float(entry_px or 0) * shares if entry_px else 0.0
|
||||
|
||||
trades.append({
|
||||
"symbol": str(row.get("symbol", "")),
|
||||
"entry_date": str(row.get("entry_date", "-")),
|
||||
"exit_date": str(row.get("exit_date", "-")),
|
||||
"entry_price": float(entry_px) if entry_px is not None else None,
|
||||
"exit_price": float(exit_px) if exit_px is not None else None,
|
||||
"shares": shares,
|
||||
"pnl": pnl_dollar,
|
||||
"reason": str(row.get("exit_reason", "-")),
|
||||
"event_type": str(row.get("event_type", "-")),
|
||||
"score": float(row.get("score", 0.0)),
|
||||
"engine_id": str(row.get("engine_id", "")),
|
||||
})
|
||||
except Exception as exc:
|
||||
logger.warning("backtest_sim_artifact_load_failed", error=str(exc))
|
||||
|
||||
# Compute summary stats
|
||||
final_equity = equity_curve[-1]["equity"] if equity_curve else initial_equity
|
||||
total_return_pct = (final_equity - initial_equity) / initial_equity * 100
|
||||
|
||||
pnls = [t["pnl"] for t in trades]
|
||||
wins = [p for p in pnls if p > 0]
|
||||
win_rate = len(wins) / len(pnls) * 100 if pnls else 0.0
|
||||
|
||||
equities = [r["equity"] for r in equity_curve]
|
||||
daily_returns = [
|
||||
(equities[i] - equities[i - 1]) / equities[i - 1]
|
||||
for i in range(1, len(equities))
|
||||
if equities[i - 1] > 0
|
||||
]
|
||||
if len(daily_returns) >= 2:
|
||||
mean_r = statistics.mean(daily_returns)
|
||||
std_r = statistics.stdev(daily_returns)
|
||||
sharpe = (mean_r / std_r) * math.sqrt(252) if std_r > 0 else 0.0
|
||||
else:
|
||||
sharpe = 0.0
|
||||
|
||||
peak = initial_equity
|
||||
max_dd_pct = 0.0
|
||||
for eq in equities:
|
||||
if eq > peak:
|
||||
peak = eq
|
||||
dd = (peak - eq) / peak * 100 if peak > 0 else 0.0
|
||||
if dd > max_dd_pct:
|
||||
max_dd_pct = dd
|
||||
|
||||
return {
|
||||
"session_name": session_name,
|
||||
"config_path": config_path,
|
||||
"initial_equity": initial_equity,
|
||||
"equity_curve": equity_curve,
|
||||
"trades": trades,
|
||||
"all_entries": [],
|
||||
"all_exits": [],
|
||||
"summary": {
|
||||
"return_pct": total_return_pct,
|
||||
"final_equity": final_equity,
|
||||
"max_dd_pct": max_dd_pct,
|
||||
"trade_count": len(trades),
|
||||
"win_rate": win_rate,
|
||||
"sharpe": sharpe,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
async def run_backtest(
|
||||
configs: list[str],
|
||||
capital: float,
|
||||
start_date: dt.date,
|
||||
end_date: dt.date,
|
||||
db_dsn: str,
|
||||
oracle_url: str,
|
||||
console=None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Run multiple strategies sequentially using BacktestRunner."""
|
||||
from libs.common.time_utils import is_trading_day
|
||||
from libs.common.logging import configure_logging
|
||||
|
||||
all_days = [
|
||||
start_date + dt.timedelta(days=i)
|
||||
for i in range((end_date - start_date).days + 1)
|
||||
]
|
||||
trading_days = [d for d in all_days if is_trading_day(d)]
|
||||
|
||||
if not trading_days:
|
||||
raise ValueError(f"No trading days found between {start_date} and {end_date}")
|
||||
|
||||
if console:
|
||||
console.print(f"[bold]Trading days:[/] {trading_days[0]} → {trading_days[-1]} ({len(trading_days)} days)")
|
||||
console.print("[bold]Engine:[/] BacktestRunner (identical to research backtester)")
|
||||
|
||||
configure_logging("WARNING")
|
||||
|
||||
results = []
|
||||
for config_path in configs:
|
||||
session_name = Path(config_path).stem
|
||||
if console:
|
||||
console.print(f"\n[bold cyan]Running:[/] {session_name}")
|
||||
|
||||
result = run_backtest_session_sync(
|
||||
session_name=session_name,
|
||||
config_path=config_path,
|
||||
initial_equity=capital,
|
||||
start_date=start_date,
|
||||
end_date=end_date,
|
||||
)
|
||||
results.append(result)
|
||||
|
||||
if console and result["summary"]["trade_count"] > 0:
|
||||
s = result["summary"]
|
||||
console.print(
|
||||
f" Trades: {s['trade_count']}, "
|
||||
f"Return: {s['return_pct']:+.2f}%, "
|
||||
f"MaxDD: {s['max_dd_pct']:.2f}%, "
|
||||
f"WR: {s['win_rate']:.0f}%"
|
||||
)
|
||||
|
||||
return results
|
||||
Loading…
Reference in New Issue