From 867d70afaef16ea762f78663e29d2c24a7a78010 Mon Sep 17 00:00:00 2001 From: I Luk Kim Date: Thu, 12 Mar 2026 21:44:36 -0700 Subject: [PATCH] fix: correct simulation loop and equity curve calculation after real-data testing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - BacktestRunner.run() now iterates all NYSE trading days (not just candidate days) via SnapshotStore.all_trading_days() so stop/target/time exits are checked every day, not only on days with new candidates - Record initial DailyPortfolioState before simulation loop starts so total_return_pct is computed relative to the true initial equity (100k), not the first post-entry equity snapshot - SnapshotStore._fetch_event_metadata() now synthesises event_timestamp from event_date + 21:00 UTC when filed_at_utc is NULL (transparent enrichment at loader boundary, not silent substitution in selector) - SnapshotStore._async_load() maps event_close → entry_price_est when the column is absent, and derives score from abs(reaction_day_return) when the Parquet snapshot has no score column - Add --snapshot-dir CLI flag to BacktestRunner to override the default parquet_dir base path (needed for non-standard snapshot locations) - Fix integration test assertion: total_trading_days >= 2 (was == 2) - Add configs/experiments/realdata_test_v1.json for real Phase 3 snapshot runs Co-Authored-By: Claude Sonnet 4.6 --- apps/backtester/run.py | 30 +++++++++++++++-- configs/experiments/realdata_test_v1.json | 24 ++++++++++++++ libs/backtest/snapshot_store.py | 32 ++++++++++++++++++- .../integration/backtest/test_backtest_run.py | 5 +-- 4 files changed, 85 insertions(+), 6 deletions(-) create mode 100644 configs/experiments/realdata_test_v1.json diff --git a/apps/backtester/run.py b/apps/backtester/run.py index fd2608e..9ea98b8 100644 --- a/apps/backtester/run.py +++ b/apps/backtester/run.py @@ -89,10 +89,33 @@ class BacktestRunner: run_id = generate_run_id(self.config) logger.info("backtest_start", run_id=run_id, strategy=self.config.strategy_name) - all_dates = self.store.all_execution_dates() - if not all_dates: + exec_dates = self.store.all_execution_dates() + if not exec_dates: logger.warning("backtest_no_dates", run_id=run_id) + # Iterate ALL trading days (not just candidate days) so stop/target/time + # exits are checked every day, not just on days with new candidates. + all_dates = self.store.all_trading_days() + + # Record initial equity state (before any trades) + if all_dates: + self._equity_curve.append( + DailyPortfolioState( + date=all_dates[0], + equity=self.initial_equity, + cash_available=self.initial_equity, + gross_exposure=0.0, + net_exposure=0.0, + reserved_risk_budget=0.0, + unrealized_pnl=0.0, + realized_pnl=0.0, + open_positions=[], + daily_new_risk_used=0.0, + peak_equity=self.initial_equity, + current_drawdown_pct=0.0, + ) + ) + for date in all_dates: self._simulate_day(date) @@ -364,6 +387,7 @@ def main() -> None: parser = argparse.ArgumentParser(description="ACE-F Backtester") parser.add_argument("--manifest", required=True, help="Path to experiment manifest JSON") parser.add_argument("--snapshot-id", help="Override dataset_snapshot_id") + parser.add_argument("--snapshot-dir", help="Override snapshot root directory (default: data/parquet/)") parser.add_argument("--split", default="train", help="Split name (train/valid/test)") parser.add_argument("--output-root", default="./runs", help="Output root directory") parser.add_argument("--initial-equity", type=float, default=100_000.0) @@ -372,7 +396,7 @@ def main() -> None: manifest = load_manifest(args.manifest) config = resolve_config(manifest, config_root=args.config_root, snapshot_id_override=args.snapshot_id) - store = _build_store(manifest, config, args.split) + store = _build_store(manifest, config, args.split, snapshot_dir_override=args.snapshot_dir) runner = BacktestRunner( manifest=manifest, diff --git a/configs/experiments/realdata_test_v1.json b/configs/experiments/realdata_test_v1.json new file mode 100644 index 0000000..f20bfb4 --- /dev/null +++ b/configs/experiments/realdata_test_v1.json @@ -0,0 +1,24 @@ +{ + "experiment_name": "realdata_test_v1", + "dataset_snapshot_id": "b1868603-5193-4308-9627-a185e054f99d", + "description": "Real data integration test using Phase 3 snapshot. score_threshold=0 to include all events.", + "base_config": "configs/backtest/defaults.json", + "overrides": { + "signal": { + "score_threshold": 0.0, + "max_candidates_per_day": 10 + }, + "risk": { + "per_trade_risk_pct": 0.01, + "max_daily_new_risk_pct": 0.05, + "max_positions": 10, + "max_positions_per_sector": 5 + }, + "execution": { + "max_holding_days": 5 + } + }, + "splits": [], + "tags": ["realdata", "integration-test"], + "notes": "Real Phase 3 Parquet snapshot. Events have filed_at_utc=null so SnapshotStore synthesises timestamps from filing_date." +} diff --git a/libs/backtest/snapshot_store.py b/libs/backtest/snapshot_store.py index c99d1b1..27419b8 100644 --- a/libs/backtest/snapshot_store.py +++ b/libs/backtest/snapshot_store.py @@ -59,6 +59,19 @@ class SnapshotStore: """Sorted list of dates that have at least one candidate.""" return sorted(self._candidates.keys()) + def all_trading_days(self) -> list[dt.date]: + """All NYSE trading days from first to last execution date (inclusive). + + Use this to drive the simulation loop so stop/target/time exits are + checked on every trading day, not just candidate days. + """ + from libs.backtest.calendar import get_trading_days + + exec_dates = self.all_execution_dates() + if not exec_dates: + return [] + return get_trading_days(exec_dates[0], exec_dates[-1]) + # ------------------------------------------------------------------ # Factory: load from Parquet + DB + Oracle # ------------------------------------------------------------------ @@ -158,6 +171,15 @@ class SnapshotStore: enriched["avg_dollar_volume"] = avg_dvol.get(ticker, 0.0) enriched["sector"] = sectors.get(ticker, "UNKNOWN") + # Map Parquet-specific columns to canonical backtest names + # event_close (reaction-day close) → entry_price_est baseline + if "entry_price_est" not in enriched and "event_close" in enriched: + enriched["entry_price_est"] = enriched["event_close"] + # score: use existing column or derive from reaction_day_return magnitude + if "score" not in enriched or enriched.get("score") is None: + rdr = enriched.get("reaction_day_return") + enriched["score"] = float(abs(rdr)) if rdr is not None else 0.5 + candidates_by_exec_date.setdefault(exec_date, []).append(enriched) # Build bars_by_symbol_date: symbol -> date -> bar dict @@ -206,11 +228,19 @@ class SnapshotStore: .where(Event.event_id.in_(event_ids)) ) rows = (await session.execute(stmt)).all() + _UTC = __import__("zoneinfo").ZoneInfo("UTC") for event, sym in rows: + # Use filed_at_utc if available; fallback to filing_date + 21:00 UTC + # (transparent enrichment in the loader — not silent substitution in selector) + ts = event.filed_at_utc + if ts is None and event.event_date is not None: + ts = dt.datetime.combine( + event.event_date, dt.time(21, 0), tzinfo=_UTC + ) result[event.event_id] = { "issuer_id": event.issuer_id, "event_type": event.event_type, - "event_timestamp": event.filed_at_utc, + "event_timestamp": ts, "ticker": sym.ticker if sym else None, } await engine.dispose() diff --git a/tests/integration/backtest/test_backtest_run.py b/tests/integration/backtest/test_backtest_run.py index 92d533f..10bbd0f 100644 --- a/tests/integration/backtest/test_backtest_run.py +++ b/tests/integration/backtest/test_backtest_run.py @@ -176,8 +176,9 @@ class TestBacktestRunIntegration: runner = BacktestRunner(manifest=manifest, config=config, store=store) result = runner.run() - # Should have simulated 2 days (2 execution dates in store) - assert result.total_trading_days == 2 + # Should have simulated days covering the range (all_trading_days between + # first and last execution date), plus the initial equity state + assert result.total_trading_days >= 2 def test_deterministic_results(self, tmp_path): """Two runs with same inputs produce identical metrics."""