"""Point-in-time snapshot data loader for the backtester.

The public interface is:
  store = SnapshotStore.load(snapshot_dir, split_name, oracle_url, db_dsn)
  rows = store.get_candidates_for_date(date)
  bar  = store.get_bar(symbol, date)
  macro = store.get_macro_for_date(date)

For tests, inject data directly:
  store = SnapshotStore(candidates_by_exec_date=..., bars_by_symbol_date=..., macro_by_date=...)
"""
from __future__ import annotations

import asyncio
import datetime as dt
import hashlib
import json
import math
import os
from functools import partial
import pickle
from pathlib import Path
import time
from typing import Any
from zoneinfo import ZoneInfo

import pyarrow as pa
import pyarrow.parquet as pq

from libs.backtest.proxies import peer_candidates_for_symbol, sector_etf_for_sector
from libs.features.market_features import compute_market_features
from libs.common.config import get_settings
from libs.common.logging import get_logger

logger = get_logger(__name__)
_UTC = ZoneInfo("UTC")
_PRICE_DERIVED_FEATURE_COLUMNS = (
    "pre_event_volatility_20d",
    "pre_event_rsi_14",
    "pre_event_bb_position",
    "pre_event_obv_slope_20d",
    "pre_event_hurst_60d",
    "pre_event_entropy_60d",
    "pre_event_ou_theta_60d",
    "pre_event_gravitational_pull",
    "pre_event_market_temperature",
)
_MACRO_BACKFILL_FEATURE_COLUMNS = (
    "macro_vix",
    "macro_hy_spread",
    "macro_t10y2y",
)
_MACRO_TRADE_BAR_PREFIXES: tuple[tuple[str, str], ...] = (
    ("SPY", "spy"),
    ("QQQ", "qqq"),
    ("SH", "sh"),
    ("PSQ", "psq"),
    ("GLD", "gld"),
    ("SPYM", "spym"),
    ("QUAL", "qual"),
    ("QQQM", "qqqm"),
    ("TQQQ", "tqqq"),
    ("SHY", "shy"),
    ("USFR", "usfr"),
    ("BIL", "bil"),
    ("VGSH", "vgsh"),
    ("IEI", "iei"),
    ("IEF", "ief"),
    ("TIP", "tip"),
    ("XLK", "xlk"),
    ("SMH", "smh"),
    ("IWM", "iwm"),
    ("XLI", "xli"),
    ("XLE", "xle"),
    ("XLF", "xlf"),
    ("DBC", "dbc"),
    ("SGOV", "sgov"),
    ("JEPQ", "jepq"),
    ("BUFB", "bufb"),
    ("MERIX", "merix"),
)
_PRICE_FEATURE_WARMUP_DAYS = 120
_RUNTIME_CACHE_VERSION = 12
_RUNTIME_CACHE_WAIT_TIMEOUT_SECONDS = 1800.0
_RUNTIME_CACHE_WAIT_LOG_INTERVAL_SECONDS = 5.0


class SnapshotStore:
    """In-memory cache of point-in-time backtest data."""

    def __init__(
        self,
        candidates_by_exec_date: dict[dt.date, list[dict[str, Any]]],
        bars_by_symbol_date: dict[str, dict[dt.date, dict[str, Any]]],
        macro_by_date: dict[dt.date, dict[str, Any]] | None = None,
    ) -> None:
        self._candidates = {
            date: list(rows)
            for date, rows in candidates_by_exec_date.items()
        }
        self._candidates_by_reaction_date = self._build_reaction_index(self._candidates)
        self._bars = bars_by_symbol_date
        self._macro = macro_by_date or {}
        self._price_bar_cache: dict[str, list[Any]] = {}
        self._market_feature_cache: dict[tuple[str, dt.date], dict[str, Any]] = {}

    # ------------------------------------------------------------------
    # Public query interface
    # ------------------------------------------------------------------

    def get_candidates_for_date(self, date: dt.date) -> list[dict[str, Any]]:
        """Return candidates where execution_date == date. No look-ahead."""
        return list(self._candidates.get(date, []))

    def get_candidates_for_reaction_date(self, date: dt.date) -> list[dict[str, Any]]:
        """Return candidates where reaction_date == date. No look-ahead."""
        return list(self._candidates_by_reaction_date.get(date, []))

    def get_bar(self, symbol: str, date: dt.date) -> dict[str, Any] | None:
        """Return OHLCV bar for symbol on date, or None if missing."""
        sym_bars = self._bars.get(symbol)
        if sym_bars is None:
            return None
        return sym_bars.get(date)

    def get_latest_bar_on_or_before(
        self,
        symbol: str,
        date: dt.date,
    ) -> tuple[dt.date, dict[str, Any]] | None:
        """Return the latest available bar for ``symbol`` on or before ``date``."""
        sym_bars = self._bars.get(symbol)
        if not sym_bars:
            return None
        eligible_dates = [bar_date for bar_date in sym_bars if bar_date <= date]
        if not eligible_dates:
            return None
        latest_date = max(eligible_dates)
        return latest_date, sym_bars[latest_date]

    def get_macro_for_date(self, date: dt.date) -> dict[str, Any]:
        """Return macro observations for date (empty dict if none)."""
        return dict(self._macro.get(date, {}))

    def get_market_features(self, symbol: str, event_date: dt.date) -> dict[str, Any]:
        """Return market-side features for a stored symbol as of event_date."""
        cache_key = (str(symbol).upper(), event_date)
        if cache_key in self._market_feature_cache:
            return dict(self._market_feature_cache[cache_key])

        price_bars = self._get_price_bars_for_symbol(
            str(symbol).upper(),
            bars_by_symbol=self._bars,
            price_bar_cache=self._price_bar_cache,
        )
        if not price_bars:
            self._market_feature_cache[cache_key] = {}
            return {}

        features = compute_market_features(price_bars, event_date.isoformat())
        self._market_feature_cache[cache_key] = dict(features)
        return dict(features)

    def all_execution_dates(self) -> list[dt.date]:
        """Sorted list of dates that have at least one candidate."""
        return sorted(self._candidates.keys())

    def all_reaction_dates(self) -> list[dt.date]:
        """Sorted list of dates that have at least one reaction-date candidate."""
        return sorted(self._candidates_by_reaction_date.keys())

    def all_trading_days(self, include_reaction_dates: bool = False) -> list[dt.date]:
        """All NYSE trading days from first execution date to last available date.

        Extends beyond the last execution date using macro data (SPY/QQQ prices)
        so parking can run on days with no events. Warm-up dates before the first
        execution date are excluded.
        """
        from libs.backtest.calendar import get_trading_days

        dates = set(self.all_execution_dates())
        if include_reaction_dates:
            dates.update(self.all_reaction_dates())
        # Extend END of range with macro dates (for parking after last event)
        if self._macro and dates:
            min_exec = min(dates)
            macro_after = {d for d in self._macro if d >= min_exec}
            dates.update(macro_after)
        elif self._macro and not dates:
            # No events at all (parking_only) — use full macro range
            dates.update(self._macro.keys())
        if not dates:
            return []
        ordered = sorted(dates)
        return get_trading_days(ordered[0], ordered[-1])

    def slice_by_date_range(
        self,
        start_date: dt.date,
        end_date: dt.date,
        *,
        require_reaction_date_in_range: bool = False,
        clamp_reaction_index_to_window: bool = True,
    ) -> "SnapshotStore":
        """Return a copy restricted to candidates within a date window.

        Candidates are always filtered by execution date. When
        ``require_reaction_date_in_range`` is True, rows with a reaction date
        outside the requested window are excluded as well. By default rows are
        kept if they execute in-range, but the reaction-date index is clamped
        to the requested window so reaction-close simulations do not expand the
        fold backwards. Bars are kept intact, while macro observations are
        trimmed to the same date range.
        """
        filtered_candidates: dict[dt.date, list[dict[str, Any]]] = {}
        for exec_date, rows in self._candidates.items():
            if exec_date < start_date or exec_date > end_date:
                continue
            kept_rows: list[dict[str, Any]] = []
            for row in rows:
                if require_reaction_date_in_range:
                    reaction_date = self._normalize_date(row.get("reaction_date"))
                    if reaction_date is not None and (
                        reaction_date < start_date or reaction_date > end_date
                    ):
                        continue
                kept_rows.append(dict(row))
            if kept_rows:
                filtered_candidates[exec_date] = kept_rows

        # Keep macro data beyond end_date so parking can run after last event.
        # Only trim dates before start_date (warm-up excluded).
        filtered_macro = {
            date: dict(values)
            for date, values in self._macro.items()
            if date >= start_date
        }
        sliced = SnapshotStore(
            candidates_by_exec_date=filtered_candidates,
            bars_by_symbol_date=self._bars,
            macro_by_date=filtered_macro,
        )
        if clamp_reaction_index_to_window:
            sliced._candidates_by_reaction_date = {
                date: list(rows)
                for date, rows in sliced._candidates_by_reaction_date.items()
                if start_date <= date <= end_date
            }
        return sliced

    # ------------------------------------------------------------------
    # Factory: load from Parquet + DB + Oracle
    # ------------------------------------------------------------------

    @classmethod
    def load(
        cls,
        snapshot_dir: str | Path,
        split_name: str,
        oracle_url: str,
        db_dsn: str,
        scoring_fn: Any | None = None,
    ) -> "SnapshotStore":
        """Synchronous factory. Internally uses asyncio.run() to prefetch data.

        scoring_fn: optional callable(row_dict) -> float to override default scoring.
        Raises RuntimeError if called from within a running event loop.
        """
        try:
            loop = asyncio.get_running_loop()
        except RuntimeError:
            loop = None

        if loop is not None and loop.is_running():
            raise RuntimeError(
                "SnapshotStore.load() cannot be called from a running event loop. "
                "Use SnapshotStore._async_load() directly in async contexts."
            )

        snapshot_path = Path(snapshot_dir)
        return cls._load_with_runtime_cache(
            snapshot_path=snapshot_path,
            split_names=[split_name],
            scoring_fn=scoring_fn,
            builder=lambda: asyncio.run(
                cls._async_load(snapshot_path, split_name, oracle_url, db_dsn, scoring_fn)
            ),
        )

    @classmethod
    def load_merged(
        cls,
        snapshot_dir: str | Path,
        split_names: list[str],
        oracle_url: str,
        db_dsn: str,
        scoring_fn: Any | None = None,
    ) -> "SnapshotStore":
        """Load multiple splits in one pass to avoid duplicate DB/Oracle fetches."""
        try:
            loop = asyncio.get_running_loop()
        except RuntimeError:
            loop = None

        if loop is not None and loop.is_running():
            raise RuntimeError(
                "SnapshotStore.load_merged() cannot be called from a running event loop. "
                "Use SnapshotStore._async_load_merged() directly in async contexts."
            )

        snapshot_path = Path(snapshot_dir)
        return cls._load_with_runtime_cache(
            snapshot_path=snapshot_path,
            split_names=split_names,
            scoring_fn=scoring_fn,
            builder=lambda: asyncio.run(
                cls._async_load_merged(snapshot_path, split_names, oracle_url, db_dsn, scoring_fn)
            ),
        )

    @classmethod
    def materialize_snapshot_dir(
        cls,
        snapshot_dir: str | Path,
        oracle_url: str,
        db_dsn: str,
        *,
        split_names: list[str] | None = None,
        output_dir: str | Path | None = None,
    ) -> list[str]:
        """Persist runtime-backfilled feature columns into snapshot parquet files."""
        try:
            loop = asyncio.get_running_loop()
        except RuntimeError:
            loop = None

        if loop is not None and loop.is_running():
            raise RuntimeError(
                "SnapshotStore.materialize_snapshot_dir() cannot be called from a running event loop. "
                "Use SnapshotStore._async_materialize_snapshot_dir() directly in async contexts."
            )

        return asyncio.run(
            cls._async_materialize_snapshot_dir(
                Path(snapshot_dir),
                oracle_url=oracle_url,
                db_dsn=db_dsn,
                split_names=split_names,
                output_dir=Path(output_dir) if output_dir is not None else None,
            )
        )

    @classmethod
    async def _async_load(
        cls,
        snapshot_dir: Path,
        split_name: str,
        oracle_url: str,
        db_dsn: str,
        scoring_fn: Any | None = None,
    ) -> dict[str, Any]:
        """Async data loading pipeline. Returns kwargs dict for __init__."""
        parquet_path = snapshot_dir / f"{split_name}.parquet"
        if not parquet_path.exists():
            raise FileNotFoundError(f"Parquet file not found: {parquet_path}")

        row_list = cls._read_parquet_rows(parquet_path)
        logger.info("snapshot_store_rows_loaded", split=split_name, count=len(row_list))

        return await cls._build_init_kwargs_from_rows(
            row_list=row_list,
            oracle_url=oracle_url,
            db_dsn=db_dsn,
            scoring_fn=scoring_fn,
        )

    @classmethod
    async def _async_load_merged(
        cls,
        snapshot_dir: Path,
        split_names: list[str],
        oracle_url: str,
        db_dsn: str,
        scoring_fn: Any | None = None,
    ) -> dict[str, Any]:
        row_list: list[dict[str, Any]] = []
        loaded_splits: list[str] = []
        for split_name in split_names:
            parquet_path = snapshot_dir / f"{split_name}.parquet"
            if not parquet_path.exists():
                continue
            loaded_splits.append(split_name)
            row_list.extend(cls._read_parquet_rows(parquet_path))

        if not loaded_splits:
            raise FileNotFoundError("No snapshot splits found.")

        logger.info(
            "snapshot_store_rows_loaded",
            split="merged",
            loaded_splits=loaded_splits,
            count=len(row_list),
        )

        return await cls._build_init_kwargs_from_rows(
            row_list=row_list,
            oracle_url=oracle_url,
            db_dsn=db_dsn,
            scoring_fn=scoring_fn,
        )

    @classmethod
    async def _async_materialize_snapshot_dir(
        cls,
        snapshot_dir: Path,
        *,
        oracle_url: str,
        db_dsn: str,
        split_names: list[str] | None = None,
        output_dir: Path | None = None,
    ) -> list[str]:
        split_names = list(split_names or ["train", "valid", "test"])
        output_dir = output_dir or snapshot_dir
        output_dir.mkdir(parents=True, exist_ok=True)

        tables_by_split: dict[str, pa.Table] = {}
        rows_by_split: dict[str, list[dict[str, Any]]] = {}
        row_list: list[dict[str, Any]] = []
        needed_columns: set[str] = set()

        for split_name in split_names:
            parquet_path = snapshot_dir / f"{split_name}.parquet"
            if not parquet_path.exists():
                continue
            table = pq.read_table(str(parquet_path))
            rows = table.to_pylist()
            tables_by_split[split_name] = table
            rows_by_split[split_name] = rows
            row_list.extend(rows)
            for column in (*_PRICE_DERIVED_FEATURE_COLUMNS, *_MACRO_BACKFILL_FEATURE_COLUMNS):
                if column not in table.column_names or table.column(column).null_count > 0:
                    needed_columns.add(column)

        if not tables_by_split:
            raise FileNotFoundError(f"No snapshot splits found under {snapshot_dir}")

        if not needed_columns:
            if output_dir != snapshot_dir:
                for split_name in tables_by_split:
                    pq.write_table(tables_by_split[split_name], output_dir / f"{split_name}.parquet")
                manifest_path = snapshot_dir / "manifest.json"
                if manifest_path.exists():
                    (output_dir / "manifest.json").write_text(manifest_path.read_text())
            return []

        date_range = cls._compute_date_range(row_list)
        unique_symbols = sorted({
            str(symbol).upper()
            for symbol in (
                row.get("ticker") or row.get("symbol")
                for row in row_list
            )
            if symbol
        })

        bars_by_symbol: dict[str, dict[dt.date, dict[str, Any]]] = {}
        macro_by_date: dict[dt.date, dict[str, Any]] = {}
        if any(column in needed_columns for column in _PRICE_DERIVED_FEATURE_COLUMNS):
            bars_by_symbol, _ = await cls._fetch_price_data(unique_symbols, date_range, oracle_url)
        if any(column in needed_columns for column in _MACRO_BACKFILL_FEATURE_COLUMNS):
            macro_by_date = await cls._fetch_macro(date_range, db_dsn)

        _vix_pairs = sorted(
            (d, v["VIXCLS"])
            for d, v in macro_by_date.items()
            if "VIXCLS" in v and v["VIXCLS"] is not None
        )
        _hy_series_key = next(
            (k for k in next(iter(macro_by_date.values()), {}) if "BAMLH0" in k),
            None,
        ) if macro_by_date else None
        _hy_pairs = sorted(
            (d, v[_hy_series_key])
            for d, v in macro_by_date.items()
            if _hy_series_key and _hy_series_key in v and v[_hy_series_key] is not None
        ) if _hy_series_key else []
        _t10y2y_pairs = sorted(
            (d, v["T10Y2Y"])
            for d, v in macro_by_date.items()
            if "T10Y2Y" in v and v["T10Y2Y"] is not None
        )

        price_bar_cache: dict[str, list[Any]] = {}
        derived_feature_cache: dict[tuple[str, dt.date], dict[str, Any]] = {}
        materialized_columns: set[str] = set()

        for split_name, table in tables_by_split.items():
            rows = rows_by_split[split_name]
            for row in rows:
                symbol = (row.get("ticker") or row.get("symbol") or "")
                look_date = (
                    cls._normalize_date(row.get("event_date"))
                    or cls._normalize_date(row.get("execution_date"))
                    or cls._normalize_date(row.get("entry_date"))
                )
                if symbol and look_date:
                    if any(column in needed_columns for column in _PRICE_DERIVED_FEATURE_COLUMNS):
                        cls._backfill_price_derived_features(
                            row,
                            symbol=str(symbol).upper(),
                            event_date=look_date,
                            bars_by_symbol=bars_by_symbol,
                            price_bar_cache=price_bar_cache,
                            derived_feature_cache=derived_feature_cache,
                        )
                    if "macro_vix" in needed_columns and row.get("macro_vix") is None:
                        value = cls._lookup_as_of(_vix_pairs, look_date)
                        if value is not None:
                            row["macro_vix"] = value
                    if "macro_hy_spread" in needed_columns and row.get("macro_hy_spread") is None:
                        value = cls._lookup_as_of(_hy_pairs, look_date)
                        if value is not None:
                            row["macro_hy_spread"] = value
                    if "macro_t10y2y" in needed_columns and row.get("macro_t10y2y") is None:
                        value = cls._lookup_as_of(_t10y2y_pairs, look_date)
                        if value is not None:
                            row["macro_t10y2y"] = value

            updated_table = table
            for column in sorted(needed_columns):
                values = [row.get(column) for row in rows]
                if not any(value is not None for value in values):
                    continue
                array = pa.array(values, type=pa.float64())
                if column in updated_table.column_names:
                    idx = updated_table.column_names.index(column)
                    updated_table = updated_table.set_column(idx, column, array)
                else:
                    updated_table = updated_table.append_column(column, array)
                materialized_columns.add(column)

            pq.write_table(updated_table, output_dir / f"{split_name}.parquet")

        manifest_path = snapshot_dir / "manifest.json"
        if manifest_path.exists():
            manifest = json.loads(manifest_path.read_text())
            manifest["output_dir"] = str(output_dir.resolve())
            manifest["materialized_feature_columns"] = sorted(materialized_columns)
            manifest["materialized_feature_last_refresh_utc"] = dt.datetime.now(dt.UTC).isoformat()
            (output_dir / "manifest.json").write_text(json.dumps(manifest, indent=2))

        return sorted(materialized_columns)

    @staticmethod
    def _read_parquet_rows(parquet_path: Path) -> list[dict[str, Any]]:
        logger.info("snapshot_store_reading_parquet", path=str(parquet_path))
        table = pq.read_table(str(parquet_path))
        cols = table.to_pydict()
        num_rows = table.num_rows
        col_names = list(cols.keys())
        return [{col: cols[col][i] for col in col_names} for i in range(num_rows)]

    @classmethod
    async def _build_init_kwargs_from_rows(
        cls,
        *,
        row_list: list[dict[str, Any]],
        oracle_url: str,
        db_dsn: str,
        scoring_fn: Any | None = None,
    ) -> dict[str, Any]:
        raw_row_count = len(row_list)

        # De-duplicate merged-boundary overlaps before any expensive fetches.
        deduped_rows: list[dict[str, Any]] = []
        seen_keys: set[tuple[Any, ...]] = set()
        for row in row_list:
            key = (
                row.get("event_id"),
                row.get("ticker") or row.get("symbol"),
                row.get("entry_date") or row.get("execution_date"),
                row.get("reaction_date"),
            )
            if key in seen_keys:
                continue
            seen_keys.add(key)
            deduped_rows.append(row)
        row_list = deduped_rows
        logger.info(
            "snapshot_store_prepare_rows",
            raw_rows=raw_row_count,
            deduped_rows=len(row_list),
        )

        # Collect event_ids for DB lookup
        event_ids = [str(r.get("event_id", "")) for r in row_list]
        date_range = cls._compute_date_range(row_list)

        # Step 2–6: DB + Oracle enrichment
        logger.info(
            "snapshot_store_event_metadata_fetch_start",
            event_count=len(event_ids),
        )
        event_meta = await cls._fetch_event_metadata(event_ids, db_dsn)
        logger.info(
            "snapshot_store_event_metadata_fetch_done",
            resolved_events=len(event_meta),
        )
        unique_symbols = sorted({
            str(symbol).upper()
            for symbol in (
                [m.get("ticker", "") for m in event_meta.values()]
                + [r.get("ticker", "") for r in row_list]
                + [r.get("symbol", "") for r in row_list]
            )
            if symbol
        })
        logger.info(
            "snapshot_store_sector_fetch_start",
            symbol_count=len(unique_symbols),
        )
        sectors = await cls._fetch_sectors(unique_symbols, oracle_url)
        logger.info(
            "snapshot_store_sector_fetch_done",
            symbol_count=len(sectors),
        )
        event_symbols_by_exec_date: dict[dt.date, set[str]] = {}
        for row in row_list:
            raw_exec = row.get("entry_date") or row.get("execution_date")
            exec_date = cls._normalize_date(raw_exec)
            if exec_date is None:
                continue
            event_id = str(row.get("event_id", ""))
            meta = event_meta.get(event_id, {})
            ticker = (
                meta.get("ticker")
                or row.get("ticker")
                or row.get("symbol")
            )
            if not ticker:
                continue
            event_symbols_by_exec_date.setdefault(exec_date, set()).add(str(ticker).upper())
        proxy_symbols = sorted(
            {
                proxy_symbol
                for proxy_symbol in (
                    sector_etf_for_sector(sectors.get(symbol))
                    for symbol in unique_symbols
                )
                if proxy_symbol
            }
        )
        peer_proxy_symbols = sorted(
            {
                proxy_symbol
                for symbol in unique_symbols
                for proxy_symbol in peer_candidates_for_symbol(symbol, sectors.get(symbol))
                if proxy_symbol
            }
        )
        price_symbols = sorted(set(unique_symbols) | set(proxy_symbols) | set(peer_proxy_symbols))
        logger.info(
            "snapshot_store_enrichment_plan",
            symbol_count=len(unique_symbols),
            trade_symbol_count=len(price_symbols),
            start_date=date_range[0].isoformat() if date_range else None,
            end_date=date_range[1].isoformat() if date_range else None,
        )
        logger.info(
            "snapshot_store_price_fetch_start",
            symbol_count=len(price_symbols),
        )
        bars_by_symbol, avg_dvol = await cls._fetch_price_data(
            price_symbols, date_range, oracle_url
        )
        logger.info(
            "snapshot_store_price_fetch_done",
            symbol_count=len(bars_by_symbol),
            avg_dvol_symbols=sum(1 for value in avg_dvol.values() if value > 0),
        )
        logger.info("snapshot_store_macro_fetch_start")
        macro_by_date = await cls._fetch_macro(date_range, db_dsn)
        logger.info(
            "snapshot_store_macro_fetch_done",
            macro_dates=len(macro_by_date),
        )

        # Fetch SPY bars for macro regime filter (SMA computation)
        logger.info("snapshot_store_spy_macro_fetch_start")
        spy_macro = await cls._fetch_spy_macro(date_range, oracle_url)
        for d, spy_data in spy_macro.items():
            macro_by_date.setdefault(d, {}).update(spy_data)
        logger.info(
            "snapshot_store_spy_macro_fetch_done",
            spy_dates=len(spy_macro),
        )

        # Compute VIX change rates for composite parking gate
        sorted_macro_dates = sorted(macro_by_date.keys())
        for i, d in enumerate(sorted_macro_dates):
            vix = macro_by_date[d].get("VIXCLS")
            if vix is not None:
                # 5-day VIX change
                for lookback in (5, 10):
                    if i >= lookback:
                        prev_d = sorted_macro_dates[i - lookback]
                        prev_vix = macro_by_date.get(prev_d, {}).get("VIXCLS")
                        if prev_vix is not None:
                            macro_by_date[d][f"vix_change_{lookback}d"] = vix - prev_vix

        # Step 7: Build candidates_by_exec_date
        # Pre-build sorted FRED lookup for macro features that may be absent in older snapshots
        _vix_pairs = sorted(
            (d, v["VIXCLS"])
            for d, v in macro_by_date.items()
            if "VIXCLS" in v and v["VIXCLS"] is not None
        )
        _hy_series_key = next(
            (k for k in next(iter(macro_by_date.values()), {}) if "BAMLH0" in k),
            None,
        ) if macro_by_date else None
        _hy_pairs = sorted(
            (d, v[_hy_series_key])
            for d, v in macro_by_date.items()
            if _hy_series_key and _hy_series_key in v and v[_hy_series_key] is not None
        ) if _hy_series_key else []
        price_bar_cache: dict[str, list[Any]] = {}
        derived_feature_cache: dict[tuple[str, dt.date], dict[str, Any]] = {}

        def _lookup_fred_as_of(pairs: list, as_of: dt.date) -> float | None:
            """Return most recent FRED value on or before as_of date."""
            lo, hi = 0, len(pairs) - 1
            result = None
            while lo <= hi:
                mid = (lo + hi) // 2
                if pairs[mid][0] <= as_of:
                    result = pairs[mid][1]
                    lo = mid + 1
                else:
                    hi = mid - 1
            return result

        candidates_by_exec_date: dict[dt.date, list[dict[str, Any]]] = {}
        total_rows = len(row_list)
        progress_step = max(1000, total_rows // 5) if total_rows else 1000
        for idx, row in enumerate(row_list, start=1):
            eid = str(row.get("event_id", ""))
            meta = event_meta.get(eid, {})
            ticker = meta.get("ticker") or row.get("ticker") or row.get("symbol")
            if not ticker:
                logger.debug("snapshot_store_skip_no_ticker", event_id=eid)
                continue

            # Map entry_date → execution_date at this boundary
            raw_exec = row.get("entry_date") or row.get("execution_date")
            if raw_exec is None:
                continue
            if isinstance(raw_exec, str):
                exec_date = dt.date.fromisoformat(raw_exec)
            elif isinstance(raw_exec, dt.date):
                exec_date = raw_exec
            else:
                continue

            enriched = dict(row)
            enriched["execution_date"] = exec_date
            enriched["symbol"] = ticker
            enriched["issuer_id"] = meta.get("issuer_id") or row.get("issuer_id")
            fallback_event_date = cls._normalize_date(row.get("event_date"))
            enriched["event_date"] = meta.get("event_date") or fallback_event_date
            enriched["event_type"] = meta.get("event_type", "") or row.get("event_type", "")
            enriched["event_timestamp"] = (
                meta.get("event_timestamp")
                or cls._normalize_timestamp(row.get("event_timestamp"), fallback_event_date)
            )
            enriched["sector"] = sectors.get(ticker, "UNKNOWN")

            # Backfill macro features from FRED data when absent in Parquet
            # (older or OOT snapshots may lack these columns)
            look_date = enriched["event_date"] or exec_date
            if isinstance(look_date, dt.datetime):
                look_date = look_date.date()
            if look_date is not None:
                if enriched.get("macro_vix") is None and _vix_pairs:
                    enriched["macro_vix"] = _lookup_fred_as_of(_vix_pairs, look_date)
                if enriched.get("macro_hy_spread") is None and _hy_pairs:
                    enriched["macro_hy_spread"] = _lookup_fred_as_of(_hy_pairs, look_date)

            if look_date is not None:
                cls._backfill_price_derived_features(
                    enriched,
                    symbol=ticker,
                    event_date=look_date,
                    bars_by_symbol=bars_by_symbol,
                    price_bar_cache=price_bar_cache,
                    derived_feature_cache=derived_feature_cache,
                )
                cls._backfill_avg_dollar_volume_features(
                    enriched,
                    symbol=ticker,
                    event_date=look_date,
                    bars_by_symbol=bars_by_symbol,
                    price_bar_cache=price_bar_cache,
                    fallback_avg_dvol=avg_dvol.get(ticker, 0.0),
                )
            else:
                cls._backfill_avg_dollar_volume_features(
                    enriched,
                    symbol=ticker,
                    event_date=None,
                    bars_by_symbol=bars_by_symbol,
                    price_bar_cache=price_bar_cache,
                    fallback_avg_dvol=avg_dvol.get(ticker, 0.0),
                )
            reaction_date = cls._normalize_date(enriched.get("reaction_date")) or exec_date
            cls._attach_sector_etf_proxy_features(
                enriched,
                sector=enriched.get("sector"),
                reaction_date=reaction_date,
                execution_date=exec_date,
                bars_by_symbol=bars_by_symbol,
                avg_dvol=avg_dvol,
                price_bar_cache=price_bar_cache,
            )
            cls._attach_peer_proxy_features(
                enriched,
                source_symbol=ticker,
                sector=enriched.get("sector"),
                reaction_date=reaction_date,
                execution_date=exec_date,
                event_symbols_by_exec_date=event_symbols_by_exec_date,
                bars_by_symbol=bars_by_symbol,
                avg_dvol=avg_dvol,
                price_bar_cache=price_bar_cache,
            )

            # Map Parquet-specific columns to canonical backtest names
            # event_close (reaction-day close) → entry_price_est baseline
            if "entry_price_est" not in enriched and "event_close" in enriched:
                enriched["entry_price_est"] = enriched["event_close"]
            # score: use existing column or compute from market features
            if "score" not in enriched or enriched.get("score") is None:
                if scoring_fn is not None:
                    enriched["score"] = scoring_fn(enriched)
                else:
                    from libs.backtest.scoring import compute_entry_score

                    enriched["score"] = compute_entry_score(enriched)

            candidates_by_exec_date.setdefault(exec_date, []).append(enriched)
            if idx % progress_step == 0 or idx == total_rows:
                logger.info(
                    "snapshot_store_candidate_build_progress",
                    processed=idx,
                    total=total_rows,
                )

        cls._attach_recent_sector_cluster_features(candidates_by_exec_date)

        # Build bars_by_symbol_date: symbol -> date -> bar dict
        bars_by_symbol_date: dict[str, dict[dt.date, dict[str, Any]]] = {}
        for sym, date_bars in bars_by_symbol.items():
            bars_by_symbol_date[sym] = date_bars

        cls._inject_macro_trade_bars(bars_by_symbol_date, macro_by_date)

        logger.info(
            "snapshot_store_built",
            exec_dates=len(candidates_by_exec_date),
            symbols=len(bars_by_symbol_date),
            candidates=sum(len(rows) for rows in candidates_by_exec_date.values()),
        )
        return {
            "candidates_by_exec_date": candidates_by_exec_date,
            "bars_by_symbol_date": bars_by_symbol_date,
            "macro_by_date": macro_by_date,
        }

    @classmethod
    def _attach_recent_sector_cluster_features(
        cls,
        candidates_by_exec_date: dict[dt.date, list[dict[str, Any]]],
    ) -> None:
        def _coerce_float(value: Any) -> float | None:
            try:
                out = float(value)
            except (TypeError, ValueError):
                return None
            if math.isnan(out):
                return None
            return out

        ordered_rows: list[dict[str, Any]] = []
        for rows in candidates_by_exec_date.values():
            ordered_rows.extend(rows)

        def _event_anchor_date(row: dict[str, Any]) -> dt.date | None:
            return (
                cls._normalize_date(row.get("event_date"))
                or cls._normalize_date(row.get("reaction_date"))
                or cls._normalize_date(row.get("execution_date"))
            )

        def _event_anchor_timestamp(row: dict[str, Any]) -> dt.datetime:
            anchor_date = _event_anchor_date(row) or dt.date.max
            return (
                cls._normalize_timestamp(row.get("event_timestamp"), anchor_date)
                or dt.datetime.combine(anchor_date, dt.time.max, tzinfo=dt.UTC)
            )

        ordered_rows.sort(
            key=lambda row: (
                _event_anchor_date(row) or dt.date.max,
                _event_anchor_timestamp(row),
                str(row.get("event_id") or ""),
            )
        )

        history_by_sector: dict[str, list[dict[str, float | dt.date | bool]]] = {}
        lookback_days = dt.timedelta(days=3)

        for row in ordered_rows:
            sector = str(row.get("sector") or "UNKNOWN")
            anchor_date = _event_anchor_date(row)
            if sector == "UNKNOWN" or anchor_date is None:
                row["sector_recent_event_count_3d"] = 0.0
                row["sector_recent_leader_count_3d"] = 0.0
                row["sector_recent_leader_reaction_max_3d"] = None
                row["sector_recent_leader_market_cap_max_3d"] = None
                continue

            sector_history = history_by_sector.setdefault(sector, [])
            cutoff = anchor_date - lookback_days
            sector_history[:] = [
                item for item in sector_history if isinstance(item["anchor_date"], dt.date) and item["anchor_date"] >= cutoff
            ]

            leaders = [item for item in sector_history if bool(item.get("is_leader"))]
            row["sector_recent_event_count_3d"] = float(len(sector_history))
            row["sector_recent_leader_count_3d"] = float(len(leaders))
            row["sector_recent_leader_reaction_max_3d"] = (
                max(float(item["reaction_day_return"]) for item in leaders)
                if leaders
                else None
            )
            row["sector_recent_leader_market_cap_max_3d"] = (
                max(float(item["market_cap_proxy"]) for item in leaders if item.get("market_cap_proxy") is not None)
                if leaders and any(item.get("market_cap_proxy") is not None for item in leaders)
                else None
            )

            reaction_day_return = _coerce_float(row.get("reaction_day_return"))
            volume_ratio = _coerce_float(row.get("volume_ratio"))
            close_location = _coerce_float(row.get("close_location"))
            market_cap_proxy = _coerce_float(row.get("market_cap_proxy"))
            is_leader = (
                str(row.get("event_type") or "") == "earnings_release"
                and reaction_day_return is not None
                and reaction_day_return >= 0.08
                and volume_ratio is not None
                and volume_ratio >= 2.0
                and close_location is not None
                and close_location >= 0.6
                and market_cap_proxy is not None
                and market_cap_proxy >= 10_000_000_000.0
            )
            sector_history.append(
                {
                    "anchor_date": anchor_date,
                    "reaction_day_return": reaction_day_return or 0.0,
                    "market_cap_proxy": market_cap_proxy,
                    "is_leader": is_leader,
                }
            )

    # ------------------------------------------------------------------
    # Internal async helpers
    # ------------------------------------------------------------------

    @staticmethod
    def _runtime_cache_dir(snapshot_dir: Path) -> Path:
        return snapshot_dir / ".runtime_cache"

    @staticmethod
    def _scoring_cache_token(scoring_fn: Any | None) -> str:
        if scoring_fn is None:
            return "none"
        if isinstance(scoring_fn, partial):
            func = scoring_fn.func
            return (
                f"partial:{getattr(func, '__module__', '')}.{getattr(func, '__qualname__', getattr(func, '__name__', type(func).__name__))}:"
                f"args={repr(scoring_fn.args)}:keywords={repr(scoring_fn.keywords)}"
            )
        return (
            f"callable:{getattr(scoring_fn, '__module__', '')}."
            f"{getattr(scoring_fn, '__qualname__', getattr(scoring_fn, '__name__', type(scoring_fn).__name__))}"
        )

    @classmethod
    def _runtime_cache_file(
        cls,
        snapshot_dir: Path,
        split_names: list[str],
        scoring_fn: Any | None,
    ) -> Path:
        split_part = "__".join(sorted(split_names))
        score_token = cls._scoring_cache_token(scoring_fn)
        token_hash = hashlib.sha1(score_token.encode("utf-8")).hexdigest()[:16]
        return cls._runtime_cache_dir(snapshot_dir) / f"{split_part}__{token_hash}.pkl"

    @classmethod
    def _runtime_cache_lock_file(
        cls,
        snapshot_dir: Path,
        split_names: list[str],
        scoring_fn: Any | None,
    ) -> Path:
        cache_file = cls._runtime_cache_file(snapshot_dir, split_names, scoring_fn)
        return cache_file.with_suffix(f"{cache_file.suffix}.lock")

    @staticmethod
    def _runtime_cache_fingerprint(snapshot_dir: Path, split_names: list[str]) -> str:
        paths: list[Path] = [snapshot_dir / "manifest.json"]
        paths.extend(snapshot_dir / f"{split_name}.parquet" for split_name in sorted(split_names))
        parts: list[str] = [f"v={_RUNTIME_CACHE_VERSION}"]
        for path in paths:
            if not path.exists():
                parts.append(f"{path.name}:missing")
                continue
            stat = path.stat()
            parts.append(f"{path.name}:{stat.st_size}:{stat.st_mtime_ns}")
        return hashlib.sha1("|".join(parts).encode("utf-8")).hexdigest()

    @classmethod
    def _try_load_runtime_cache(
        cls,
        snapshot_dir: Path,
        split_names: list[str],
        *,
        scoring_fn: Any | None,
    ) -> dict[str, Any] | None:
        cache_file = cls._runtime_cache_file(snapshot_dir, split_names, scoring_fn)
        if not cache_file.exists():
            logger.info(
                "snapshot_store_runtime_cache_miss",
                reason="file_missing",
                cache_file=str(cache_file),
            )
            return None
        try:
            payload = pickle.loads(cache_file.read_bytes())
            expected_fingerprint = cls._runtime_cache_fingerprint(snapshot_dir, split_names)
            if payload.get("fingerprint") != expected_fingerprint:
                logger.info(
                    "snapshot_store_runtime_cache_miss",
                    reason="fingerprint_mismatch",
                    cache_file=str(cache_file),
                )
                return None
            data = payload.get("data")
            if not isinstance(data, dict):
                logger.info(
                    "snapshot_store_runtime_cache_miss",
                    reason="invalid_payload",
                    cache_file=str(cache_file),
                )
                return None
            logger.info(
                "snapshot_store_runtime_cache_hit",
                cache_file=str(cache_file),
                split="merged" if len(split_names) > 1 else split_names[0],
            )
            return data
        except Exception as exc:
            logger.warning(
                "snapshot_store_runtime_cache_read_failed",
                cache_file=str(cache_file),
                error=str(exc),
            )
            return None

    @classmethod
    def _load_with_runtime_cache(
        cls,
        *,
        snapshot_path: Path,
        split_names: list[str],
        scoring_fn: Any | None,
        builder: Any,
    ) -> "SnapshotStore":
        cache_data = cls._try_load_runtime_cache(
            snapshot_path,
            split_names,
            scoring_fn=scoring_fn,
        )
        if cache_data is not None:
            return cls(**cache_data)

        lock_file = cls._runtime_cache_lock_file(snapshot_path, split_names, scoring_fn)
        if cls._acquire_runtime_cache_lock(lock_file):
            try:
                cache_data = cls._try_load_runtime_cache(
                    snapshot_path,
                    split_names,
                    scoring_fn=scoring_fn,
                )
                if cache_data is not None:
                    return cls(**cache_data)
                data = builder()
                cls._write_runtime_cache(
                    snapshot_path,
                    split_names,
                    scoring_fn=scoring_fn,
                    data=data,
                )
                return cls(**data)
            finally:
                cls._release_runtime_cache_lock(lock_file)

        logger.info(
            "snapshot_store_runtime_cache_wait_start",
            cache_file=str(cls._runtime_cache_file(snapshot_path, split_names, scoring_fn)),
            split="merged" if len(split_names) > 1 else split_names[0],
        )
        cache_data = cls._wait_for_runtime_cache(
            snapshot_path,
            split_names,
            scoring_fn=scoring_fn,
            lock_file=lock_file,
        )
        if cache_data is not None:
            return cls(**cache_data)

        if cls._acquire_runtime_cache_lock(lock_file):
            try:
                cache_data = cls._try_load_runtime_cache(
                    snapshot_path,
                    split_names,
                    scoring_fn=scoring_fn,
                )
                if cache_data is not None:
                    return cls(**cache_data)
                data = builder()
                cls._write_runtime_cache(
                    snapshot_path,
                    split_names,
                    scoring_fn=scoring_fn,
                    data=data,
                )
                return cls(**data)
            finally:
                cls._release_runtime_cache_lock(lock_file)

        logger.warning(
            "snapshot_store_runtime_cache_wait_failed",
            cache_file=str(cls._runtime_cache_file(snapshot_path, split_names, scoring_fn)),
        )
        data = builder()
        cls._write_runtime_cache(
            snapshot_path,
            split_names,
            scoring_fn=scoring_fn,
            data=data,
        )
        return cls(**data)

    @staticmethod
    def _runtime_cache_lock_payload() -> dict[str, Any]:
        return {
            "pid": os.getpid(),
            "created_at": time.time(),
        }

    @staticmethod
    def _process_is_running(pid: int | None) -> bool:
        if not isinstance(pid, int) or pid <= 0:
            return False
        try:
            os.kill(pid, 0)
        except ProcessLookupError:
            return False
        except PermissionError:
            return True
        return True

    @classmethod
    def _lock_is_stale(cls, lock_file: Path) -> bool:
        try:
            payload = json.loads(lock_file.read_text())
        except Exception:
            return True
        pid = payload.get("pid")
        created_at = payload.get("created_at")
        if not cls._process_is_running(pid):
            return True
        if not isinstance(created_at, (int, float)):
            return True
        return (time.time() - float(created_at)) > _RUNTIME_CACHE_WAIT_TIMEOUT_SECONDS

    @classmethod
    def _acquire_runtime_cache_lock(cls, lock_file: Path) -> bool:
        lock_file.parent.mkdir(parents=True, exist_ok=True)
        payload = json.dumps(cls._runtime_cache_lock_payload())
        for _ in range(2):
            try:
                fd = os.open(str(lock_file), os.O_CREAT | os.O_EXCL | os.O_WRONLY)
                with os.fdopen(fd, "w", encoding="utf-8") as fh:
                    fh.write(payload)
                return True
            except FileExistsError:
                if cls._lock_is_stale(lock_file):
                    try:
                        lock_file.unlink()
                    except FileNotFoundError:
                        pass
                    continue
                return False
        return False

    @staticmethod
    def _release_runtime_cache_lock(lock_file: Path) -> None:
        try:
            lock_file.unlink()
        except FileNotFoundError:
            pass

    @classmethod
    def _wait_for_runtime_cache(
        cls,
        snapshot_dir: Path,
        split_names: list[str],
        *,
        scoring_fn: Any | None,
        lock_file: Path,
    ) -> dict[str, Any] | None:
        deadline = time.monotonic() + _RUNTIME_CACHE_WAIT_TIMEOUT_SECONDS
        next_log = time.monotonic()
        while time.monotonic() < deadline:
            cache_file = cls._runtime_cache_file(snapshot_dir, split_names, scoring_fn)
            if cache_file.exists():
                cache_data = cls._try_load_runtime_cache(
                    snapshot_dir,
                    split_names,
                    scoring_fn=scoring_fn,
                )
                if cache_data is not None:
                    return cache_data
            if not lock_file.exists():
                return None
            if cls._lock_is_stale(lock_file):
                cls._release_runtime_cache_lock(lock_file)
                return None
            now = time.monotonic()
            if now >= next_log:
                logger.info(
                    "snapshot_store_runtime_cache_waiting",
                    cache_file=str(cache_file),
                    split="merged" if len(split_names) > 1 else split_names[0],
                )
                next_log = now + _RUNTIME_CACHE_WAIT_LOG_INTERVAL_SECONDS
            time.sleep(1.0)
        return None

    @classmethod
    def _write_runtime_cache(
        cls,
        snapshot_dir: Path,
        split_names: list[str],
        *,
        scoring_fn: Any | None,
        data: dict[str, Any],
    ) -> None:
        cache_file = cls._runtime_cache_file(snapshot_dir, split_names, scoring_fn)
        cache_dir = cache_file.parent
        cache_dir.mkdir(parents=True, exist_ok=True)
        payload = {
            "fingerprint": cls._runtime_cache_fingerprint(snapshot_dir, split_names),
            "data": data,
        }
        tmp_file = cache_file.with_suffix(f"{cache_file.suffix}.tmp.{os.getpid()}")
        tmp_file.write_bytes(pickle.dumps(payload, protocol=pickle.HIGHEST_PROTOCOL))
        os.replace(tmp_file, cache_file)
        logger.info(
            "snapshot_store_runtime_cache_written",
            cache_file=str(cache_file),
            split="merged" if len(split_names) > 1 else split_names[0],
        )

    @staticmethod
    async def _fetch_event_metadata(
        event_ids: list[str],
        db_dsn: str,
    ) -> dict[str, dict[str, Any]]:
        """Batch-query Event + SymbolMaster for event metadata."""
        if not event_ids:
            return {}
        try:
            from sqlalchemy import select
            from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine

            from libs.db.models import Event, SymbolMaster

            engine = create_async_engine(db_dsn, echo=False)
            async_session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)

            result: dict[str, dict[str, Any]] = {}
            async with async_session() as session:
                # Single IN clause for all event_ids
                stmt = (
                    select(Event, SymbolMaster)
                    .outerjoin(SymbolMaster, Event.symbol_id == SymbolMaster.symbol_id)
                    .where(Event.event_id.in_(event_ids))
                )
                rows = (await session.execute(stmt)).all()
                _UTC = __import__("zoneinfo").ZoneInfo("UTC")
                for event, sym in rows:
                    # Use filed_at_utc if available; fallback to filing_date + 21:00 UTC
                    # (transparent enrichment in the loader — not silent substitution in selector)
                    ts = event.filed_at_utc
                    if ts is None and event.event_date is not None:
                        ts = dt.datetime.combine(
                            event.event_date, dt.time(21, 0), tzinfo=_UTC
                        )
                    result[event.event_id] = {
                        "issuer_id": event.issuer_id,
                        "event_date": event.event_date,
                        "event_type": event.event_type,
                        "event_timestamp": ts,
                        "ticker": sym.ticker if sym else None,
                    }
            await engine.dispose()
            return result
        except Exception as exc:
            logger.warning("snapshot_store_db_fetch_failed", error=str(exc))
            return {}

    @staticmethod
    async def _fetch_price_data(
        symbols: list[str],
        date_range: tuple[dt.date, dt.date] | None,
        oracle_url: str,
        concurrency: int = 12,
    ) -> tuple[dict[str, dict[dt.date, dict[str, Any]]], dict[str, float]]:
        """Fetch daily OHLCV bars and compute avg_dollar_volume per symbol."""
        if not symbols or date_range is None:
            return {}, {}
        try:
            from libs.oracle_client import OracleClient, PriceService

            fetch_start = date_range[0] - dt.timedelta(days=_PRICE_FEATURE_WARMUP_DAYS)
            start_str = fetch_start.isoformat()
            end_str = date_range[1].isoformat()

            bars_by_symbol: dict[str, dict[dt.date, dict[str, Any]]] = {}
            avg_dvol: dict[str, float] = {}
            semaphore = asyncio.Semaphore(concurrency)
            total_symbols = len(symbols)
            progress_step = max(25, total_symbols // 10) if total_symbols else 25

            async with OracleClient(base_url=oracle_url) as client:
                svc = PriceService(client)
                async def _fetch_symbol(sym: str) -> tuple[str, dict[dt.date, dict[str, Any]], float]:
                    async with semaphore:
                        try:
                            resp = await svc.get_daily_bars(sym, start=start_str, end=end_str)
                            date_bars: dict[dt.date, dict[str, Any]] = {}
                            dollar_vols: list[float] = []
                            for bar in resp.bars:
                                d = dt.date.fromisoformat(bar.date)
                                b = {
                                    "date": d,
                                    "open": bar.open,
                                    "high": bar.high,
                                    "low": bar.low,
                                    "close": bar.close,
                                    "volume": bar.volume,
                                }
                                date_bars[d] = b
                                dollar_vols.append(bar.close * bar.volume)
                            if dollar_vols:
                                last_20 = dollar_vols[-20:]
                                mean_dvol = sum(last_20) / len(last_20)
                            else:
                                mean_dvol = 0.0
                            return sym, date_bars, mean_dvol
                        except Exception as sym_exc:
                            logger.warning(
                                "snapshot_store_price_fetch_failed",
                                symbol=sym,
                                error=str(sym_exc),
                            )
                            return sym, {}, 0.0

                tasks = [asyncio.create_task(_fetch_symbol(sym)) for sym in symbols]
                completed = 0
                for result in asyncio.as_completed(tasks):
                    sym, date_bars, mean_dvol = await result
                    bars_by_symbol[sym] = date_bars
                    avg_dvol[sym] = mean_dvol
                    completed += 1
                    if completed % progress_step == 0 or completed == total_symbols:
                        logger.info(
                            "snapshot_store_price_fetch_progress",
                            completed=completed,
                            total=total_symbols,
                        )
            return bars_by_symbol, avg_dvol
        except Exception as exc:
            logger.warning("snapshot_store_oracle_failed", error=str(exc))
            return {}, {}

    @staticmethod
    def _backfill_price_derived_features(
        row: dict[str, Any],
        *,
        symbol: str,
        event_date: dt.date,
        bars_by_symbol: dict[str, dict[dt.date, dict[str, Any]]],
        price_bar_cache: dict[str, list[Any]],
        derived_feature_cache: dict[tuple[str, dt.date], dict[str, Any]],
    ) -> None:
        if all(row.get(col) is not None for col in _PRICE_DERIVED_FEATURE_COLUMNS):
            return

        cache_key = (symbol, event_date)
        if cache_key not in derived_feature_cache:
            price_bars = SnapshotStore._get_price_bars_for_symbol(
                symbol,
                bars_by_symbol=bars_by_symbol,
                price_bar_cache=price_bar_cache,
            )
            if not price_bars:
                derived_feature_cache[cache_key] = {}
                return
            features = compute_market_features(price_bars, event_date.isoformat())
            derived_feature_cache[cache_key] = {
                col: features.get(col)
                for col in _PRICE_DERIVED_FEATURE_COLUMNS
            }

        for col, value in derived_feature_cache[cache_key].items():
            if row.get(col) is None and value is not None:
                row[col] = value

    @classmethod
    def _backfill_avg_dollar_volume_features(
        cls,
        row: dict[str, Any],
        *,
        symbol: str,
        event_date: dt.date | None,
        bars_by_symbol: dict[str, dict[dt.date, dict[str, Any]]],
        price_bar_cache: dict[str, list[Any]],
        fallback_avg_dvol: float,
    ) -> None:
        existing_adv_20d = row.get("avg_dollar_volume_20d")
        existing_adv = row.get("avg_dollar_volume")
        if existing_adv_20d is not None and existing_adv is not None:
            return

        derived_adv = None
        if event_date is not None:
            price_bars = cls._get_price_bars_for_symbol(
                symbol,
                bars_by_symbol=bars_by_symbol,
                price_bar_cache=price_bar_cache,
            )
            if price_bars:
                features = compute_market_features(price_bars, event_date.isoformat())
                derived_adv = features.get("avg_dollar_volume_20d")

        resolved_adv = (
            existing_adv_20d
            if existing_adv_20d is not None
            else derived_adv
            if derived_adv is not None
            else existing_adv
            if existing_adv is not None
            else fallback_avg_dvol
        )
        if row.get("avg_dollar_volume_20d") is None:
            row["avg_dollar_volume_20d"] = resolved_adv
        if row.get("avg_dollar_volume") is None:
            row["avg_dollar_volume"] = resolved_adv

    @staticmethod
    def _get_price_bars_for_symbol(
        symbol: str,
        *,
        bars_by_symbol: dict[str, dict[dt.date, dict[str, Any]]],
        price_bar_cache: dict[str, list[Any]],
    ) -> list[Any]:
        if symbol in price_bar_cache:
            return price_bar_cache[symbol]

        symbol_bars = bars_by_symbol.get(symbol) or {}
        if not symbol_bars:
            price_bar_cache[symbol] = []
            return price_bar_cache[symbol]

        from libs.oracle_client.models import PriceBar

        price_bar_cache[symbol] = [
            PriceBar(
                date=bar_date.isoformat(),
                open=float(bar.get("open", 0.0) or 0.0),
                high=float(bar.get("high", 0.0) or 0.0),
                low=float(bar.get("low", 0.0) or 0.0),
                close=float(bar.get("close", 0.0) or 0.0),
                volume=int(bar.get("volume", 0) or 0),
            )
            for bar_date, bar in sorted(symbol_bars.items())
        ]
        return price_bar_cache[symbol]

    @staticmethod
    def _inject_macro_trade_bars(
        bars_by_symbol_date: dict[str, dict[dt.date, dict[str, Any]]],
        macro_by_date: dict[dt.date, dict[str, Any]],
    ) -> None:
        """Expose macro ETFs as normal OHLCV bars for synthetic engines."""
        for symbol, prefix in _MACRO_TRADE_BAR_PREFIXES:
            injected: dict[dt.date, dict[str, Any]] = {}
            for bar_date, macro_vals in macro_by_date.items():
                close_value = macro_vals.get(f"{prefix}_close")
                if close_value is None or float(close_value) <= 0:
                    continue
                injected[bar_date] = {
                    "open": macro_vals.get(f"{prefix}_open", close_value),
                    "high": macro_vals.get(f"{prefix}_high", close_value),
                    "low": macro_vals.get(f"{prefix}_low", close_value),
                    "close": close_value,
                    "volume": macro_vals.get(f"{prefix}_volume", 1_000_000),
                }
            if not injected:
                continue
            existing = bars_by_symbol_date.setdefault(symbol, {})
            for bar_date, bar in injected.items():
                existing.setdefault(bar_date, bar)

    @classmethod
    def _attach_sector_etf_proxy_features(
        cls,
        row: dict[str, Any],
        *,
        sector: str | None,
        reaction_date: dt.date,
        execution_date: dt.date,
        bars_by_symbol: dict[str, dict[dt.date, dict[str, Any]]],
        avg_dvol: dict[str, float],
        price_bar_cache: dict[str, list[Any]],
    ) -> None:
        proxy_symbol = sector_etf_for_sector(sector)
        if not proxy_symbol:
            return

        row["sector_etf_proxy"] = proxy_symbol
        proxy_bars = cls._get_price_bars_for_symbol(
            proxy_symbol,
            bars_by_symbol=bars_by_symbol,
            price_bar_cache=price_bar_cache,
        )
        if not proxy_bars:
            return

        proxy_features = compute_market_features(proxy_bars, reaction_date.isoformat())
        row["sector_etf_event_close"] = proxy_features.get("event_close")
        row["sector_etf_reaction_day_low"] = proxy_features.get("reaction_day_low")
        row["sector_etf_reaction_day_high"] = proxy_features.get("reaction_day_high")
        row["sector_etf_reaction_day_return"] = proxy_features.get("reaction_day_return")
        row["sector_etf_volume_ratio_20d"] = proxy_features.get("volume_ratio_20d")
        row["sector_etf_gap_size"] = proxy_features.get("gap_size")
        row["sector_etf_close_location"] = proxy_features.get("close_location")
        row["sector_etf_avg_dollar_volume"] = (
            proxy_features.get("avg_dollar_volume_20d")
            or avg_dvol.get(proxy_symbol, 0.0)
        )
        row["sector_etf_atr_14"] = proxy_features.get("atr_14")

        execution_bar = (bars_by_symbol.get(proxy_symbol) or {}).get(execution_date)
        if execution_bar is not None:
            row["sector_etf_entry_price"] = (
                execution_bar.get("open")
                or execution_bar.get("close")
            )
        else:
            row["sector_etf_entry_price"] = row.get("sector_etf_event_close")

    @classmethod
    def _attach_peer_proxy_features(
        cls,
        row: dict[str, Any],
        *,
        source_symbol: str,
        sector: str | None,
        reaction_date: dt.date,
        execution_date: dt.date,
        event_symbols_by_exec_date: dict[dt.date, set[str]],
        bars_by_symbol: dict[str, dict[dt.date, dict[str, Any]]],
        avg_dvol: dict[str, float],
        price_bar_cache: dict[str, list[Any]],
    ) -> None:
        blocked_symbols = {
            str(symbol).upper()
            for symbol in event_symbols_by_exec_date.get(execution_date, set())
            if symbol
        }
        for proxy_symbol in peer_candidates_for_symbol(source_symbol, sector):
            if proxy_symbol in blocked_symbols:
                continue
            proxy_bars = cls._get_price_bars_for_symbol(
                proxy_symbol,
                bars_by_symbol=bars_by_symbol,
                price_bar_cache=price_bar_cache,
            )
            if not proxy_bars:
                continue
            proxy_features = compute_market_features(proxy_bars, reaction_date.isoformat())
            execution_bar = (bars_by_symbol.get(proxy_symbol) or {}).get(execution_date)
            entry_price = None
            if execution_bar is not None:
                entry_price = execution_bar.get("open") or execution_bar.get("close")
            if not entry_price:
                entry_price = proxy_features.get("event_close")
            if not entry_price:
                continue

            row["peer_proxy_symbol"] = proxy_symbol
            row["peer_proxy_event_close"] = proxy_features.get("event_close")
            row["peer_proxy_reaction_day_low"] = proxy_features.get("reaction_day_low")
            row["peer_proxy_reaction_day_high"] = proxy_features.get("reaction_day_high")
            row["peer_proxy_reaction_day_return"] = proxy_features.get("reaction_day_return")
            row["peer_proxy_volume_ratio_20d"] = proxy_features.get("volume_ratio_20d")
            row["peer_proxy_gap_size"] = proxy_features.get("gap_size")
            row["peer_proxy_close_location"] = proxy_features.get("close_location")
            row["peer_proxy_avg_dollar_volume"] = (
                proxy_features.get("avg_dollar_volume_20d")
                or avg_dvol.get(proxy_symbol, 0.0)
            )
            row["peer_proxy_atr_14"] = proxy_features.get("atr_14")
            row["peer_proxy_entry_price"] = entry_price
            break

    @staticmethod
    def _lookup_as_of(pairs: list[tuple[dt.date, float]], as_of: dt.date) -> float | None:
        lo, hi = 0, len(pairs) - 1
        result = None
        while lo <= hi:
            mid = (lo + hi) // 2
            if pairs[mid][0] <= as_of:
                result = pairs[mid][1]
                lo = mid + 1
            else:
                hi = mid - 1
        return result

    @staticmethod
    async def _fetch_sectors(
        symbols: list[str],
        oracle_url: str,
        concurrency: int = 12,
    ) -> dict[str, str]:
        """Fetch company sector for each symbol. Default 'UNKNOWN' if unavailable."""
        if not symbols:
            return {}
        cache = SnapshotStore._load_sector_cache()
        result: dict[str, str] = {}
        semaphore = asyncio.Semaphore(concurrency)
        total_symbols = len(symbols)
        progress_step = max(25, total_symbols // 10) if total_symbols else 25
        try:
            from libs.oracle_client import CompanyService, OracleClient

            async with OracleClient(base_url=oracle_url) as client:
                company_svc = CompanyService(client)
                async def _fetch_sector(sym: str) -> tuple[str, str]:
                    async with semaphore:
                        cached = cache.get(sym)
                        if isinstance(cached, str) and cached:
                            return sym, cached
                        try:
                            info = await company_svc.get_company(sym)
                            sector = info.sector or "UNKNOWN"
                            if SnapshotStore._sector_info_is_placeholder(info):
                                yf_sector = SnapshotStore._fetch_sector_from_yfinance(sym)
                                if isinstance(yf_sector, str) and yf_sector and yf_sector != "UNKNOWN":
                                    sector = yf_sector
                            return sym, sector
                        except Exception:
                            return sym, cached or "UNKNOWN"

                tasks = [asyncio.create_task(_fetch_sector(sym)) for sym in symbols]
                completed = 0
                for sector_result in asyncio.as_completed(tasks):
                    sym, sector = await sector_result
                    result[sym] = sector
                    completed += 1
                    if completed % progress_step == 0 or completed == total_symbols:
                        logger.info(
                            "snapshot_store_sector_fetch_progress",
                            completed=completed,
                            total=total_symbols,
                        )
        except Exception as exc:
            logger.warning("snapshot_store_sector_fetch_failed", error=str(exc))

        # Default all remaining to UNKNOWN
        for sym in symbols:
            result.setdefault(sym, cache.get(sym, "UNKNOWN"))
            if result[sym] != "UNKNOWN":
                cache[sym] = result[sym]
        SnapshotStore._write_sector_cache(cache)
        return result

    @staticmethod
    def _sector_cache_path() -> Path:
        settings = get_settings()
        return Path(settings.data_root) / "cache" / "sector_cache.json"

    @staticmethod
    def _load_sector_cache() -> dict[str, str]:
        path = SnapshotStore._sector_cache_path()
        if not path.exists():
            return {}
        try:
            return json.loads(path.read_text())
        except Exception:
            return {}

    @staticmethod
    def _write_sector_cache(cache: dict[str, str]) -> None:
        path = SnapshotStore._sector_cache_path()
        path.parent.mkdir(parents=True, exist_ok=True)
        path.write_text(json.dumps(dict(sorted(cache.items())), indent=2))

    @staticmethod
    def _sector_info_is_placeholder(info: Any) -> bool:
        sector = getattr(info, "sector", None)
        industry = getattr(info, "industry", None)
        exchange = getattr(info, "exchange", None)
        market_cap = getattr(info, "market_cap", None)
        return (
            sector == "Technology"
            and industry == "Software"
            and exchange is None
            and market_cap is None
        )

    @staticmethod
    def _fetch_sector_from_yfinance(symbol: str) -> str:
        try:
            import yfinance as yf

            info = yf.Ticker(symbol).get_info()
            sector = info.get("sector")
            if isinstance(sector, str) and sector.strip():
                return sector.strip()
        except Exception:
            return "UNKNOWN"
        return "UNKNOWN"

    @staticmethod
    async def _fetch_macro(
        date_range: tuple[dt.date, dt.date] | None,
        db_dsn: str,
    ) -> dict[dt.date, dict[str, Any]]:
        """Load MacroObservation regime data from DB."""
        if date_range is None:
            return {}
        try:
            from sqlalchemy import select
            from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine

            from libs.db.models import MacroObservation

            engine = create_async_engine(db_dsn, echo=False)
            async_session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)

            result: dict[dt.date, dict[str, Any]] = {}
            async with async_session() as session:
                stmt = select(MacroObservation).where(
                    MacroObservation.observation_date >= date_range[0],
                    MacroObservation.observation_date <= date_range[1],
                )
                rows = (await session.execute(stmt)).scalars().all()
                for obs in rows:
                    d = obs.observation_date
                    result.setdefault(d, {})[obs.series_id] = (
                        float(obs.value) if obs.value is not None else None
                    )
            await engine.dispose()
            return result
        except Exception as exc:
            logger.warning("snapshot_store_macro_fetch_failed", error=str(exc))
            return {}

    @staticmethod
    async def _fetch_spy_macro(
        date_range: tuple[dt.date, dt.date] | None,
        oracle_url: str,
        sma_periods: tuple[int, ...] = (10, 20, 30, 40, 50),
    ) -> dict[dt.date, dict[str, Any]]:
        """Fetch SPY/QQQ daily bars and compute SMAs for macro regime filtering.

        Returns dict with benchmark closes/SMAs merged by date.
        SMA is None for the first (period - 1) bars of each period.
        """
        if date_range is None:
            return {}
        try:
            from libs.oracle_client import OracleClient, PriceService

            max_period = max(sma_periods)
            # Extend start date back by max SMA period for warm-up
            warmup_days = max(max_period, 80) * 2  # calendar days (Hurst needs 60 trading days)
            extended_start = date_range[0] - dt.timedelta(days=warmup_days)

            async with OracleClient(base_url=oracle_url) as client:
                svc = PriceService(client)
                spy_resp = await svc.get_daily_bars(
                    "SPY", start=extended_start.isoformat(), end=date_range[1].isoformat()
                )
                qqq_resp = await svc.get_daily_bars(
                    "QQQ", start=extended_start.isoformat(), end=date_range[1].isoformat()
                )
                extra_symbol_responses: dict[str, Any | None] = {}
                for extra_symbol in (
                    "TQQQ",
                    "QQQM",
                    "SPYM",
                    "QUAL",
                    "SH",
                    "PSQ",
                    "GLD",
                    "SHY",
                    "USFR",
                    "BIL",
                    "VGSH",
                    "IEI",
                    "IEF",
                    "TIP",
                    "XLK",
                    "SMH",
                    "IWM",
                    "XLI",
                    "XLE",
                    "XLF",
                    "DBC",
                    "SGOV",
                    "JEPQ",
                    "BUFB",
                    "MERIX",
                ):
                    try:
                        extra_symbol_responses[extra_symbol] = await svc.get_daily_bars(
                            extra_symbol,
                            start=extended_start.isoformat(),
                            end=date_range[1].isoformat(),
                        )
                    except Exception:
                        extra_symbol_responses[extra_symbol] = None

                tqqq_resp = extra_symbol_responses.get("TQQQ")
                qqqm_resp = extra_symbol_responses.get("QQQM")
                spym_resp = extra_symbol_responses.get("SPYM")
                qual_resp = extra_symbol_responses.get("QUAL")
                sh_resp = extra_symbol_responses.get("SH")
                psq_resp = extra_symbol_responses.get("PSQ")
                gld_resp = extra_symbol_responses.get("GLD")
                shy_resp = extra_symbol_responses.get("SHY")
                usfr_resp = extra_symbol_responses.get("USFR")
                bil_resp = extra_symbol_responses.get("BIL")
                vgsh_resp = extra_symbol_responses.get("VGSH")
                iei_resp = extra_symbol_responses.get("IEI")
                ief_resp = extra_symbol_responses.get("IEF")
                tip_resp = extra_symbol_responses.get("TIP")
                xlk_resp = extra_symbol_responses.get("XLK")
                smh_resp = extra_symbol_responses.get("SMH")
                iwm_resp = extra_symbol_responses.get("IWM")
                xli_resp = extra_symbol_responses.get("XLI")
                xle_resp = extra_symbol_responses.get("XLE")
                xlf_resp = extra_symbol_responses.get("XLF")
                dbc_resp = extra_symbol_responses.get("DBC")
                sgov_resp = extra_symbol_responses.get("SGOV")
                jepq_resp = extra_symbol_responses.get("JEPQ")
                bufb_resp = extra_symbol_responses.get("BUFB")
                merix_resp = extra_symbol_responses.get("MERIX")

            result: dict[dt.date, dict[str, Any]] = {}

            # Lookback periods for rolling high / momentum / volatility
            rolling_high_periods = (5, 20, 50, 100)
            momentum_periods = (5, 10, 20, 50)
            vol_periods = (5, 15, 20, 30, 50)
            efficiency_periods = (10, 20)
            downside_vol_periods = (10, 20)
            ulcer_periods = (10, 20)
            drawdown_accel_days = 5

            def _merge_series(prefix: str, bars: Any) -> None:
                import math
                sorted_bars = sorted(bars, key=lambda b: b.date)
                closes: list[tuple[dt.date, float]] = [
                    (dt.date.fromisoformat(b.date), float(b.close)) for b in sorted_bars
                ]
                for i, (d, close) in enumerate(closes):
                    if d < date_range[0]:
                        continue
                    result.setdefault(d, {})
                    result[d][f"{prefix}_close"] = close
                    result[d][f"{prefix}_open"] = float(sorted_bars[i].open)
                    result[d][f"{prefix}_high"] = float(sorted_bars[i].high)
                    result[d][f"{prefix}_low"] = float(sorted_bars[i].low)
                    result[d][f"{prefix}_volume"] = float(sorted_bars[i].volume)
                    for period in sma_periods:
                        sma = None
                        if i >= period - 1:
                            window = [c for _, c in closes[i - period + 1 : i + 1]]
                            sma = sum(window) / len(window)
                        result[d][f"{prefix}_sma_{period}"] = sma
                    # Rolling high (for drawdown gate)
                    for rh_p in rolling_high_periods:
                        rh = None
                        if i >= rh_p - 1:
                            rh = max(c for _, c in closes[i - rh_p + 1 : i + 1])
                        result[d][f"{prefix}_high_{rh_p}"] = rh
                    # Momentum / N-day return (for momentum gate)
                    for mom_p in momentum_periods:
                        mom = None
                        if i >= mom_p:
                            prev_close = closes[i - mom_p][1]
                            if prev_close > 0:
                                mom = (close - prev_close) / prev_close
                        result[d][f"{prefix}_mom_{mom_p}"] = mom
                    # Realized volatility (annualized std of daily log returns)
                    for vol_p in vol_periods:
                        vol = None
                        if i >= vol_p:
                            log_rets = []
                            for j in range(i - vol_p + 1, i + 1):
                                if closes[j - 1][1] > 0:
                                    log_rets.append(math.log(closes[j][1] / closes[j - 1][1]))
                            if len(log_rets) >= vol_p - 1:
                                mean_r = sum(log_rets) / len(log_rets)
                                var_r = sum((r - mean_r) ** 2 for r in log_rets) / len(log_rets)
                                vol = math.sqrt(var_r * 252)  # annualized
                        result[d][f"{prefix}_vol_{vol_p}"] = vol
                    # Trend efficiency ratio (Kaufman): net progress / total path length.
                    # High = smooth persistent trend, low = noisy/random walk.
                    for eff_p in efficiency_periods:
                        efficiency = None
                        if i >= eff_p:
                            net_move = abs(close - closes[i - eff_p][1])
                            gross_move = sum(
                                abs(closes[j][1] - closes[j - 1][1])
                                for j in range(i - eff_p + 1, i + 1)
                            )
                            if gross_move > 0:
                                efficiency = net_move / gross_move
                            else:
                                efficiency = 0.0
                        result[d][f"{prefix}_efficiency_{eff_p}"] = efficiency
                    # Downside semivolatility: only penalize harmful volatility.
                    for dv_p in downside_vol_periods:
                        downside_vol = None
                        if i >= dv_p:
                            neg_sq = []
                            for j in range(i - dv_p + 1, i + 1):
                                if closes[j - 1][1] > 0:
                                    ret = closes[j][1] / closes[j - 1][1] - 1
                                    neg_sq.append(min(ret, 0.0) ** 2)
                            if len(neg_sq) >= dv_p - 1:
                                downside_vol = math.sqrt(sum(neg_sq) / len(neg_sq) * 252)
                        result[d][f"{prefix}_downside_vol_{dv_p}"] = downside_vol
                    # Shannon entropy of daily returns (measures market predictability)
                    # Low entropy = trending (predictable), high entropy = chaotic (uncertain)
                    for ent_p in (10, 20):
                        entropy = None
                        if i >= ent_p:
                            # Bin daily returns into categories
                            daily_rets = []
                            for j in range(i - ent_p + 1, i + 1):
                                if closes[j - 1][1] > 0:
                                    daily_rets.append(closes[j][1] / closes[j - 1][1] - 1)
                            if len(daily_rets) >= ent_p - 1:
                                # Count positive/negative/flat days
                                n_pos = sum(1 for r in daily_rets if r > 0.001)
                                n_neg = sum(1 for r in daily_rets if r < -0.001)
                                n_flat = len(daily_rets) - n_pos - n_neg
                                n_total = len(daily_rets)
                                # Shannon entropy H = -sum(p * log2(p))
                                entropy = 0.0
                                for count in (n_pos, n_neg, n_flat):
                                    if count > 0:
                                        p = count / n_total
                                        entropy -= p * math.log2(p)
                        result[d][f"{prefix}_entropy_{ent_p}"] = entropy
                    # Ulcer index: rolling RMS drawdown pain from recent peaks.
                    for ulcer_p in ulcer_periods:
                        ulcer = None
                        current_dd = None
                        if i >= ulcer_p - 1:
                            window_closes = [c for _, c in closes[i - ulcer_p + 1 : i + 1]]
                            peak_in_window = 0.0
                            drawdowns = []
                            for window_close in window_closes:
                                peak_in_window = max(peak_in_window, window_close)
                                if peak_in_window > 0:
                                    drawdowns.append(window_close / peak_in_window - 1.0)
                            if drawdowns:
                                ulcer = math.sqrt(sum(dd * dd for dd in drawdowns) / len(drawdowns))
                                current_dd = abs(drawdowns[-1])
                        result[d][f"{prefix}_ulcer_{ulcer_p}"] = ulcer
                        result[d][f"{prefix}_drawdown_{ulcer_p}"] = current_dd
                    # Drawdown acceleration: how quickly recent pain is worsening.
                    dd_accel = None
                    dd_lb = 20
                    if i >= dd_lb - 1 + drawdown_accel_days:
                        cur_window = [c for _, c in closes[i - dd_lb + 1 : i + 1]]
                        prev_i = i - drawdown_accel_days
                        prev_window = [c for _, c in closes[prev_i - dd_lb + 1 : prev_i + 1]]
                        cur_peak = max(cur_window) if cur_window else 0.0
                        prev_peak = max(prev_window) if prev_window else 0.0
                        if cur_peak > 0 and prev_peak > 0:
                            cur_dd = (cur_peak - close) / cur_peak
                            prev_close = closes[prev_i][1]
                            prev_dd = (prev_peak - prev_close) / prev_peak
                            dd_accel = cur_dd - prev_dd
                    result[d][f"{prefix}_drawdown_accel_{drawdown_accel_days}"] = dd_accel

                    # Hurst exponent via R/S analysis (fractal dimension)
                    # H > 0.5 = trending/persistent, H < 0.5 = mean-reverting, H ≈ 0.5 = random walk
                    hurst_lookback = 60
                    hurst = None
                    if i >= hurst_lookback + 1:
                        h_rets = []
                        for j in range(i - hurst_lookback, i):
                            if closes[j][1] > 0:
                                h_rets.append((closes[j + 1][1] - closes[j][1]) / closes[j][1])
                        if len(h_rets) >= 30:
                            def _rs_stat(series: list[float]) -> float:
                                n_ = len(series)
                                mean_ = sum(series) / n_
                                devs = [x - mean_ for x in series]
                                cumdev = []
                                s_ = 0.0
                                for dd in devs:
                                    s_ += dd
                                    cumdev.append(s_)
                                r_ = max(cumdev) - min(cumdev)
                                std_ = (sum(dd ** 2 for dd in devs) / n_) ** 0.5
                                return r_ / std_ if std_ > 0 else 0.0

                            win_sizes = [s for s in [8, 12, 16, 24, 32] if s <= len(h_rets) // 2]
                            if len(win_sizes) >= 2:
                                log_n, log_rs = [], []
                                for w in win_sizes:
                                    rs_vals = []
                                    for st in range(0, len(h_rets) - w + 1, w):
                                        chunk = h_rets[st:st + w]
                                        if len(chunk) == w:
                                            rs_vals.append(_rs_stat(chunk))
                                    if rs_vals:
                                        avg_rs = sum(rs_vals) / len(rs_vals)
                                        if avg_rs > 0:
                                            log_n.append(math.log(w))
                                            log_rs.append(math.log(avg_rs))
                                if len(log_n) >= 2:
                                    n_h = len(log_n)
                                    x_m = sum(log_n) / n_h
                                    y_m = sum(log_rs) / n_h
                                    num = sum((log_n[k] - x_m) * (log_rs[k] - y_m) for k in range(n_h))
                                    den = sum((log_n[k] - x_m) ** 2 for k in range(n_h))
                                    hurst = num / den if den > 0 else 0.5
                    result[d][f"{prefix}_hurst_60"] = hurst

                    # Rolling excess kurtosis (fat tail detection)
                    # Normal = 0, high = extreme moves more likely (Mandelbrot/Taleb)
                    kurt_lookback = 20
                    kurtosis = None
                    if i >= kurt_lookback:
                        k_rets = []
                        for j in range(i - kurt_lookback + 1, i + 1):
                            if closes[j - 1][1] > 0:
                                k_rets.append(math.log(closes[j][1] / closes[j - 1][1]))
                        if len(k_rets) >= kurt_lookback - 1:
                            k_mean = sum(k_rets) / len(k_rets)
                            k_var = sum((r - k_mean) ** 2 for r in k_rets) / len(k_rets)
                            if k_var > 1e-12:
                                m4 = sum((r - k_mean) ** 4 for r in k_rets) / len(k_rets)
                                kurtosis = m4 / (k_var ** 2) - 3.0
                    result[d][f"{prefix}_kurtosis_20"] = kurtosis

                    # Return autocorrelation (lag-1 Pearson correlation)
                    # Positive = trending, negative = mean-reverting (Lo, 2004)
                    ac_lookback = 20
                    autocorr = None
                    if i >= ac_lookback + 1:
                        ac_rets = []
                        for j in range(i - ac_lookback, i + 1):
                            if closes[j - 1][1] > 0:
                                ac_rets.append(closes[j][1] / closes[j - 1][1] - 1)
                        if len(ac_rets) >= ac_lookback:
                            x_ac = ac_rets[:-1]
                            y_ac = ac_rets[1:]
                            n_ac = len(x_ac)
                            mx = sum(x_ac) / n_ac
                            my = sum(y_ac) / n_ac
                            cov_xy = sum((x_ac[k] - mx) * (y_ac[k] - my) for k in range(n_ac)) / n_ac
                            sx = (sum((x_ac[k] - mx) ** 2 for k in range(n_ac)) / n_ac) ** 0.5
                            sy = (sum((y_ac[k] - my) ** 2 for k in range(n_ac)) / n_ac) ** 0.5
                            if sx > 1e-12 and sy > 1e-12:
                                autocorr = cov_xy / (sx * sy)
                    result[d][f"{prefix}_autocorr_20"] = autocorr

            _merge_series("spy", spy_resp.bars)
            _merge_series("qqq", qqq_resp.bars)
            if spym_resp and hasattr(spym_resp, "bars") and spym_resp.bars:
                _merge_series("spym", spym_resp.bars)
            if qual_resp and hasattr(qual_resp, "bars") and qual_resp.bars:
                _merge_series("qual", qual_resp.bars)
            if xlk_resp and hasattr(xlk_resp, "bars") and xlk_resp.bars:
                _merge_series("xlk", xlk_resp.bars)
            if smh_resp and hasattr(smh_resp, "bars") and smh_resp.bars:
                _merge_series("smh", smh_resp.bars)
            if iwm_resp and hasattr(iwm_resp, "bars") and iwm_resp.bars:
                _merge_series("iwm", iwm_resp.bars)
            if xli_resp and hasattr(xli_resp, "bars") and xli_resp.bars:
                _merge_series("xli", xli_resp.bars)
            if xle_resp and hasattr(xle_resp, "bars") and xle_resp.bars:
                _merge_series("xle", xle_resp.bars)
            if xlf_resp and hasattr(xlf_resp, "bars") and xlf_resp.bars:
                _merge_series("xlf", xlf_resp.bars)

            def _merge_pair_correlation(left_prefix: str, right_prefix: str, output_key: str) -> None:
                # Rolling cross-asset correlation (regime shift detection).
                corr_lookback = 20
                sorted_result_dates = sorted(result.keys())
                for idx_c, d_c in enumerate(sorted_result_dates):
                    corr_val = None
                    if idx_c >= corr_lookback:
                        left_r, right_r = [], []
                        for jj in range(idx_c - corr_lookback + 1, idx_c + 1):
                            d_j = sorted_result_dates[jj]
                            d_prev = sorted_result_dates[jj - 1]
                            lc = result.get(d_j, {}).get(f"{left_prefix}_close")
                            lp = result.get(d_prev, {}).get(f"{left_prefix}_close")
                            rc = result.get(d_j, {}).get(f"{right_prefix}_close")
                            rp = result.get(d_prev, {}).get(f"{right_prefix}_close")
                            if all(v and v > 0 for v in [lc, lp, rc, rp]):
                                left_r.append(lc / lp - 1)
                                right_r.append(rc / rp - 1)
                        if len(left_r) >= corr_lookback - 2:
                            import math as _m

                            n_cr = len(left_r)
                            mx_l = sum(left_r) / n_cr
                            mx_r = sum(right_r) / n_cr
                            cov_lr = sum(
                                (left_r[k] - mx_l) * (right_r[k] - mx_r) for k in range(n_cr)
                            ) / n_cr
                            ss_l = _m.sqrt(sum((left_r[k] - mx_l) ** 2 for k in range(n_cr)) / n_cr)
                            ss_r = _m.sqrt(sum((right_r[k] - mx_r) ** 2 for k in range(n_cr)) / n_cr)
                            if ss_l > 1e-12 and ss_r > 1e-12:
                                corr_val = cov_lr / (ss_l * ss_r)
                    result[d_c][output_key] = corr_val

            # SPY/SPYM-vs-QQQ rolling correlation (regime shift detection)
            _merge_pair_correlation("spy", "qqq", "spy_qqq_corr_20")
            if spym_resp and hasattr(spym_resp, "bars") and spym_resp.bars:
                _merge_pair_correlation("spym", "qqq", "spym_qqq_corr_20")
            if qual_resp and hasattr(qual_resp, "bars") and qual_resp.bars:
                _merge_pair_correlation("qual", "qqq", "qual_qqq_corr_20")

            if tqqq_resp and hasattr(tqqq_resp, 'bars') and tqqq_resp.bars:
                # TQQQ: only need close price, skip indicators
                for b in sorted(tqqq_resp.bars, key=lambda b: b.date):
                    d = dt.date.fromisoformat(b.date)
                    if d >= date_range[0]:
                        result.setdefault(d, {})
                        result[d]["tqqq_close"] = float(b.close)

            if qqqm_resp and hasattr(qqqm_resp, 'bars') and qqqm_resp.bars:
                # QQQM: close price for parking (lower ER than QQQ)
                for b in sorted(qqqm_resp.bars, key=lambda b: b.date):
                    d = dt.date.fromisoformat(b.date)
                    if d >= date_range[0]:
                        result.setdefault(d, {})
                        result[d]["qqqm_close"] = float(b.close)

            if sgov_resp and hasattr(sgov_resp, 'bars') and sgov_resp.bars:
                # SGOV: iShares 0-3 Month Treasury Bond ETF — close price for real parking
                for b in sorted(sgov_resp.bars, key=lambda b: b.date):
                    d = dt.date.fromisoformat(b.date)
                    if d >= date_range[0]:
                        result.setdefault(d, {})
                        result[d]["sgov_close"] = float(b.close)

            if sh_resp and hasattr(sh_resp, 'bars') and sh_resp.bars:
                # SH: ProShares Short S&P 500 — full indicators for macro_short engine ATR calculation
                _merge_series("sh", sh_resp.bars)

            if psq_resp and hasattr(psq_resp, 'bars') and psq_resp.bars:
                _merge_series("psq", psq_resp.bars)
            if gld_resp and hasattr(gld_resp, 'bars') and gld_resp.bars:
                _merge_series("gld", gld_resp.bars)
            if shy_resp and hasattr(shy_resp, 'bars') and shy_resp.bars:
                _merge_series("shy", shy_resp.bars)
            if usfr_resp and hasattr(usfr_resp, 'bars') and usfr_resp.bars:
                _merge_series("usfr", usfr_resp.bars)
            if bil_resp and hasattr(bil_resp, 'bars') and bil_resp.bars:
                _merge_series("bil", bil_resp.bars)
            if vgsh_resp and hasattr(vgsh_resp, 'bars') and vgsh_resp.bars:
                _merge_series("vgsh", vgsh_resp.bars)
            if iei_resp and hasattr(iei_resp, 'bars') and iei_resp.bars:
                _merge_series("iei", iei_resp.bars)
            if ief_resp and hasattr(ief_resp, 'bars') and ief_resp.bars:
                _merge_series("ief", ief_resp.bars)
            if tip_resp and hasattr(tip_resp, 'bars') and tip_resp.bars:
                _merge_series("tip", tip_resp.bars)
            if dbc_resp and hasattr(dbc_resp, 'bars') and dbc_resp.bars:
                _merge_series("dbc", dbc_resp.bars)
            if jepq_resp and hasattr(jepq_resp, 'bars') and jepq_resp.bars:
                _merge_series("jepq", jepq_resp.bars)
            if bufb_resp and hasattr(bufb_resp, 'bars') and bufb_resp.bars:
                _merge_series("bufb", bufb_resp.bars)
            if merix_resp and hasattr(merix_resp, 'bars') and merix_resp.bars:
                _merge_series("merix", merix_resp.bars)

            # Fetch VIX index for macro regime filtering (VIXCLS equivalent)
            # Used as fallback when the snapshot Parquet doesn't embed macro_vix
            try:
                async with OracleClient(base_url=oracle_url) as vix_client:
                    vix_svc = PriceService(vix_client)
                    vix_resp = await vix_svc.get_daily_bars(
                        "^VIX", start=extended_start.isoformat(), end=date_range[1].isoformat()
                    )
                for b in sorted(vix_resp.bars, key=lambda b: b.date):
                    d = dt.date.fromisoformat(b.date)
                    if d >= date_range[0]:
                        result.setdefault(d, {})
                        result[d]["VIXCLS"] = float(b.close)
            except Exception:
                pass  # VIX data optional — falls back to Parquet-embedded macro_vix

            logger.info(
                "snapshot_store_spy_macro_loaded",
                bars=len(spy_resp.bars),
                dates_with_sma=sum(1 for v in result.values() if v.get("spy_sma_20") is not None),
            )
            return result
        except Exception as exc:
            logger.warning("snapshot_store_spy_macro_failed", error=str(exc))
            return {}

    @staticmethod
    def _compute_date_range(
        rows: list[dict[str, Any]],
    ) -> tuple[dt.date, dt.date] | None:
        """Compute (min_date, max_date) from execution and reaction-date columns."""
        dates: list[dt.date] = []
        for r in rows:
            for raw in (
                r.get("event_date"),
                r.get("entry_date"),
                r.get("execution_date"),
                r.get("reaction_date"),
            ):
                if raw is None:
                    continue
                if isinstance(raw, str):
                    try:
                        dates.append(dt.date.fromisoformat(raw))
                    except ValueError:
                        pass
                elif isinstance(raw, dt.date):
                    dates.append(raw)
        if not dates:
            return None
        return min(dates), max(dates)

    @staticmethod
    def _build_reaction_index(
        candidates_by_exec_date: dict[dt.date, list[dict[str, Any]]],
    ) -> dict[dt.date, list[dict[str, Any]]]:
        reaction_index: dict[dt.date, list[dict[str, Any]]] = {}
        for rows in candidates_by_exec_date.values():
            for row in rows:
                reaction_date = SnapshotStore._normalize_date(row.get("reaction_date"))
                if reaction_date is None:
                    continue
                reaction_index.setdefault(reaction_date, []).append(row)
        return reaction_index

    @staticmethod
    def _normalize_date(raw: Any) -> dt.date | None:
        if isinstance(raw, dt.datetime):
            return raw.date()
        if isinstance(raw, dt.date):
            return raw
        if isinstance(raw, str):
            try:
                return dt.date.fromisoformat(raw)
            except ValueError:
                return None
        return None

    @staticmethod
    def _normalize_timestamp(raw: Any, fallback_event_date: dt.date | None = None) -> dt.datetime | None:
        if isinstance(raw, dt.datetime):
            return raw if raw.tzinfo is not None else raw.replace(tzinfo=_UTC)
        if isinstance(raw, str):
            try:
                parsed = dt.datetime.fromisoformat(raw)
            except ValueError:
                parsed = None
            if parsed is not None:
                return parsed if parsed.tzinfo is not None else parsed.replace(tzinfo=_UTC)
        if fallback_event_date is not None:
            return dt.datetime.combine(fallback_event_date, dt.time(21, 0), tzinfo=_UTC)
        return None
