"""Synthetic macro data generation for scenario backtesting. Generates the full macro_by_date dict consumed by SnapshotStore, including: - SPY/QQQ rolling indicators (SMA, momentum, vol, entropy, Hurst, etc.) - VIX via Ornstein-Uhlenbeck process correlated with market returns - HY credit spread via OU correlated with VIX - Proxy ETF close prices (TQQQ, QQQM, SPYM, SGOV) The rolling indicators exactly mirror SnapshotStore._fetch_spy_macro._merge_series() so that allocator Gates (macro regime, VIX scaler, parking momentum) behave correctly. """ from __future__ import annotations import datetime as dt import math from dataclasses import dataclass from typing import Any import numpy as np @dataclass class VIXConfig: """Parameters for the VIX Ornstein-Uhlenbeck simulation.""" base_level: float = 18.0 """Long-run mean (θ). Bull ~14, normal ~18, bear ~25, crash ~35+.""" mean_reversion: float = 5.0 """Speed of reversion (κ, annualized). Higher = faster mean-reversion.""" volatility: float = 5.0 """VIX process volatility (σ, per year).""" market_corr: float = -0.7 """Correlation between VIX changes and market log-returns (negative).""" @dataclass class HYSpreadConfig: """Parameters for the HY credit spread OU simulation.""" base_level: float = 4.0 """Long-run mean spread in percentage points.""" mean_reversion: float = 3.0 """Speed of reversion (annualized).""" volatility: float = 1.5 """Spread process volatility (per year).""" vix_corr: float = 0.6 """Correlation with VIX level changes.""" # --------------------------------------------------------------------------- # Internal: rolling indicator computation (mirrors _merge_series in snapshot_store.py) # --------------------------------------------------------------------------- _SMA_PERIODS = (10, 20, 30, 40, 50) _ROLLING_HIGH_PERIODS = (20, 50, 100) _MOMENTUM_PERIODS = (5, 10, 20, 50) _VOL_PERIODS = (15, 20, 30, 50) _EFFICIENCY_PERIODS = (10, 20) _DOWNSIDE_VOL_PERIODS = (10, 20) _ULCER_PERIODS = (10, 20) _ENTROPY_PERIODS = (10, 20) _DRAWDOWN_ACCEL_DAYS = 5 def _compute_rolling_indicators( prefix: str, bars: dict[dt.date, dict[str, Any]], result: dict[dt.date, dict[str, Any]], ) -> None: """Compute all rolling macro indicators for a price series prefix (spy, qqq, etc.). Fills result[date][f"{prefix}_*"] in-place, mirroring exactly the keys produced by SnapshotStore._fetch_spy_macro._merge_series(). """ sorted_dates = sorted(bars.keys()) closes: list[tuple[dt.date, float]] = [ (d, float(bars[d]["close"])) for d in sorted_dates ] for i, (date, close) in enumerate(closes): result.setdefault(date, {}) bar = bars[date] result[date][f"{prefix}_close"] = close result[date][f"{prefix}_open"] = float(bar.get("open", close)) result[date][f"{prefix}_high"] = float(bar.get("high", close)) result[date][f"{prefix}_low"] = float(bar.get("low", close)) result[date][f"{prefix}_volume"] = float(bar.get("volume", 0)) # ---- SMAs ---- for period in _SMA_PERIODS: sma = None if i >= period - 1: window = [c for _, c in closes[i - period + 1 : i + 1]] sma = sum(window) / len(window) result[date][f"{prefix}_sma_{period}"] = sma # ---- Rolling highs (for drawdown gates) ---- for rh_p in _ROLLING_HIGH_PERIODS: rh = None if i >= rh_p - 1: rh = max(c for _, c in closes[i - rh_p + 1 : i + 1]) result[date][f"{prefix}_high_{rh_p}"] = rh # ---- Momentum / N-day return ---- for mom_p in _MOMENTUM_PERIODS: mom = None if i >= mom_p: prev = closes[i - mom_p][1] if prev > 0: mom = (close - prev) / prev result[date][f"{prefix}_mom_{mom_p}"] = mom # ---- Realized volatility (annualised std of log returns) ---- for vol_p in _VOL_PERIODS: vol = None if i >= vol_p: log_rets = [ math.log(closes[j][1] / closes[j - 1][1]) for j in range(i - vol_p + 1, i + 1) if closes[j - 1][1] > 0 ] if len(log_rets) >= vol_p - 1: mean_r = sum(log_rets) / len(log_rets) var_r = sum((r - mean_r) ** 2 for r in log_rets) / len(log_rets) vol = math.sqrt(var_r * 252) result[date][f"{prefix}_vol_{vol_p}"] = vol # ---- Efficiency ratio (Kaufman) ---- for eff_p in _EFFICIENCY_PERIODS: efficiency = None if i >= eff_p: net = abs(close - closes[i - eff_p][1]) gross = sum( abs(closes[j][1] - closes[j - 1][1]) for j in range(i - eff_p + 1, i + 1) ) efficiency = net / gross if gross > 0 else 0.0 result[date][f"{prefix}_efficiency_{eff_p}"] = efficiency # ---- Downside semi-volatility ---- for dv_p in _DOWNSIDE_VOL_PERIODS: downside_vol = None if i >= dv_p: neg_sq = [ min(closes[j][1] / closes[j - 1][1] - 1, 0.0) ** 2 for j in range(i - dv_p + 1, i + 1) if closes[j - 1][1] > 0 ] if len(neg_sq) >= dv_p - 1: downside_vol = math.sqrt(sum(neg_sq) / len(neg_sq) * 252) result[date][f"{prefix}_downside_vol_{dv_p}"] = downside_vol # ---- Shannon entropy (market predictability) ---- for ent_p in _ENTROPY_PERIODS: entropy = None if i >= ent_p: daily_rets = [ closes[j][1] / closes[j - 1][1] - 1 for j in range(i - ent_p + 1, i + 1) if closes[j - 1][1] > 0 ] if len(daily_rets) >= ent_p - 1: n_pos = sum(1 for r in daily_rets if r > 0.001) n_neg = sum(1 for r in daily_rets if r < -0.001) n_flat = len(daily_rets) - n_pos - n_neg entropy = 0.0 for cnt in (n_pos, n_neg, n_flat): if cnt > 0: p = cnt / len(daily_rets) entropy -= p * math.log2(p) result[date][f"{prefix}_entropy_{ent_p}"] = entropy # ---- Ulcer index + current drawdown ---- for ulcer_p in _ULCER_PERIODS: ulcer = None current_dd = None if i >= ulcer_p - 1: window_closes = [c for _, c in closes[i - ulcer_p + 1 : i + 1]] peak = 0.0 drawdowns: list[float] = [] for wc in window_closes: peak = max(peak, wc) if peak > 0: drawdowns.append(wc / peak - 1.0) if drawdowns: ulcer = math.sqrt(sum(dd * dd for dd in drawdowns) / len(drawdowns)) current_dd = abs(drawdowns[-1]) result[date][f"{prefix}_ulcer_{ulcer_p}"] = ulcer result[date][f"{prefix}_drawdown_{ulcer_p}"] = current_dd # ---- Drawdown acceleration ---- dd_lb = 20 dd_accel = None if i >= dd_lb - 1 + _DRAWDOWN_ACCEL_DAYS: cur_window = [c for _, c in closes[i - dd_lb + 1 : i + 1]] prev_i = i - _DRAWDOWN_ACCEL_DAYS prev_window = [c for _, c in closes[prev_i - dd_lb + 1 : prev_i + 1]] cur_peak = max(cur_window) if cur_window else 0.0 prev_peak = max(prev_window) if prev_window else 0.0 if cur_peak > 0 and prev_peak > 0: cur_dd = (cur_peak - close) / cur_peak prev_dd = (prev_peak - closes[prev_i][1]) / prev_peak dd_accel = cur_dd - prev_dd result[date][f"{prefix}_drawdown_accel_{_DRAWDOWN_ACCEL_DAYS}"] = dd_accel # ---- Hurst exponent (R/S analysis) ---- hurst_lookback = 60 hurst = None if i >= hurst_lookback + 1: h_rets = [ (closes[j + 1][1] - closes[j][1]) / closes[j][1] for j in range(i - hurst_lookback, i) if closes[j][1] > 0 ] if len(h_rets) >= 30: def _rs(series: list[float]) -> float: n_ = len(series) m_ = sum(series) / n_ devs = [x - m_ for x in series] cum, s_ = [], 0.0 for d_ in devs: s_ += d_ cum.append(s_) r_ = max(cum) - min(cum) std_ = (sum(d_ ** 2 for d_ in devs) / n_) ** 0.5 return r_ / std_ if std_ > 0 else 0.0 win_sizes = [s for s in [8, 12, 16, 24, 32] if s <= len(h_rets) // 2] if len(win_sizes) >= 2: log_n, log_rs = [], [] for w in win_sizes: chunks = [h_rets[st: st + w] for st in range(0, len(h_rets) - w + 1, w) if len(h_rets[st: st + w]) == w] rs_vals = [_rs(c) for c in chunks] if rs_vals: avg_rs = sum(rs_vals) / len(rs_vals) if avg_rs > 0: log_n.append(math.log(w)) log_rs.append(math.log(avg_rs)) if len(log_n) >= 2: n_h = len(log_n) xm = sum(log_n) / n_h ym = sum(log_rs) / n_h num = sum((log_n[k] - xm) * (log_rs[k] - ym) for k in range(n_h)) den = sum((log_n[k] - xm) ** 2 for k in range(n_h)) hurst = num / den if den > 0 else 0.5 result[date][f"{prefix}_hurst_60"] = hurst # ---- Lag-1 autocorrelation (Lo, 2004) ---- ac_lb = 20 autocorr = None if i >= ac_lb + 1: ac_rets = [ closes[j][1] / closes[j - 1][1] - 1 for j in range(i - ac_lb, i + 1) if closes[j - 1][1] > 0 ] if len(ac_rets) >= ac_lb: x_ac, y_ac = ac_rets[:-1], ac_rets[1:] n_ac = len(x_ac) mx, my = sum(x_ac) / n_ac, sum(y_ac) / n_ac cov_xy = sum((x_ac[k] - mx) * (y_ac[k] - my) for k in range(n_ac)) / n_ac sx = (sum((x_ac[k] - mx) ** 2 for k in range(n_ac)) / n_ac) ** 0.5 sy = (sum((y_ac[k] - my) ** 2 for k in range(n_ac)) / n_ac) ** 0.5 if sx > 1e-12 and sy > 1e-12: autocorr = cov_xy / (sx * sy) result[date][f"{prefix}_autocorr_20"] = autocorr def _compute_pair_correlation( left_prefix: str, right_prefix: str, output_key: str, result: dict[dt.date, dict[str, Any]], ) -> None: """Compute rolling 20-day cross-asset correlation (mirrors snapshot_store logic).""" corr_lb = 20 sorted_dates = sorted(result.keys()) for idx_c, d_c in enumerate(sorted_dates): corr_val = None if idx_c >= corr_lb: left_r, right_r = [], [] for jj in range(idx_c - corr_lb + 1, idx_c + 1): d_j = sorted_dates[jj] d_prev = sorted_dates[jj - 1] lc = result.get(d_j, {}).get(f"{left_prefix}_close") lp = result.get(d_prev, {}).get(f"{left_prefix}_close") rc = result.get(d_j, {}).get(f"{right_prefix}_close") rp = result.get(d_prev, {}).get(f"{right_prefix}_close") if all(v and v > 0 for v in [lc, lp, rc, rp]): left_r.append(lc / lp - 1) right_r.append(rc / rp - 1) if len(left_r) >= corr_lb - 2: n_cr = len(left_r) ml = sum(left_r) / n_cr mr = sum(right_r) / n_cr cov = sum((left_r[k] - ml) * (right_r[k] - mr) for k in range(n_cr)) / n_cr sl = (sum((left_r[k] - ml) ** 2 for k in range(n_cr)) / n_cr) ** 0.5 sr = (sum((right_r[k] - mr) ** 2 for k in range(n_cr)) / n_cr) ** 0.5 if sl > 1e-12 and sr > 1e-12: corr_val = cov / (sl * sr) result[d_c][output_key] = corr_val # --------------------------------------------------------------------------- # VIX and HY spread simulation # --------------------------------------------------------------------------- def _generate_vix_series( market_log_rets: list[float], trading_dates: list[dt.date], config: VIXConfig, rng: np.random.Generator, ) -> dict[dt.date, float]: """Generate VIX time series via OU process correlated with market returns. dVIX = κ(θ-VIX)dt + σ·dW where dW is partially driven by market return sign. """ dt_step = 1.0 / 252 vix = config.base_level vix_series: dict[dt.date, float] = {} for date, mkt_lr in zip(trading_dates, market_log_rets): # Approximate market z-score from log return market_daily_std = config.base_level / 100.0 * math.sqrt(dt_step) + 1e-8 mkt_z = mkt_lr / market_daily_std z_idio = rng.standard_normal() rho = config.market_corr # Negative market → positive VIX shock z_combined = rho * (-mkt_z) + math.sqrt(max(0.0, 1 - rho ** 2)) * z_idio kappa = config.mean_reversion theta = config.base_level sigma = config.volatility dVIX = kappa * (theta - vix) * dt_step + sigma * math.sqrt(dt_step) * z_combined vix = max(5.0, min(90.0, vix + dVIX)) vix_series[date] = round(vix, 2) return vix_series def _generate_hy_series( vix_series: dict[dt.date, float], trading_dates: list[dt.date], config: HYSpreadConfig, rng: np.random.Generator, ) -> dict[dt.date, float]: """Generate HY credit spread via OU process correlated with VIX changes.""" dt_step = 1.0 / 252 spread = config.base_level hy_series: dict[dt.date, float] = {} prev_vix = config.base_level for date in trading_dates: cur_vix = vix_series.get(date, config.base_level) vix_chg = (cur_vix - prev_vix) / (prev_vix + 1e-8) z_idio = rng.standard_normal() rho = config.vix_corr z_combined = rho * vix_chg * 5.0 + math.sqrt(max(0.0, 1 - rho ** 2)) * z_idio kappa = config.mean_reversion theta = config.base_level sigma = config.volatility dSpread = kappa * (theta - spread) * dt_step + sigma * math.sqrt(dt_step) * z_combined spread = max(1.5, min(30.0, spread + dSpread)) hy_series[date] = round(spread, 3) prev_vix = cur_vix return hy_series # --------------------------------------------------------------------------- # Main entry point # --------------------------------------------------------------------------- def generate_macro_data( spy_bars: dict[dt.date, dict[str, Any]], qqq_bars: dict[dt.date, dict[str, Any]], vix_config: VIXConfig, hy_config: HYSpreadConfig, trading_dates: list[dt.date], rng: np.random.Generator, market_log_rets: list[float] | None = None, ) -> dict[dt.date, dict[str, Any]]: """Generate complete macro_by_date dict for SnapshotStore. Includes: - SPY/QQQ rolling indicators (matches _merge_series keys exactly) - spy_qqq_corr_20 cross-correlation - VIXCLS, macro_vix, macro_hy_spread - Parking ETF proxies: tqqq_close, qqqm_close, spym_close, sgov_close Args: spy_bars: SPY OHLCV bars {date: {open, high, low, close, volume}}. qqq_bars: QQQ OHLCV bars. vix_config: VIX OU simulation parameters. hy_config: HY spread OU simulation parameters. trading_dates: Ordered list of NYSE trading dates. rng: NumPy random generator. market_log_rets: Optional market log-return series for VIX correlation. Returns: macro_by_date dict compatible with SnapshotStore. """ result: dict[dt.date, dict[str, Any]] = {} # Rolling indicators for SPY and QQQ _compute_rolling_indicators("spy", spy_bars, result) _compute_rolling_indicators("qqq", qqq_bars, result) # SPY-QQQ cross correlation (regime shift detection) _compute_pair_correlation("spy", "qqq", "spy_qqq_corr_20", result) # Generate VIX and HY spread if market_log_rets is None: market_log_rets = [0.0] * len(trading_dates) vix_series = _generate_vix_series(market_log_rets, trading_dates, vix_config, rng) hy_series = _generate_hy_series(vix_series, trading_dates, hy_config, rng) # Inject VIX, HY, and parking ETF proxies sgov_price = 100.0 sgov_daily_yield = 0.0525 / 252 # ~5.25% money market yield for i, date in enumerate(trading_dates): result.setdefault(date, {}) vix = vix_series.get(date) if vix is not None: result[date]["VIXCLS"] = vix result[date]["macro_vix"] = vix hy = hy_series.get(date) if hy is not None: result[date]["macro_hy_spread"] = hy qqq_close = result[date].get("qqq_close") spy_close = result[date].get("spy_close") if qqq_close is not None and qqq_close > 0: # TQQQ ≈ 3x leveraged QQQ (simplified proxy) result[date]["tqqq_close"] = round(qqq_close * 3.0 / 415.0 * 55.0, 4) # QQQM ≈ QQQ (slightly lower price, same ETF) result[date]["qqqm_close"] = round(qqq_close * 0.99, 4) if spy_close is not None and spy_close > 0: # SPYM ≈ SPY (2x leveraged; simplified as 1.9× SPY level / 480 × 95) result[date]["spym_close"] = round(spy_close * 0.98, 4) # SGOV ≈ T-bill ETF with daily accrual sgov_price *= (1 + sgov_daily_yield) result[date]["sgov_close"] = round(sgov_price, 4) return result