diff --git a/apps/paper_trader/engine.py b/apps/paper_trader/engine.py index d64c8ac..6ed0707 100644 --- a/apps/paper_trader/engine.py +++ b/apps/paper_trader/engine.py @@ -50,6 +50,13 @@ logger = get_logger(__name__) # Kill-switch threshold (matches backtest) _KILL_SWITCH_DRAWDOWN_PCT = 25.0 +# Max age (seconds) a cached FRED observation is allowed to substitute for a +# fresh fetch when Oracle returns 5xx. 24h covers a single trading-day outage +# without letting genuinely stale data silently feed live gating logic. Beyond +# this threshold the helper returns None, which the selector treats as a hard +# veto on engines with macro_vix_max set (strict-conservative). +_FRED_FALLBACK_MAX_AGE_SEC = 24 * 3600 + @dataclass class ReconciliationReport: @@ -141,6 +148,12 @@ class PaperTradingEngine: # Overlay shock brake cooldown (in-memory, session-scoped) self._parking_brake_cooldown_remaining: int = 0 self._parking_brake_skip_buy_today: bool = False # skip same-day re-buy after brake fires + # FRED macro cache for transient-outage tolerance. + # series_id -> (value, fetched_at_utc). In-memory only; cleared on daemon + # restart. Used by _fetch_fred_series_with_fallback() to bridge Oracle 5xx + # blips so the strict VIX gate doesn't unnecessarily veto on a calm day. + # Staleness > _FRED_FALLBACK_MAX_AGE_SEC -> treated as unavailable (return None). + self._macro_fred_cache: dict[str, tuple[float, dt.datetime]] = {} logger.warning( "paper_trader_scoring_dispatch", @@ -3497,24 +3510,30 @@ class PaperTradingEngine: if len(closes) >= sma_period: _macro[f"{key_prefix}_sma_{sma_period}"] = sum(closes[-sma_period:]) / sma_period - # Fetch FRED macro data (VIX, HY spread) for regime sizing + # Fetch FRED macro data (VIX, HY spread) for regime sizing. + # Each series is fetched via the cache-aware helper which: + # - returns the fresh value on success + # - falls back to last-known value if cached <24h (logged warning) + # - returns None if cached >=24h or never seen (selector vetoes) try: from libs.oracle_client import FredService, OracleClient as _OC async with _OC(base_url=self._detector._oracle_url, timeout=5.0) as fred_client: fred_svc = FredService(fred_client) for series_id in ("VIXCLS", "BAMLH0A0HYM2"): - try: - resp = await fred_svc.get_observations(series_id, start=start.isoformat(), end=date.isoformat()) - if resp.observations: - latest = [o for o in resp.observations if o.value is not None] - if latest: - _macro[series_id] = latest[-1].value - except Exception as _vix_exc: - logger.warning("paper_engine_vix_fred_unavailable", - series=series_id, err=str(_vix_exc), - category="macro") - except Exception: - pass + value = await self._fetch_fred_series_with_fallback( + fred_svc, series_id, start, date, + ) + if value is not None: + _macro[series_id] = value + except Exception as exc: + # Connection-level failure (e.g. Oracle process down) — leave + # _macro without VIXCLS/HY so the selector vetoes engines with + # macro_vix_max set. Cache is not consulted here because we + # cannot distinguish a connect error from "FredService init + # blew up" cleanly; the per-series helper handles transient + # 5xx and is where stale-fallback belongs. + logger.error("paper_engine_fred_client_unavailable", + err=str(exc), category="macro") return _macro # Hard cap: don't let Oracle hangs block trading for more than 8s @@ -3530,6 +3549,82 @@ class PaperTradingEngine: logger.warning("paper_engine_macro_fetch_failed", error=str(exc)) return {} + async def _fetch_fred_series_with_fallback( + self, + fred_svc: Any, + series_id: str, + start: dt.date, + end: dt.date, + ) -> float | None: + """Fetch a FRED series with three-tier strict-conservative fallback. + + Tiers: + 1. Fresh fetch succeeds -> cache (value, now_utc), emit + ``paper_engine_vix_fred_ok`` (info), return value. + 2. Fresh fetch fails AND cached value younger than + ``_FRED_FALLBACK_MAX_AGE_SEC`` -> emit + ``paper_engine_vix_fred_stale_fallback`` (warning) with staleness in + seconds, return cached value. + 3. Fresh fetch fails AND cache absent OR >= max-age -> emit + ``paper_engine_vix_fred_unavailable_blocking`` (error), return None. + Selector treats macro_vix=None as a hard veto for any engine with + macro_vix_max set, so live trades are blocked rather than silently + passing on unknown VIX. + + Pure data — no side effects on session state or files. Cache lives on + the engine instance and is lost on daemon restart (intentional: a fresh + process must re-prove FRED is reachable before relying on staleness). + """ + now = dt.datetime.now(dt.timezone.utc) + try: + resp = await fred_svc.get_observations( + series_id, start=start.isoformat(), end=end.isoformat(), + ) + if resp.observations: + latest = [o for o in resp.observations if o.value is not None] + if latest: + value = float(latest[-1].value) + self._macro_fred_cache[series_id] = (value, now) + logger.info( + "paper_engine_vix_fred_ok", + series=series_id, value=value, category="macro", + ) + return value + # 200 OK but no usable observations: treat as outage so we don't + # silently lose the gate. Falls through to error path. + err_msg = "no_observations_in_response" + except Exception as exc: + err_msg = str(exc) + + cached = self._macro_fred_cache.get(series_id) + if cached is not None: + cached_value, cached_at = cached + staleness_sec = (now - cached_at).total_seconds() + if staleness_sec < _FRED_FALLBACK_MAX_AGE_SEC: + logger.warning( + "paper_engine_vix_fred_stale_fallback", + series=series_id, value=cached_value, + staleness_sec=int(staleness_sec), err=err_msg, + max_age_sec=_FRED_FALLBACK_MAX_AGE_SEC, category="macro", + ) + return cached_value + # Cached but too old to trust — drop into hard-veto path below. + logger.error( + "paper_engine_vix_fred_unavailable_blocking", + series=series_id, err=err_msg, + staleness_sec=int(staleness_sec), + max_age_sec=_FRED_FALLBACK_MAX_AGE_SEC, + reason="cache_too_stale", category="macro", + ) + return None + + logger.error( + "paper_engine_vix_fred_unavailable_blocking", + series=series_id, err=err_msg, + reason="no_cache", category="macro", + ) + return None + # ------------------------------------------------------------------ # # Macro Risk-On Sleeve (Phase 3) # ------------------------------------------------------------------ # diff --git a/tests/unit/paper_trader/test_vix_fred_fallback.py b/tests/unit/paper_trader/test_vix_fred_fallback.py new file mode 100644 index 0000000..df6828a --- /dev/null +++ b/tests/unit/paper_trader/test_vix_fred_fallback.py @@ -0,0 +1,188 @@ +"""Tests: VIX/FRED fetch fallback for paper trading engine. + +Covers PaperTradingEngine._fetch_fred_series_with_fallback() — the helper +introduced to bridge transient Oracle 5xx without silently letting the +macro_vix gate pass on unknown VIX. + +Behavior contract: +1. Fresh fetch OK -> cache value, log paper_engine_vix_fred_ok, return value. +2. Fresh fetch fails AND cached <24h -> use cached, log + paper_engine_vix_fred_stale_fallback, return cached value. +3. Fresh fetch fails AND no cache OR cache >=24h -> log + paper_engine_vix_fred_unavailable_blocking, return None (selector vetoes). + +The helper is engine-instance-scoped; we instantiate the engine via +``object.__new__`` to bypass the heavy ``__init__`` and only set the +fields the helper actually touches. This keeps the unit truly unit-scope +and avoids a SessionRow / Alpaca / SnapshotStore stack. +""" +from __future__ import annotations + +import datetime as dt +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from apps.paper_trader.engine import ( + PaperTradingEngine, + _FRED_FALLBACK_MAX_AGE_SEC, +) + + +def _bare_engine() -> PaperTradingEngine: + """Build a minimally initialized engine for helper-only tests. + + The cache and method under test do not depend on any other instance + state; bypassing __init__ keeps these tests fast and decoupled from + config/snapshot/manifest plumbing. + """ + eng = object.__new__(PaperTradingEngine) + eng._macro_fred_cache = {} + return eng + + +def _fred_obs(value: float | None, date: str = "2026-05-08") -> MagicMock: + obs = MagicMock() + obs.value = value + obs.date = date + return obs + + +def _fred_resp(observations: list[MagicMock]) -> MagicMock: + resp = MagicMock() + resp.observations = observations + return resp + + +@pytest.mark.asyncio +async def test_vix_fetch_success_uses_value(): + """Tier 1: a healthy fetch returns the latest non-null value and caches it.""" + eng = _bare_engine() + fred_svc = MagicMock() + fred_svc.get_observations = AsyncMock( + return_value=_fred_resp([_fred_obs(15.5), _fred_obs(17.08)]), + ) + + result = await eng._fetch_fred_series_with_fallback( + fred_svc, "VIXCLS", dt.date(2026, 5, 1), dt.date(2026, 5, 8), + ) + + assert result == 17.08 + assert "VIXCLS" in eng._macro_fred_cache + cached_value, cached_at = eng._macro_fred_cache["VIXCLS"] + assert cached_value == 17.08 + assert cached_at.tzinfo is not None # tz-aware UTC + + +@pytest.mark.asyncio +async def test_vix_fetch_500_uses_last_known_within_24h(): + """Tier 2: with a recent cached value, a 5xx falls back to it (warning).""" + eng = _bare_engine() + # Pre-seed cache with a 6h-old value + six_hours_ago = dt.datetime.now(dt.timezone.utc) - dt.timedelta(hours=6) + eng._macro_fred_cache["VIXCLS"] = (16.42, six_hours_ago) + + fred_svc = MagicMock() + fred_svc.get_observations = AsyncMock( + side_effect=Exception("Server error 500: /api/v1/fred/proxy/series/observations"), + ) + + result = await eng._fetch_fred_series_with_fallback( + fred_svc, "VIXCLS", dt.date(2026, 5, 1), dt.date(2026, 5, 8), + ) + + # Returns the cached value rather than None — strategy keeps trading. + assert result == 16.42 + # Cache itself is not refreshed by a fallback (only success refreshes). + cached_value, cached_at = eng._macro_fred_cache["VIXCLS"] + assert cached_value == 16.42 + assert cached_at == six_hours_ago + + +@pytest.mark.asyncio +async def test_vix_fetch_stale_more_than_24h_blocks_trades(): + """Tier 3a: a cached value older than 24h is treated as no cache -> None.""" + eng = _bare_engine() + # Cache age > _FRED_FALLBACK_MAX_AGE_SEC + too_old = dt.datetime.now(dt.timezone.utc) - dt.timedelta( + seconds=_FRED_FALLBACK_MAX_AGE_SEC + 60, + ) + eng._macro_fred_cache["VIXCLS"] = (12.0, too_old) + + fred_svc = MagicMock() + fred_svc.get_observations = AsyncMock( + side_effect=Exception("Server error 500"), + ) + + result = await eng._fetch_fred_series_with_fallback( + fred_svc, "VIXCLS", dt.date(2026, 5, 1), dt.date(2026, 5, 8), + ) + + # None -> selector treats macro_vix as missing -> any engine with + # macro_vix_max set vetoes the candidate. STRICT-CONSERVATIVE. + assert result is None + + +@pytest.mark.asyncio +async def test_vix_fetch_no_cache_and_failure_blocks_trades(): + """Tier 3b: failure with empty cache returns None — never silently allows.""" + eng = _bare_engine() + assert eng._macro_fred_cache == {} + + fred_svc = MagicMock() + fred_svc.get_observations = AsyncMock( + side_effect=Exception("Server error 500"), + ) + + result = await eng._fetch_fred_series_with_fallback( + fred_svc, "VIXCLS", dt.date(2026, 5, 1), dt.date(2026, 5, 8), + ) + + assert result is None + # Failed fetch must NOT poison the cache with bogus values. + assert "VIXCLS" not in eng._macro_fred_cache + + +@pytest.mark.asyncio +async def test_vix_fetch_returns_empty_observations_does_not_silently_allow_trades(): + """200 OK with no usable observations is treated as outage, not as VIX=0. + + Without this, an upstream regression returning an empty list could + flip the gate from "missing -> veto" to "0.0 -> always pass". + """ + eng = _bare_engine() + fred_svc = MagicMock() + # Observations exist but all values are None ('.' in raw FRED == missing). + fred_svc.get_observations = AsyncMock( + return_value=_fred_resp([_fred_obs(None), _fred_obs(None)]), + ) + + result = await eng._fetch_fred_series_with_fallback( + fred_svc, "VIXCLS", dt.date(2026, 5, 1), dt.date(2026, 5, 8), + ) + + assert result is None + assert "VIXCLS" not in eng._macro_fred_cache + + +@pytest.mark.asyncio +async def test_vix_fetch_success_after_failure_refreshes_cache(): + """A healthy fetch after a fallback overwrites the stale cache entry.""" + eng = _bare_engine() + older = dt.datetime.now(dt.timezone.utc) - dt.timedelta(hours=12) + eng._macro_fred_cache["VIXCLS"] = (16.42, older) + + fred_svc = MagicMock() + fred_svc.get_observations = AsyncMock( + return_value=_fred_resp([_fred_obs(17.08)]), + ) + + result = await eng._fetch_fred_series_with_fallback( + fred_svc, "VIXCLS", dt.date(2026, 5, 1), dt.date(2026, 5, 8), + ) + + assert result == 17.08 + cached_value, cached_at = eng._macro_fred_cache["VIXCLS"] + assert cached_value == 17.08 + # Cache timestamp moved forward. + assert cached_at > older