Add three-tier VIX/FRED fallback to paper trader (strict-conservative)

On 2026-05-08 19:45 UTC a transient Oracle FRED-proxy 5xx storm caused
paper_engine_vix_fred_unavailable to fire (3 sequential 500s; the 4th
attempt returned 200 OK with VIX=17.08). The pre-fix engine just left
VIXCLS missing from the macro dict, which the selector at
libs/backtest/selector.py:1171-1173 already treats strict-conservatively
(None → veto). So the 5/8 incident vetoed v7.356 PEAD candidates for
~1 minute with no money-loss exposure. But:

  - Log severity was thin (info-level "unavailable", no escalation).
  - No tolerance for short outages — every 500 cost the gate's signal.
  - EventDetector PostgreSQL rows do not pre-populate macro_vix per
    engine.py:2810-2812 comment, so live trading depends entirely on
    the FRED fetch path.

Fix: in-memory session-scoped cache + 3-tier fallback in
PaperTradingEngine._fetch_macro:

  Tier 1  fetch ok          → cache (value, now_utc), log ..._ok (info)
  Tier 2  fail, cache <24h  → return cached value, log ..._stale_fallback
                              (warning) with staleness_sec
  Tier 3  fail, cache stale → None, log ..._unavailable_blocking (error)
                              with reason={no_cache,cache_too_stale}

The None-veto path through the selector is preserved exactly, so no
silent-pass on unknown VIX. Empty/0 observations now treated as outage
to defend against an upstream regression flipping "missing→veto" into
"0→always-pass".

The thin libs/oracle_client/fred.py is intentionally untouched — fallback
policy belongs in the engine, not the generic client.

6 new tests in tests/unit/paper_trader/test_vix_fred_fallback.py
(success/cache-write, 500+stale-<24h, 500+stale->24h-blocks,
no-cache+500-blocks, empty-observations-blocks, recovery-refresh).
All 18 tests in -k "vix or fred" pass; full paper_trader suite 45/45.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

apps/paper_trader/engine.py

@@ -50,6 +50,13 @@ logger = get_logger(__name__)
 # Kill-switch threshold (matches backtest)
 _KILL_SWITCH_DRAWDOWN_PCT = 25.0
 
+# Max age (seconds) a cached FRED observation is allowed to substitute for a
+# fresh fetch when Oracle returns 5xx. 24h covers a single trading-day outage
+# without letting genuinely stale data silently feed live gating logic. Beyond
+# this threshold the helper returns None, which the selector treats as a hard
+# veto on engines with macro_vix_max set (strict-conservative).
+_FRED_FALLBACK_MAX_AGE_SEC = 24 * 3600
+
 
 @dataclass
 class ReconciliationReport:
@@ -141,6 +148,12 @@ class PaperTradingEngine:
         # Overlay shock brake cooldown (in-memory, session-scoped)
         self._parking_brake_cooldown_remaining: int = 0
         self._parking_brake_skip_buy_today: bool = False  # skip same-day re-buy after brake fires
+        # FRED macro cache for transient-outage tolerance.
+        # series_id -> (value, fetched_at_utc). In-memory only; cleared on daemon
+        # restart. Used by _fetch_fred_series_with_fallback() to bridge Oracle 5xx
+        # blips so the strict VIX gate doesn't unnecessarily veto on a calm day.
+        # Staleness > _FRED_FALLBACK_MAX_AGE_SEC -> treated as unavailable (return None).
+        self._macro_fred_cache: dict[str, tuple[float, dt.datetime]] = {}
 
         logger.warning(
             "paper_trader_scoring_dispatch",
@@ -3497,24 +3510,30 @@ class PaperTradingEngine:
                         if len(closes) >= sma_period:
                             _macro[f"{key_prefix}_sma_{sma_period}"] = sum(closes[-sma_period:]) / sma_period
 
-                # Fetch FRED macro data (VIX, HY spread) for regime sizing
+                # Fetch FRED macro data (VIX, HY spread) for regime sizing.
+                # Each series is fetched via the cache-aware helper which:
+                #   - returns the fresh value on success
+                #   - falls back to last-known value if cached <24h (logged warning)
+                #   - returns None if cached >=24h or never seen (selector vetoes)
                 try:
                     from libs.oracle_client import FredService, OracleClient as _OC
                     async with _OC(base_url=self._detector._oracle_url, timeout=5.0) as fred_client:
                         fred_svc = FredService(fred_client)
                         for series_id in ("VIXCLS", "BAMLH0A0HYM2"):
-                            try:
+                            value = await self._fetch_fred_series_with_fallback(
-                                resp = await fred_svc.get_observations(series_id, start=start.isoformat(), end=date.isoformat())
+                                fred_svc, series_id, start, date,
-                                if resp.observations:
+                            )
-                                    latest = [o for o in resp.observations if o.value is not None]
+                            if value is not None:
-                                    if latest:
+                                _macro[series_id] = value
-                                        _macro[series_id] = latest[-1].value
+                except Exception as exc:
-                            except Exception as _vix_exc:
+                    # Connection-level failure (e.g. Oracle process down) — leave
-                                logger.warning("paper_engine_vix_fred_unavailable",
+                    # _macro without VIXCLS/HY so the selector vetoes engines with
-                                               series=series_id, err=str(_vix_exc),
+                    # macro_vix_max set. Cache is not consulted here because we
-                                               category="macro")
+                    # cannot distinguish a connect error from "FredService init
-                except Exception:
+                    # blew up" cleanly; the per-series helper handles transient
-                    pass
+                    # 5xx and is where stale-fallback belongs.
+                    logger.error("paper_engine_fred_client_unavailable",
+                                 err=str(exc), category="macro")
                 return _macro
 
             # Hard cap: don't let Oracle hangs block trading for more than 8s
@@ -3530,6 +3549,82 @@ class PaperTradingEngine:
             logger.warning("paper_engine_macro_fetch_failed", error=str(exc))
             return {}
 
+    async def _fetch_fred_series_with_fallback(
+        self,
+        fred_svc: Any,
+        series_id: str,
+        start: dt.date,
+        end: dt.date,
+    ) -> float | None:
+        """Fetch a FRED series with three-tier strict-conservative fallback.
+
+        Tiers:
+        1. Fresh fetch succeeds -> cache (value, now_utc), emit
+           ``paper_engine_vix_fred_ok`` (info), return value.
+        2. Fresh fetch fails AND cached value younger than
+           ``_FRED_FALLBACK_MAX_AGE_SEC`` -> emit
+           ``paper_engine_vix_fred_stale_fallback`` (warning) with staleness in
+           seconds, return cached value.
+        3. Fresh fetch fails AND cache absent OR >= max-age -> emit
+           ``paper_engine_vix_fred_unavailable_blocking`` (error), return None.
+           Selector treats macro_vix=None as a hard veto for any engine with
+           macro_vix_max set, so live trades are blocked rather than silently
+           passing on unknown VIX.
+
+        Pure data — no side effects on session state or files. Cache lives on
+        the engine instance and is lost on daemon restart (intentional: a fresh
+        process must re-prove FRED is reachable before relying on staleness).
+        """
+        now = dt.datetime.now(dt.timezone.utc)
+        try:
+            resp = await fred_svc.get_observations(
+                series_id, start=start.isoformat(), end=end.isoformat(),
+            )
+            if resp.observations:
+                latest = [o for o in resp.observations if o.value is not None]
+                if latest:
+                    value = float(latest[-1].value)
+                    self._macro_fred_cache[series_id] = (value, now)
+                    logger.info(
+                        "paper_engine_vix_fred_ok",
+                        series=series_id, value=value, category="macro",
+                    )
+                    return value
+            # 200 OK but no usable observations: treat as outage so we don't
+            # silently lose the gate. Falls through to error path.
+            err_msg = "no_observations_in_response"
+        except Exception as exc:
+            err_msg = str(exc)
+
+        cached = self._macro_fred_cache.get(series_id)
+        if cached is not None:
+            cached_value, cached_at = cached
+            staleness_sec = (now - cached_at).total_seconds()
+            if staleness_sec < _FRED_FALLBACK_MAX_AGE_SEC:
+                logger.warning(
+                    "paper_engine_vix_fred_stale_fallback",
+                    series=series_id, value=cached_value,
+                    staleness_sec=int(staleness_sec), err=err_msg,
+                    max_age_sec=_FRED_FALLBACK_MAX_AGE_SEC, category="macro",
+                )
+                return cached_value
+            # Cached but too old to trust — drop into hard-veto path below.
+            logger.error(
+                "paper_engine_vix_fred_unavailable_blocking",
+                series=series_id, err=err_msg,
+                staleness_sec=int(staleness_sec),
+                max_age_sec=_FRED_FALLBACK_MAX_AGE_SEC,
+                reason="cache_too_stale", category="macro",
+            )
+            return None
+
+        logger.error(
+            "paper_engine_vix_fred_unavailable_blocking",
+            series=series_id, err=err_msg,
+            reason="no_cache", category="macro",
+        )
+        return None
+
     # ------------------------------------------------------------------ #
     # Macro Risk-On Sleeve (Phase 3)
     # ------------------------------------------------------------------ #

tests/unit/paper_trader/test_vix_fred_fallback.py

@@ -0,0 +1,188 @@
+"""Tests: VIX/FRED fetch fallback for paper trading engine.
+
+Covers PaperTradingEngine._fetch_fred_series_with_fallback() — the helper
+introduced to bridge transient Oracle 5xx without silently letting the
+macro_vix gate pass on unknown VIX.
+
+Behavior contract:
+1. Fresh fetch OK -> cache value, log paper_engine_vix_fred_ok, return value.
+2. Fresh fetch fails AND cached <24h -> use cached, log
+   paper_engine_vix_fred_stale_fallback, return cached value.
+3. Fresh fetch fails AND no cache OR cache >=24h -> log
+   paper_engine_vix_fred_unavailable_blocking, return None (selector vetoes).
+
+The helper is engine-instance-scoped; we instantiate the engine via
+``object.__new__`` to bypass the heavy ``__init__`` and only set the
+fields the helper actually touches. This keeps the unit truly unit-scope
+and avoids a SessionRow / Alpaca / SnapshotStore stack.
+"""
+from __future__ import annotations
+
+import datetime as dt
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from apps.paper_trader.engine import (
+    PaperTradingEngine,
+    _FRED_FALLBACK_MAX_AGE_SEC,
+)
+
+
+def _bare_engine() -> PaperTradingEngine:
+    """Build a minimally initialized engine for helper-only tests.
+
+    The cache and method under test do not depend on any other instance
+    state; bypassing __init__ keeps these tests fast and decoupled from
+    config/snapshot/manifest plumbing.
+    """
+    eng = object.__new__(PaperTradingEngine)
+    eng._macro_fred_cache = {}
+    return eng
+
+
+def _fred_obs(value: float | None, date: str = "2026-05-08") -> MagicMock:
+    obs = MagicMock()
+    obs.value = value
+    obs.date = date
+    return obs
+
+
+def _fred_resp(observations: list[MagicMock]) -> MagicMock:
+    resp = MagicMock()
+    resp.observations = observations
+    return resp
+
+
+@pytest.mark.asyncio
+async def test_vix_fetch_success_uses_value():
+    """Tier 1: a healthy fetch returns the latest non-null value and caches it."""
+    eng = _bare_engine()
+    fred_svc = MagicMock()
+    fred_svc.get_observations = AsyncMock(
+        return_value=_fred_resp([_fred_obs(15.5), _fred_obs(17.08)]),
+    )
+
+    result = await eng._fetch_fred_series_with_fallback(
+        fred_svc, "VIXCLS", dt.date(2026, 5, 1), dt.date(2026, 5, 8),
+    )
+
+    assert result == 17.08
+    assert "VIXCLS" in eng._macro_fred_cache
+    cached_value, cached_at = eng._macro_fred_cache["VIXCLS"]
+    assert cached_value == 17.08
+    assert cached_at.tzinfo is not None  # tz-aware UTC
+
+
+@pytest.mark.asyncio
+async def test_vix_fetch_500_uses_last_known_within_24h():
+    """Tier 2: with a recent cached value, a 5xx falls back to it (warning)."""
+    eng = _bare_engine()
+    # Pre-seed cache with a 6h-old value
+    six_hours_ago = dt.datetime.now(dt.timezone.utc) - dt.timedelta(hours=6)
+    eng._macro_fred_cache["VIXCLS"] = (16.42, six_hours_ago)
+
+    fred_svc = MagicMock()
+    fred_svc.get_observations = AsyncMock(
+        side_effect=Exception("Server error 500: /api/v1/fred/proxy/series/observations"),
+    )
+
+    result = await eng._fetch_fred_series_with_fallback(
+        fred_svc, "VIXCLS", dt.date(2026, 5, 1), dt.date(2026, 5, 8),
+    )
+
+    # Returns the cached value rather than None — strategy keeps trading.
+    assert result == 16.42
+    # Cache itself is not refreshed by a fallback (only success refreshes).
+    cached_value, cached_at = eng._macro_fred_cache["VIXCLS"]
+    assert cached_value == 16.42
+    assert cached_at == six_hours_ago
+
+
+@pytest.mark.asyncio
+async def test_vix_fetch_stale_more_than_24h_blocks_trades():
+    """Tier 3a: a cached value older than 24h is treated as no cache -> None."""
+    eng = _bare_engine()
+    # Cache age > _FRED_FALLBACK_MAX_AGE_SEC
+    too_old = dt.datetime.now(dt.timezone.utc) - dt.timedelta(
+        seconds=_FRED_FALLBACK_MAX_AGE_SEC + 60,
+    )
+    eng._macro_fred_cache["VIXCLS"] = (12.0, too_old)
+
+    fred_svc = MagicMock()
+    fred_svc.get_observations = AsyncMock(
+        side_effect=Exception("Server error 500"),
+    )
+
+    result = await eng._fetch_fred_series_with_fallback(
+        fred_svc, "VIXCLS", dt.date(2026, 5, 1), dt.date(2026, 5, 8),
+    )
+
+    # None -> selector treats macro_vix as missing -> any engine with
+    # macro_vix_max set vetoes the candidate. STRICT-CONSERVATIVE.
+    assert result is None
+
+
+@pytest.mark.asyncio
+async def test_vix_fetch_no_cache_and_failure_blocks_trades():
+    """Tier 3b: failure with empty cache returns None — never silently allows."""
+    eng = _bare_engine()
+    assert eng._macro_fred_cache == {}
+
+    fred_svc = MagicMock()
+    fred_svc.get_observations = AsyncMock(
+        side_effect=Exception("Server error 500"),
+    )
+
+    result = await eng._fetch_fred_series_with_fallback(
+        fred_svc, "VIXCLS", dt.date(2026, 5, 1), dt.date(2026, 5, 8),
+    )
+
+    assert result is None
+    # Failed fetch must NOT poison the cache with bogus values.
+    assert "VIXCLS" not in eng._macro_fred_cache
+
+
+@pytest.mark.asyncio
+async def test_vix_fetch_returns_empty_observations_does_not_silently_allow_trades():
+    """200 OK with no usable observations is treated as outage, not as VIX=0.
+
+    Without this, an upstream regression returning an empty list could
+    flip the gate from "missing -> veto" to "0.0 -> always pass".
+    """
+    eng = _bare_engine()
+    fred_svc = MagicMock()
+    # Observations exist but all values are None ('.' in raw FRED == missing).
+    fred_svc.get_observations = AsyncMock(
+        return_value=_fred_resp([_fred_obs(None), _fred_obs(None)]),
+    )
+
+    result = await eng._fetch_fred_series_with_fallback(
+        fred_svc, "VIXCLS", dt.date(2026, 5, 1), dt.date(2026, 5, 8),
+    )
+
+    assert result is None
+    assert "VIXCLS" not in eng._macro_fred_cache
+
+
+@pytest.mark.asyncio
+async def test_vix_fetch_success_after_failure_refreshes_cache():
+    """A healthy fetch after a fallback overwrites the stale cache entry."""
+    eng = _bare_engine()
+    older = dt.datetime.now(dt.timezone.utc) - dt.timedelta(hours=12)
+    eng._macro_fred_cache["VIXCLS"] = (16.42, older)
+
+    fred_svc = MagicMock()
+    fred_svc.get_observations = AsyncMock(
+        return_value=_fred_resp([_fred_obs(17.08)]),
+    )
+
+    result = await eng._fetch_fred_series_with_fallback(
+        fred_svc, "VIXCLS", dt.date(2026, 5, 1), dt.date(2026, 5, 8),
+    )
+
+    assert result == 17.08
+    cached_value, cached_at = eng._macro_fred_cache["VIXCLS"]
+    assert cached_value == 17.08
+    # Cache timestamp moved forward.
+    assert cached_at > older