""" V29 Attention Cache Backfill Calls Oracle admin endpoints to collect Wikipedia pageview data for the midlarge universe over 200 trading days, then fetches and caches attention features so the V29 backtest has real wiki_spike values instead of soft-miss Nones. Usage: python scripts/v29/attention_backfill.py [--days 200] [--limit-tickers N] Steps per (ticker, date) pair: 1. resolve_entity — ensure ticker→Wikipedia entity mapping (once per ticker) 2. collect_wiki — trigger Oracle to fetch Wikipedia pageview data 3. get_event_attention — fetch results and write to cache Cache format: data/cache/orb_attention/.json.gz Skip logic: skip pairs where cache already has non-None wiki_spike_10d. """ from __future__ import annotations import asyncio import gzip import json import os import sys from datetime import date, timedelta from pathlib import Path import yaml BASE_DIR = Path(__file__).resolve().parents[2] sys.path.insert(0, str(BASE_DIR)) from libs.common.config import Settings from libs.intraday.catalyst import AttentionEventCache, _extract_attention_features from libs.oracle_client.attention import AttentionService from libs.oracle_client.client import OracleClient from libs.common.time_utils import is_trading_day, trading_days_between UNIVERSE_YAML = BASE_DIR / "configs" / "symbols_midlarge_snapshot_exact.yaml" CACHE_DIR = BASE_DIR / "data" / "cache" / "orb_attention" V23_RUN = BASE_DIR / "runs" / "intraday_orb" / "intraday_20260419_195924_f325f91a.json" CONCURRENCY_ENTITY = 20 CONCURRENCY_COLLECT = 40 CONCURRENCY_FETCH = 40 def load_universe() -> list[str]: with open(UNIVERSE_YAML) as f: data = yaml.safe_load(f) if isinstance(data, dict): tickers = sorted(data.get("symbols", data.get("tickers", list(data.keys())))) else: tickers = sorted(data) # Sort by V23 trade frequency — high-ORB-activity tickers first (most likely to have wiki spikes) try: with open(V23_RUN) as f: v23 = json.load(f) from collections import Counter freq = Counter(t["ticker"] for t in v23.get("trades", [])) tickers.sort(key=lambda t: -freq.get(t, 0)) except Exception: pass return tickers def get_trading_days(lookback: int) -> list[str]: today = date.today() start = today - timedelta(days=lookback * 2) days = [d.isoformat() for d in trading_days_between(start, today)] return days[-lookback:] def has_real_wiki(cache: AttentionEventCache, ticker: str, event_date: str) -> bool: entry = cache.get(ticker, event_date) if entry is None: return False return entry.get("attention_wiki_spike_10d") is not None async def resolve_entities_bulk( tickers: list[str], svc: AttentionService, ) -> dict[str, bool]: semaphore = asyncio.Semaphore(CONCURRENCY_ENTITY) results: dict[str, bool] = {} async def resolve_one(ticker: str) -> None: async with semaphore: try: await svc.resolve_entity(ticker) results[ticker] = True except Exception: results[ticker] = False print(f" Resolving {len(tickers)} entities...") await asyncio.gather(*(resolve_one(t) for t in tickers)) resolved = sum(1 for v in results.values() if v) print(f" Resolved: {resolved}/{len(tickers)}") return results async def collect_and_fetch_bulk( pairs: list[tuple[str, str]], svc: AttentionService, cache: AttentionEventCache, ) -> tuple[int, int]: semaphore = asyncio.Semaphore(CONCURRENCY_COLLECT) filled = 0 errors = 0 completed = 0 total = len(pairs) last_pct = [-1] def _progress() -> None: pct = int(completed / total * 10) * 10 if total > 0 else 0 if pct > last_pct[0] or completed == total: last_pct[0] = pct bar = "█" * (pct // 5) + "░" * (20 - pct // 5) print(f"\r [{bar}] {completed}/{total} ({pct}%) filled={filled} errors={errors}", end="", flush=True) async def collect_one(ticker: str, event_date: str) -> None: nonlocal filled, errors, completed async with semaphore: try: payload = await svc.get_event_attention(ticker, event_date) features = _extract_attention_features(payload) await asyncio.to_thread(cache.put, ticker, event_date, features) if features.get("attention_wiki_spike_10d") is not None: filled += 1 except Exception as exc: errors += 1 _ = exc completed += 1 _progress() _progress() await asyncio.gather(*(collect_one(t, d) for t, d in pairs)) print() return filled, errors async def main(days: int = 200, limit_tickers: int | None = None) -> None: settings = Settings() cache = AttentionEventCache(str(CACHE_DIR)) trading_days = get_trading_days(days) universe = load_universe() if limit_tickers: universe = universe[:limit_tickers] print(f"V29 Attention Backfill") print(f" Universe: {len(universe)} tickers") print(f" Trading days: {trading_days[0]} → {trading_days[-1]} ({len(trading_days)} days)") print(f" Cache dir: {CACHE_DIR}") # Count what's already filled total_pairs = len(universe) * len(trading_days) already_filled = sum( 1 for t in universe for d in trading_days if has_real_wiki(cache, t, d) ) print(f"\n Pre-check: {already_filled}/{total_pairs} pairs already have real wiki data") # Pairs that need backfill missing_pairs = [ (t, d) for t in universe for d in trading_days if not has_real_wiki(cache, t, d) ] print(f" To fill: {len(missing_pairs)} pairs") if not missing_pairs: print("\n Nothing to do — cache already complete.") return async with OracleClient( base_url=settings.stock_oracle_url, timeout=max(float(settings.stock_oracle_timeout), 120.0), ) as client: svc = AttentionService(client) print(f"\n[1/2] Resolving entities for {len(universe)} tickers...") await resolve_entities_bulk(universe, svc) print(f"\n[2/2] Fetching attention for {len(missing_pairs)} pairs...") filled, errors = await collect_and_fetch_bulk(missing_pairs, svc, cache) print(f"\n Done. Filled={filled} Errors={errors}") final_filled = sum( 1 for t in universe for d in trading_days if has_real_wiki(cache, t, d) ) print(f" Final coverage: {final_filled}/{total_pairs} = {final_filled/total_pairs*100:.1f}%") if __name__ == "__main__": import argparse parser = argparse.ArgumentParser(description="V29 wiki attention backfill") parser.add_argument("--days", type=int, default=200, help="Trading days to backfill") parser.add_argument("--limit-tickers", type=int, default=None, help="Test mode: limit to N tickers") args = parser.parse_args() asyncio.run(main(days=args.days, limit_tickers=args.limit_tickers))