fithia2/scripts/v29/attention_backfill.py

"""
V29 Attention Cache Backfill

Calls Oracle admin endpoints to collect Wikipedia pageview data for the midlarge
universe over 200 trading days, then fetches and caches attention features so the
V29 backtest has real wiki_spike values instead of soft-miss Nones.

Usage:
    python scripts/v29/attention_backfill.py [--days 200] [--limit-tickers N]

Steps per (ticker, date) pair:
    1. resolve_entity — ensure ticker→Wikipedia entity mapping (once per ticker)
    2. collect_wiki   — trigger Oracle to fetch Wikipedia pageview data
    3. get_event_attention — fetch results and write to cache

Cache format: data/cache/orb_attention/<TICKER>.json.gz
Skip logic: skip pairs where cache already has non-None wiki_spike_10d.
"""

from __future__ import annotations

import asyncio
import gzip
import json
import os
import sys
from datetime import date, timedelta
from pathlib import Path

import yaml

BASE_DIR = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(BASE_DIR))

from libs.common.config import Settings
from libs.intraday.catalyst import AttentionEventCache, _extract_attention_features
from libs.oracle_client.attention import AttentionService
from libs.oracle_client.client import OracleClient
from libs.common.time_utils import is_trading_day, trading_days_between


UNIVERSE_YAML = BASE_DIR / "configs" / "symbols_midlarge_snapshot_exact.yaml"
CACHE_DIR = BASE_DIR / "data" / "cache" / "orb_attention"
V23_RUN = BASE_DIR / "runs" / "intraday_orb" / "intraday_20260419_195924_f325f91a.json"
CONCURRENCY_ENTITY = 20
CONCURRENCY_COLLECT = 40
CONCURRENCY_FETCH = 40


def load_universe() -> list[str]:
    with open(UNIVERSE_YAML) as f:
        data = yaml.safe_load(f)
    if isinstance(data, dict):
        tickers = sorted(data.get("symbols", data.get("tickers", list(data.keys()))))
    else:
        tickers = sorted(data)

    # Sort by V23 trade frequency — high-ORB-activity tickers first (most likely to have wiki spikes)
    try:
        with open(V23_RUN) as f:
            v23 = json.load(f)
        from collections import Counter
        freq = Counter(t["ticker"] for t in v23.get("trades", []))
        tickers.sort(key=lambda t: -freq.get(t, 0))
    except Exception:
        pass

    return tickers


def get_trading_days(lookback: int) -> list[str]:
    today = date.today()
    start = today - timedelta(days=lookback * 2)
    days = [d.isoformat() for d in trading_days_between(start, today)]
    return days[-lookback:]


def has_real_wiki(cache: AttentionEventCache, ticker: str, event_date: str) -> bool:
    entry = cache.get(ticker, event_date)
    if entry is None:
        return False
    return entry.get("attention_wiki_spike_10d") is not None


async def resolve_entities_bulk(
    tickers: list[str],
    svc: AttentionService,
) -> dict[str, bool]:
    semaphore = asyncio.Semaphore(CONCURRENCY_ENTITY)
    results: dict[str, bool] = {}

    async def resolve_one(ticker: str) -> None:
        async with semaphore:
            try:
                await svc.resolve_entity(ticker)
                results[ticker] = True
            except Exception:
                results[ticker] = False

    print(f"  Resolving {len(tickers)} entities...")
    await asyncio.gather(*(resolve_one(t) for t in tickers))
    resolved = sum(1 for v in results.values() if v)
    print(f"  Resolved: {resolved}/{len(tickers)}")
    return results


async def collect_and_fetch_bulk(
    pairs: list[tuple[str, str]],
    svc: AttentionService,
    cache: AttentionEventCache,
) -> tuple[int, int]:
    semaphore = asyncio.Semaphore(CONCURRENCY_COLLECT)
    filled = 0
    errors = 0
    completed = 0
    total = len(pairs)
    last_pct = [-1]

    def _progress() -> None:
        pct = int(completed / total * 10) * 10 if total > 0 else 0
        if pct > last_pct[0] or completed == total:
            last_pct[0] = pct
            bar = "█" * (pct // 5) + "░" * (20 - pct // 5)
            print(f"\r  [{bar}] {completed}/{total} ({pct}%)  filled={filled} errors={errors}", end="", flush=True)

    async def collect_one(ticker: str, event_date: str) -> None:
        nonlocal filled, errors, completed
        async with semaphore:
            try:
                payload = await svc.get_event_attention(ticker, event_date)
                features = _extract_attention_features(payload)
                await asyncio.to_thread(cache.put, ticker, event_date, features)
                if features.get("attention_wiki_spike_10d") is not None:
                    filled += 1
            except Exception as exc:
                errors += 1
                _ = exc
        completed += 1
        _progress()

    _progress()
    await asyncio.gather(*(collect_one(t, d) for t, d in pairs))
    print()
    return filled, errors


async def main(days: int = 200, limit_tickers: int | None = None) -> None:
    settings = Settings()
    cache = AttentionEventCache(str(CACHE_DIR))
    trading_days = get_trading_days(days)
    universe = load_universe()
    if limit_tickers:
        universe = universe[:limit_tickers]

    print(f"V29 Attention Backfill")
    print(f"  Universe:     {len(universe)} tickers")
    print(f"  Trading days: {trading_days[0]} → {trading_days[-1]} ({len(trading_days)} days)")
    print(f"  Cache dir:    {CACHE_DIR}")

    # Count what's already filled
    total_pairs = len(universe) * len(trading_days)
    already_filled = sum(
        1 for t in universe for d in trading_days if has_real_wiki(cache, t, d)
    )
    print(f"\n  Pre-check: {already_filled}/{total_pairs} pairs already have real wiki data")

    # Pairs that need backfill
    missing_pairs = [
        (t, d) for t in universe for d in trading_days if not has_real_wiki(cache, t, d)
    ]
    print(f"  To fill: {len(missing_pairs)} pairs")

    if not missing_pairs:
        print("\n  Nothing to do — cache already complete.")
        return

    async with OracleClient(
        base_url=settings.stock_oracle_url,
        timeout=max(float(settings.stock_oracle_timeout), 120.0),
    ) as client:
        svc = AttentionService(client)

        print(f"\n[1/2] Resolving entities for {len(universe)} tickers...")
        await resolve_entities_bulk(universe, svc)

        print(f"\n[2/2] Fetching attention for {len(missing_pairs)} pairs...")
        filled, errors = await collect_and_fetch_bulk(missing_pairs, svc, cache)

    print(f"\n  Done. Filled={filled}  Errors={errors}")
    final_filled = sum(
        1 for t in universe for d in trading_days if has_real_wiki(cache, t, d)
    )
    print(f"  Final coverage: {final_filled}/{total_pairs} = {final_filled/total_pairs*100:.1f}%")


if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser(description="V29 wiki attention backfill")
    parser.add_argument("--days", type=int, default=200, help="Trading days to backfill")
    parser.add_argument("--limit-tickers", type=int, default=None, help="Test mode: limit to N tickers")
    args = parser.parse_args()
    asyncio.run(main(days=args.days, limit_tickers=args.limit_tickers))