stock-oracle/app/services/news/stocktwits_universe.py

"""
Dynamic StockTwits subscription universe.

Single V49 universe ("today only") would create a 16-hour data hole during
the premarket window (prev 16:00 ET → today 09:30 ET). Instead we union:

  (last N days of UniverseSnapshot active tickers)
    ∪
  (today's premarket gap movers above threshold)

Capped at STOCKTWITS_UNIVERSE_MAX_SIZE (default 300) to fit comfortably under
the 200-req/hr StockTwits ceiling at one poll per 5 minutes.

Computed once per trading day at 09:00 ET and cached in Redis under
`news_v2:stocktwits:universe`. The poller reads that key each cycle.
"""

from __future__ import annotations

import logging
from datetime import datetime, timedelta, timezone

import orjson
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession

from app.core.config import settings
from app.core.database import AsyncSessionLocal
from app.models.universe_snapshot import UniverseSnapshot
from app.utils.cache import get_redis

logger = logging.getLogger(__name__)

REDIS_KEY = "news_v2:stocktwits:universe"
REDIS_TTL_SEC = 24 * 60 * 60  # 1 day


async def compute_stocktwits_universe(
    db: AsyncSession | None = None,
    lookback_days: int | None = None,
    max_size: int | None = None,
    gap_threshold: float | None = None,
) -> list[str]:
    """Compute and persist the StockTwits subscription universe to Redis."""
    lookback_days = lookback_days or settings.STOCKTWITS_UNIVERSE_LOOKBACK_DAYS
    max_size = max_size or settings.STOCKTWITS_UNIVERSE_MAX_SIZE
    gap_threshold = gap_threshold if gap_threshold is not None else settings.STOCKTWITS_PREMARKET_GAP_THRESHOLD

    own_session = db is None
    if own_session:
        db = AsyncSessionLocal()
        await db.__aenter__()  # type: ignore[attr-defined]
    try:
        recent_universe = await _recent_universe_tickers(db, lookback_days)  # type: ignore[arg-type]
        movers = await _premarket_gap_movers(db, gap_threshold)  # type: ignore[arg-type]
    finally:
        if own_session and db is not None:
            await db.__aexit__(None, None, None)  # type: ignore[attr-defined]

    union: list[str] = []
    seen: set[str] = set()
    # Movers first so they survive if we hit max_size truncation
    for t in movers + recent_universe:
        if t and t not in seen:
            seen.add(t)
            union.append(t)
        if len(union) >= max_size:
            break

    await _persist_to_redis(union)
    logger.info(
        f"StockTwits universe: {len(union)} tickers "
        f"({len(movers)} movers, {len(recent_universe)} recent universe, capped at {max_size})"
    )
    return union


async def get_cached_universe() -> list[str]:
    """Read the current Redis-cached universe; returns [] if absent."""
    redis = await get_redis()
    if redis is None:
        return []
    try:
        raw = await redis.get(REDIS_KEY)
        if raw is None:
            return []
        data = orjson.loads(raw)
        if isinstance(data, list):
            return [str(t).upper() for t in data]
    except Exception as e:
        logger.warning(f"StockTwits universe read failed: {e}")
    return []


# ---------------------------------------------------------------------------
# Internals
# ---------------------------------------------------------------------------

async def _recent_universe_tickers(db: AsyncSession, lookback_days: int) -> list[str]:
    """Tickers from UniverseSnapshot, ranked by market cap.

    Prefers the recent ``lookback_days`` window. If that's empty (the monthly
    snapshot job hasn't refreshed in a while), falls back to the most recent
    snapshot in the table — top market-cap tickers stay roughly stable across
    months, so a stale snapshot is still useful for keeping the StockTwits
    poll alive instead of returning [].
    """
    from sqlalchemy import desc, func

    cutoff = datetime.now(timezone.utc) - timedelta(days=lookback_days)
    # No DISTINCT here — caller dedupes via a set. Adding DISTINCT alongside
    # ORDER BY market_cap fights asyncpg's "ORDER BY must appear in SELECT
    # list" rule for SELECT DISTINCT.
    stmt = (
        select(UniverseSnapshot.ticker)
        .where(UniverseSnapshot.snapshot_date >= cutoff)
        .order_by(desc(UniverseSnapshot.market_cap))
    )
    rows = (await db.execute(stmt)).scalars().all()
    if rows:
        return [t.upper() for t in rows if t]

    # Fallback — pick the latest snapshot date and take its top tickers.
    latest_dt = (
        await db.execute(select(func.max(UniverseSnapshot.snapshot_date)))
    ).scalar()
    if not latest_dt:
        return []
    fallback_stmt = (
        select(UniverseSnapshot.ticker)
        .where(UniverseSnapshot.snapshot_date == latest_dt)
        .order_by(desc(UniverseSnapshot.market_cap))
    )
    rows = (await db.execute(fallback_stmt)).scalars().all()
    logger.warning(
        f"StockTwits universe: no snapshots in last {lookback_days}d; "
        f"falling back to {latest_dt.date()} snapshot ({len(rows)} tickers)"
    )
    return [t.upper() for t in rows if t]


async def _premarket_gap_movers(db: AsyncSession, gap_threshold: float) -> list[str]:
    """
    Today's premarket gap movers above |threshold|.

    Sources:
      AlpacaPriceData: yesterday's daily close (1d) + today's intraday (1m/5m)
      first available premarket bar.

    Returns at most a few hundred tickers; ordered by absolute gap descending
    so movers survive max_size truncation.
    """
    # Lazy import to keep startup fast & break import cycles
    from app.models.alpaca_price import AlpacaPriceData

    today_utc = datetime.now(timezone.utc).date()
    yday_utc = today_utc - timedelta(days=1)

    # Daily close for the prior trading day
    daily_stmt = select(
        AlpacaPriceData.ticker, AlpacaPriceData.close, AlpacaPriceData.date
    ).where(
        AlpacaPriceData.interval == "1d",
        AlpacaPriceData.date >= datetime(yday_utc.year, yday_utc.month, yday_utc.day, tzinfo=timezone.utc) - timedelta(days=4),
        AlpacaPriceData.date < datetime(today_utc.year, today_utc.month, today_utc.day, tzinfo=timezone.utc),
    )
    daily_rows = (await db.execute(daily_stmt)).all()

    # Sort ascending by date so the loop's last write per ticker is the latest
    last_close: dict[str, float] = {}
    for tkr, close, dt in sorted(daily_rows, key=lambda r: r[2]):
        last_close[tkr] = float(close)

    if not last_close:
        return []

    # Premarket bars: between today 04:00 ET (≈ 08:00 UTC EDT, 09:00 EST) and 09:30 ET
    today_start = datetime(today_utc.year, today_utc.month, today_utc.day, 8, 0, tzinfo=timezone.utc)
    today_open = datetime(today_utc.year, today_utc.month, today_utc.day, 13, 30, tzinfo=timezone.utc)
    pm_stmt = select(
        AlpacaPriceData.ticker, AlpacaPriceData.close, AlpacaPriceData.date
    ).where(
        AlpacaPriceData.interval.in_(("1m", "5m", "15m")),
        AlpacaPriceData.date >= today_start,
        AlpacaPriceData.date < today_open,
        AlpacaPriceData.ticker.in_(list(last_close.keys())),
    )
    pm_rows = (await db.execute(pm_stmt)).all()

    # Keep latest premarket close per ticker
    pm_last: dict[str, float] = {}
    for tkr, close, dt in sorted(pm_rows, key=lambda r: r[2]):
        pm_last[tkr] = float(close)

    gaps: list[tuple[str, float]] = []
    for tkr, pm_close in pm_last.items():
        prev_close = last_close.get(tkr)
        if not prev_close or prev_close == 0:
            continue
        gap = (pm_close - prev_close) / prev_close
        if abs(gap) >= gap_threshold:
            gaps.append((tkr, abs(gap)))

    gaps.sort(key=lambda x: x[1], reverse=True)
    return [t for t, _ in gaps]


async def _persist_to_redis(tickers: list[str]) -> None:
    redis = await get_redis()
    if redis is None:
        logger.warning("Redis unavailable — StockTwits universe not persisted")
        return
    try:
        await redis.set(REDIS_KEY, orjson.dumps(tickers), ex=REDIS_TTL_SEC)
    except Exception as e:
        logger.warning(f"Redis set failed: {e}")