""" Dynamic StockTwits subscription universe. Single V49 universe ("today only") would create a 16-hour data hole during the premarket window (prev 16:00 ET → today 09:30 ET). Instead we union: (last N days of UniverseSnapshot active tickers) ∪ (today's premarket gap movers above threshold) Capped at STOCKTWITS_UNIVERSE_MAX_SIZE (default 300) to fit comfortably under the 200-req/hr StockTwits ceiling at one poll per 5 minutes. Computed once per trading day at 09:00 ET and cached in Redis under `news_v2:stocktwits:universe`. The poller reads that key each cycle. """ from __future__ import annotations import logging from datetime import datetime, timedelta, timezone import orjson from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from app.core.config import settings from app.core.database import AsyncSessionLocal from app.models.universe_snapshot import UniverseSnapshot from app.utils.cache import get_redis logger = logging.getLogger(__name__) REDIS_KEY = "news_v2:stocktwits:universe" REDIS_TTL_SEC = 24 * 60 * 60 # 1 day async def compute_stocktwits_universe( db: AsyncSession | None = None, lookback_days: int | None = None, max_size: int | None = None, gap_threshold: float | None = None, ) -> list[str]: """Compute and persist the StockTwits subscription universe to Redis.""" lookback_days = lookback_days or settings.STOCKTWITS_UNIVERSE_LOOKBACK_DAYS max_size = max_size or settings.STOCKTWITS_UNIVERSE_MAX_SIZE gap_threshold = gap_threshold if gap_threshold is not None else settings.STOCKTWITS_PREMARKET_GAP_THRESHOLD own_session = db is None if own_session: db = AsyncSessionLocal() await db.__aenter__() # type: ignore[attr-defined] try: recent_universe = await _recent_universe_tickers(db, lookback_days) # type: ignore[arg-type] movers = await _premarket_gap_movers(db, gap_threshold) # type: ignore[arg-type] finally: if own_session and db is not None: await db.__aexit__(None, None, None) # type: ignore[attr-defined] union: list[str] = [] seen: set[str] = set() # Movers first so they survive if we hit max_size truncation for t in movers + recent_universe: if t and t not in seen: seen.add(t) union.append(t) if len(union) >= max_size: break await _persist_to_redis(union) logger.info( f"StockTwits universe: {len(union)} tickers " f"({len(movers)} movers, {len(recent_universe)} recent universe, capped at {max_size})" ) return union async def get_cached_universe() -> list[str]: """Read the current Redis-cached universe; returns [] if absent.""" redis = await get_redis() if redis is None: return [] try: raw = await redis.get(REDIS_KEY) if raw is None: return [] data = orjson.loads(raw) if isinstance(data, list): return [str(t).upper() for t in data] except Exception as e: logger.warning(f"StockTwits universe read failed: {e}") return [] # --------------------------------------------------------------------------- # Internals # --------------------------------------------------------------------------- async def _recent_universe_tickers(db: AsyncSession, lookback_days: int) -> list[str]: """Tickers from UniverseSnapshot, ranked by market cap. Prefers the recent ``lookback_days`` window. If that's empty (the monthly snapshot job hasn't refreshed in a while), falls back to the most recent snapshot in the table — top market-cap tickers stay roughly stable across months, so a stale snapshot is still useful for keeping the StockTwits poll alive instead of returning []. """ from sqlalchemy import desc, func cutoff = datetime.now(timezone.utc) - timedelta(days=lookback_days) # No DISTINCT here — caller dedupes via a set. Adding DISTINCT alongside # ORDER BY market_cap fights asyncpg's "ORDER BY must appear in SELECT # list" rule for SELECT DISTINCT. stmt = ( select(UniverseSnapshot.ticker) .where(UniverseSnapshot.snapshot_date >= cutoff) .order_by(desc(UniverseSnapshot.market_cap)) ) rows = (await db.execute(stmt)).scalars().all() if rows: return [t.upper() for t in rows if t] # Fallback — pick the latest snapshot date and take its top tickers. latest_dt = ( await db.execute(select(func.max(UniverseSnapshot.snapshot_date))) ).scalar() if not latest_dt: return [] fallback_stmt = ( select(UniverseSnapshot.ticker) .where(UniverseSnapshot.snapshot_date == latest_dt) .order_by(desc(UniverseSnapshot.market_cap)) ) rows = (await db.execute(fallback_stmt)).scalars().all() logger.warning( f"StockTwits universe: no snapshots in last {lookback_days}d; " f"falling back to {latest_dt.date()} snapshot ({len(rows)} tickers)" ) return [t.upper() for t in rows if t] async def _premarket_gap_movers(db: AsyncSession, gap_threshold: float) -> list[str]: """ Today's premarket gap movers above |threshold|. Sources: AlpacaPriceData: yesterday's daily close (1d) + today's intraday (1m/5m) first available premarket bar. Returns at most a few hundred tickers; ordered by absolute gap descending so movers survive max_size truncation. """ # Lazy import to keep startup fast & break import cycles from app.models.alpaca_price import AlpacaPriceData today_utc = datetime.now(timezone.utc).date() yday_utc = today_utc - timedelta(days=1) # Daily close for the prior trading day daily_stmt = select( AlpacaPriceData.ticker, AlpacaPriceData.close, AlpacaPriceData.date ).where( AlpacaPriceData.interval == "1d", AlpacaPriceData.date >= datetime(yday_utc.year, yday_utc.month, yday_utc.day, tzinfo=timezone.utc) - timedelta(days=4), AlpacaPriceData.date < datetime(today_utc.year, today_utc.month, today_utc.day, tzinfo=timezone.utc), ) daily_rows = (await db.execute(daily_stmt)).all() # Sort ascending by date so the loop's last write per ticker is the latest last_close: dict[str, float] = {} for tkr, close, dt in sorted(daily_rows, key=lambda r: r[2]): last_close[tkr] = float(close) if not last_close: return [] # Premarket bars: between today 04:00 ET (≈ 08:00 UTC EDT, 09:00 EST) and 09:30 ET today_start = datetime(today_utc.year, today_utc.month, today_utc.day, 8, 0, tzinfo=timezone.utc) today_open = datetime(today_utc.year, today_utc.month, today_utc.day, 13, 30, tzinfo=timezone.utc) pm_stmt = select( AlpacaPriceData.ticker, AlpacaPriceData.close, AlpacaPriceData.date ).where( AlpacaPriceData.interval.in_(("1m", "5m", "15m")), AlpacaPriceData.date >= today_start, AlpacaPriceData.date < today_open, AlpacaPriceData.ticker.in_(list(last_close.keys())), ) pm_rows = (await db.execute(pm_stmt)).all() # Keep latest premarket close per ticker pm_last: dict[str, float] = {} for tkr, close, dt in sorted(pm_rows, key=lambda r: r[2]): pm_last[tkr] = float(close) gaps: list[tuple[str, float]] = [] for tkr, pm_close in pm_last.items(): prev_close = last_close.get(tkr) if not prev_close or prev_close == 0: continue gap = (pm_close - prev_close) / prev_close if abs(gap) >= gap_threshold: gaps.append((tkr, abs(gap))) gaps.sort(key=lambda x: x[1], reverse=True) return [t for t, _ in gaps] async def _persist_to_redis(tickers: list[str]) -> None: redis = await get_redis() if redis is None: logger.warning("Redis unavailable — StockTwits universe not persisted") return try: await redis.set(REDIS_KEY, orjson.dumps(tickers), ex=REDIS_TTL_SEC) except Exception as e: logger.warning(f"Redis set failed: {e}")