""" Feature builder - aggregate raw events into normalized (z-scored) feature dicts. """ import logging import statistics from datetime import datetime, timedelta, timezone from typing import Dict, List, Optional from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy import select, and_, func from app.models.overlay_raw_event import ( OverlayHeadlineEvent, OverlayVideoEvent, OverlayWikiPageview, ) from app.services.overlay.finra_overlay_loader import FinraOverlayLoader from app.core.overlay_config import ZSCORE_WINDOW_DAYS, WINSOR_LOWER, WINSOR_UPPER logger = logging.getLogger(__name__) # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- def winsorize(value: float, lower: float = WINSOR_LOWER, upper: float = WINSOR_UPPER) -> float: return max(lower, min(upper, value)) def _ensure_utc(dt) -> datetime: """Ensure a datetime is UTC-aware (SQLite returns naive datetimes).""" if dt is None: return dt if isinstance(dt, datetime) and dt.tzinfo is None: return dt.replace(tzinfo=timezone.utc) return dt def compute_zscore(value: float, values: List[float]) -> Optional[float]: """Compute z-score of *value* within *values* (requires ≥2 data points).""" if len(values) < 2: return None mean = statistics.mean(values) stdev = statistics.pstdev(values) # population stdev for stability if stdev == 0: return 0.0 z = (value - mean) / stdev return winsorize(z) def _day_key(ts) -> str: """Return YYYY-MM-DD string from a datetime or date object.""" if hasattr(ts, "date"): return ts.date().isoformat() return str(ts)[:10] # --------------------------------------------------------------------------- # Feature builder # --------------------------------------------------------------------------- class FeatureBuilder: """Build overlay feature records from raw event tables.""" def __init__(self): self.finra_loader = FinraOverlayLoader() # ------------------------------------------------------------------ # Headline features # ------------------------------------------------------------------ async def build_headline_features( self, db: AsyncSession, symbol: str, as_of: datetime ) -> Dict: cutoff_24h = as_of - timedelta(hours=24) cutoff_6h = as_of - timedelta(hours=6) window_cutoff = as_of - timedelta(days=ZSCORE_WINDOW_DAYS) # Fetch all headlines in z-score window (we need matched_symbols for filtering) result = await db.execute( select( OverlayHeadlineEvent.publisher, OverlayHeadlineEvent.published_at, OverlayHeadlineEvent.matched_symbols, ).where( and_( OverlayHeadlineEvent.published_at >= window_cutoff, OverlayHeadlineEvent.published_at <= as_of, ) ) ) all_rows = result.fetchall() # Filter by symbol sym_rows_hist = [r for r in all_rows if symbol in (r.matched_symbols or [])] sym_rows_24h = [r for r in sym_rows_hist if _ensure_utc(r.published_at) >= cutoff_24h] sym_rows_6h = [r for r in sym_rows_24h if _ensure_utc(r.published_at) >= cutoff_6h] headline_count_24h = len(sym_rows_24h) headline_count_6h = len(sym_rows_6h) publishers = {r.publisher for r in sym_rows_24h if r.publisher} publisher_breadth_24h = len(publishers) # Build daily counts for z-score window daily_counts: Dict[str, int] = {} for row in sym_rows_hist: key = _day_key(row.published_at) daily_counts[key] = daily_counts.get(key, 0) + 1 hist_values = list(daily_counts.values()) headline_burst_z = compute_zscore(float(headline_count_24h), hist_values) if hist_values else None return { "headline_count_6h": headline_count_6h, "headline_count_24h": headline_count_24h, "publisher_breadth_24h": publisher_breadth_24h, "headline_burst_z": headline_burst_z, } # ------------------------------------------------------------------ # YouTube features # ------------------------------------------------------------------ async def build_youtube_features( self, db: AsyncSession, symbol: str, as_of: datetime ) -> Dict: cutoff_24h = as_of - timedelta(hours=24) window_cutoff = as_of - timedelta(days=ZSCORE_WINDOW_DAYS) result = await db.execute( select( OverlayVideoEvent.view_count, OverlayVideoEvent.channel_weight, OverlayVideoEvent.published_at, OverlayVideoEvent.matched_symbols, ).where( and_( OverlayVideoEvent.published_at >= window_cutoff, OverlayVideoEvent.published_at <= as_of, ) ) ) all_rows = result.fetchall() sym_rows = [r for r in all_rows if symbol in (r.matched_symbols or [])] sym_rows_24h = [r for r in sym_rows if _ensure_utc(r.published_at) >= cutoff_24h] mentions_24h = len(sym_rows_24h) weighted_views_24h = sum(r.view_count * r.channel_weight for r in sym_rows_24h) # Daily weighted views for z-score daily_weighted: Dict[str, float] = {} for row in sym_rows: key = _day_key(row.published_at) daily_weighted[key] = daily_weighted.get(key, 0.0) + row.view_count * row.channel_weight hist_values = list(daily_weighted.values()) youtube_influence_z = compute_zscore(weighted_views_24h, hist_values) if hist_values else None return { "youtube_mentions_24h": mentions_24h, "youtube_weighted_views_24h": round(weighted_views_24h, 2), "youtube_influence_z": youtube_influence_z, } # ------------------------------------------------------------------ # Wiki features # ------------------------------------------------------------------ async def build_wiki_features( self, db: AsyncSession, symbol: str, as_of: datetime ) -> Dict: cutoff_7d = as_of - timedelta(days=7) cutoff_1d = as_of - timedelta(days=1) window_cutoff = as_of - timedelta(days=ZSCORE_WINDOW_DAYS) result = await db.execute( select( OverlayWikiPageview.views, OverlayWikiPageview.date, ).where( and_( OverlayWikiPageview.mapped_symbol == symbol, OverlayWikiPageview.date >= window_cutoff, ) ).order_by(OverlayWikiPageview.date.desc()) ) rows = result.fetchall() if not rows: return {"wiki_views_1d": None, "wiki_views_7d_avg": None, "wiki_attention_z": None} # Latest day's views (most recent row, regardless of exact time) views_1d = rows[0].views if rows else None # 7-day average rows_7d = [r for r in rows if _ensure_utc(r.date) >= cutoff_7d] views_7d_avg = sum(r.views for r in rows_7d) / len(rows_7d) if rows_7d else None # Historical z-score hist_views = [r.views for r in rows] wiki_attention_z = None if views_1d is not None and hist_views: wiki_attention_z = compute_zscore(float(views_1d), hist_views) return { "wiki_views_1d": views_1d, "wiki_views_7d_avg": round(views_7d_avg, 2) if views_7d_avg else None, "wiki_attention_z": wiki_attention_z, } # ------------------------------------------------------------------ # FINRA crowding features # ------------------------------------------------------------------ async def build_crowding_features(self, db: AsyncSession, symbol: str) -> Dict: return await self.finra_loader.get_crowding_metrics(db, symbol) # ------------------------------------------------------------------ # Build all features # ------------------------------------------------------------------ async def build_all_features( self, db: AsyncSession, symbol: str, as_of: Optional[datetime] = None ) -> Dict: """Build all features for a symbol and return a combined feature dict.""" if as_of is None: as_of = datetime.now(timezone.utc) headline = await self.build_headline_features(db, symbol, as_of) youtube = await self.build_youtube_features(db, symbol, as_of) wiki = await self.build_wiki_features(db, symbol, as_of) crowding = await self.build_crowding_features(db, symbol) return { **headline, **youtube, **wiki, **crowding, "as_of_ts": as_of, # theme_heat_z comes from Google Trends; left None here (no Trends data yet) "theme_heat_z": None, }