""" Feature builder - aggregate raw events into normalized (z-scored) feature dicts. """ import logging import statistics from datetime import datetime, timedelta, timezone from typing import Dict, List, Optional from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy import select, and_, func, cast from sqlalchemy.dialects.postgresql import JSONB from app.models.overlay_raw_event import ( OverlayHeadlineEvent, OverlayVideoEvent, OverlayWikiPageview, OverlayTrendObservation, ) from app.models.overlay_registry import ThemeTopicMap from app.services.overlay.finra_overlay_loader import FinraOverlayLoader from app.core.overlay_config import ZSCORE_WINDOW_DAYS, WINSOR_LOWER, WINSOR_UPPER logger = logging.getLogger(__name__) # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- def winsorize(value: float, lower: float = WINSOR_LOWER, upper: float = WINSOR_UPPER) -> float: return max(lower, min(upper, value)) def compute_zscore(value: float, values: List[float]) -> Optional[float]: """Compute z-score of *value* within *values* (requires ≥2 data points).""" if len(values) < 2: return None mean = statistics.mean(values) stdev = statistics.pstdev(values) # population stdev for stability if stdev == 0: return 0.0 z = (value - mean) / stdev return winsorize(z) def _day_key(ts) -> str: """Return YYYY-MM-DD string from a datetime or date object.""" if hasattr(ts, "date"): return ts.date().isoformat() return str(ts)[:10] # --------------------------------------------------------------------------- # Feature builder # --------------------------------------------------------------------------- class FeatureBuilder: """Build overlay feature records from raw event tables.""" def __init__(self): self.finra_loader = FinraOverlayLoader() # ------------------------------------------------------------------ # Headline features # ------------------------------------------------------------------ async def build_headline_features( self, db: AsyncSession, symbol: str, as_of: datetime ) -> Dict: cutoff_24h = as_of - timedelta(hours=24) cutoff_6h = as_of - timedelta(hours=6) window_cutoff = as_of - timedelta(days=ZSCORE_WINDOW_DAYS) # Fetch headlines matching symbol in z-score window (DB-level JSON filter) symbol_json = cast([symbol], JSONB) result = await db.execute( select( OverlayHeadlineEvent.publisher, OverlayHeadlineEvent.published_at, ).where( and_( OverlayHeadlineEvent.published_at >= window_cutoff, OverlayHeadlineEvent.published_at <= as_of, cast(OverlayHeadlineEvent.matched_symbols, JSONB).op('@>')(symbol_json), ) ) ) sym_rows_hist = result.fetchall() sym_rows_24h = [r for r in sym_rows_hist if r.published_at >= cutoff_24h] sym_rows_6h = [r for r in sym_rows_24h if r.published_at >= cutoff_6h] headline_count_24h = len(sym_rows_24h) headline_count_6h = len(sym_rows_6h) publishers = {r.publisher for r in sym_rows_24h if r.publisher} publisher_breadth_24h = len(publishers) # Build daily counts for z-score window daily_counts: Dict[str, int] = {} for row in sym_rows_hist: key = _day_key(row.published_at) daily_counts[key] = daily_counts.get(key, 0) + 1 hist_values = list(daily_counts.values()) headline_burst_z = compute_zscore(float(headline_count_24h), hist_values) if hist_values else None return { "headline_count_6h": headline_count_6h, "headline_count_24h": headline_count_24h, "publisher_breadth_24h": publisher_breadth_24h, "headline_burst_z": headline_burst_z, } # ------------------------------------------------------------------ # YouTube features # ------------------------------------------------------------------ async def build_youtube_features( self, db: AsyncSession, symbol: str, as_of: datetime ) -> Dict: cutoff_24h = as_of - timedelta(hours=24) window_cutoff = as_of - timedelta(days=ZSCORE_WINDOW_DAYS) symbol_json = cast([symbol], JSONB) result = await db.execute( select( OverlayVideoEvent.view_count, OverlayVideoEvent.channel_weight, OverlayVideoEvent.published_at, ).where( and_( OverlayVideoEvent.published_at >= window_cutoff, OverlayVideoEvent.published_at <= as_of, cast(OverlayVideoEvent.matched_symbols, JSONB).op('@>')(symbol_json), ) ) ) sym_rows = result.fetchall() sym_rows_24h = [r for r in sym_rows if r.published_at >= cutoff_24h] mentions_24h = len(sym_rows_24h) weighted_views_24h = sum((r.view_count or 0) * (r.channel_weight or 0.5) for r in sym_rows_24h) # Daily weighted views for z-score daily_weighted: Dict[str, float] = {} for row in sym_rows: key = _day_key(row.published_at) daily_weighted[key] = daily_weighted.get(key, 0.0) + (row.view_count or 0) * (row.channel_weight or 0.5) hist_values = list(daily_weighted.values()) youtube_influence_z = compute_zscore(weighted_views_24h, hist_values) if hist_values else None return { "youtube_mentions_24h": mentions_24h, "youtube_weighted_views_24h": round(weighted_views_24h, 2), "youtube_influence_z": youtube_influence_z, } # ------------------------------------------------------------------ # Wiki features # ------------------------------------------------------------------ async def build_wiki_features( self, db: AsyncSession, symbol: str, as_of: datetime ) -> Dict: cutoff_7d = as_of - timedelta(days=7) cutoff_1d = as_of - timedelta(days=1) window_cutoff = as_of - timedelta(days=ZSCORE_WINDOW_DAYS) result = await db.execute( select( OverlayWikiPageview.views, OverlayWikiPageview.date, ).where( and_( OverlayWikiPageview.mapped_symbol == symbol, OverlayWikiPageview.date >= window_cutoff, ) ).order_by(OverlayWikiPageview.date.desc()) ) rows = result.fetchall() if not rows: return {"wiki_views_1d": None, "wiki_views_7d_avg": None, "wiki_attention_z": None} # Latest day's views (most recent row, regardless of exact time) views_1d = rows[0].views if rows else None # 7-day average rows_7d = [r for r in rows if r.date >= cutoff_7d] views_7d_avg = sum(r.views for r in rows_7d) / len(rows_7d) if rows_7d else None # Historical z-score hist_views = [r.views for r in rows] wiki_attention_z = None if views_1d is not None and hist_views: wiki_attention_z = compute_zscore(float(views_1d), hist_views) return { "wiki_views_1d": views_1d, "wiki_views_7d_avg": round(views_7d_avg, 2) if views_7d_avg else None, "wiki_attention_z": wiki_attention_z, } # ------------------------------------------------------------------ # Google Trends features # ------------------------------------------------------------------ async def build_trends_features( self, db: AsyncSession, symbol: str, as_of: datetime ) -> Dict: window_cutoff = as_of - timedelta(days=ZSCORE_WINDOW_DAYS) cutoff_1d = as_of - timedelta(days=1) # Find topic IDs mapped to this symbol topics_result = await db.execute( select(ThemeTopicMap).where(ThemeTopicMap.active == True) ) topics = topics_result.scalars().all() topic_ids = [t.topic_id for t in topics if symbol in (t.mapped_symbols or [])] if not topic_ids: return {"theme_heat_z": None} result = await db.execute( select( OverlayTrendObservation.interest_value, OverlayTrendObservation.observed_at, ).where( and_( OverlayTrendObservation.topic_id.in_(topic_ids), OverlayTrendObservation.observed_at >= window_cutoff, ) ).order_by(OverlayTrendObservation.observed_at) ) rows = result.fetchall() if not rows: return {"theme_heat_z": None} recent = [r for r in rows if r.observed_at >= cutoff_1d] current_value = float(sum(r.interest_value for r in recent) / len(recent)) if recent else None if current_value is None: return {"theme_heat_z": None} hist_values = [float(r.interest_value) for r in rows] theme_heat_z = compute_zscore(current_value, hist_values) if len(hist_values) >= 2 else None return {"theme_heat_z": theme_heat_z} # ------------------------------------------------------------------ # FINRA crowding features # ------------------------------------------------------------------ async def build_crowding_features(self, db: AsyncSession, symbol: str) -> Dict: return await self.finra_loader.get_crowding_metrics(db, symbol) # ------------------------------------------------------------------ # Build all features # ------------------------------------------------------------------ async def build_all_features( self, db: AsyncSession, symbol: str, as_of: Optional[datetime] = None ) -> Dict: """Build all features for a symbol and return a combined feature dict.""" if as_of is None: as_of = datetime.now(timezone.utc) headline = await self.build_headline_features(db, symbol, as_of) youtube = await self.build_youtube_features(db, symbol, as_of) wiki = await self.build_wiki_features(db, symbol, as_of) trends = await self.build_trends_features(db, symbol, as_of) crowding = await self.build_crowding_features(db, symbol) return { **headline, **youtube, **wiki, **trends, **crowding, "as_of_ts": as_of, }