You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
286 lines
10 KiB
Python
286 lines
10 KiB
Python
"""
|
|
Feature builder - aggregate raw events into normalized (z-scored) feature dicts.
|
|
"""
|
|
|
|
import logging
|
|
import statistics
|
|
from datetime import datetime, timedelta, timezone
|
|
from typing import Dict, List, Optional
|
|
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
from sqlalchemy import select, and_, func, cast
|
|
from sqlalchemy.dialects.postgresql import JSONB
|
|
|
|
from app.models.overlay_raw_event import (
|
|
OverlayHeadlineEvent,
|
|
OverlayVideoEvent,
|
|
OverlayWikiPageview,
|
|
OverlayTrendObservation,
|
|
)
|
|
from app.models.overlay_registry import ThemeTopicMap
|
|
from app.services.overlay.finra_overlay_loader import FinraOverlayLoader
|
|
from app.core.overlay_config import ZSCORE_WINDOW_DAYS, WINSOR_LOWER, WINSOR_UPPER
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def winsorize(value: float, lower: float = WINSOR_LOWER, upper: float = WINSOR_UPPER) -> float:
|
|
return max(lower, min(upper, value))
|
|
|
|
|
|
|
|
def compute_zscore(value: float, values: List[float]) -> Optional[float]:
|
|
"""Compute z-score of *value* within *values* (requires ≥2 data points)."""
|
|
if len(values) < 2:
|
|
return None
|
|
mean = statistics.mean(values)
|
|
stdev = statistics.pstdev(values) # population stdev for stability
|
|
if stdev == 0:
|
|
return 0.0
|
|
z = (value - mean) / stdev
|
|
return winsorize(z)
|
|
|
|
|
|
def _day_key(ts) -> str:
|
|
"""Return YYYY-MM-DD string from a datetime or date object."""
|
|
if hasattr(ts, "date"):
|
|
return ts.date().isoformat()
|
|
return str(ts)[:10]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Feature builder
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class FeatureBuilder:
|
|
"""Build overlay feature records from raw event tables."""
|
|
|
|
def __init__(self):
|
|
self.finra_loader = FinraOverlayLoader()
|
|
|
|
# ------------------------------------------------------------------
|
|
# Headline features
|
|
# ------------------------------------------------------------------
|
|
|
|
async def build_headline_features(
|
|
self, db: AsyncSession, symbol: str, as_of: datetime
|
|
) -> Dict:
|
|
cutoff_24h = as_of - timedelta(hours=24)
|
|
cutoff_6h = as_of - timedelta(hours=6)
|
|
window_cutoff = as_of - timedelta(days=ZSCORE_WINDOW_DAYS)
|
|
|
|
# Fetch headlines matching symbol in z-score window (DB-level JSON filter)
|
|
symbol_json = cast([symbol], JSONB)
|
|
result = await db.execute(
|
|
select(
|
|
OverlayHeadlineEvent.publisher,
|
|
OverlayHeadlineEvent.published_at,
|
|
).where(
|
|
and_(
|
|
OverlayHeadlineEvent.published_at >= window_cutoff,
|
|
OverlayHeadlineEvent.published_at <= as_of,
|
|
cast(OverlayHeadlineEvent.matched_symbols, JSONB).op('@>')(symbol_json),
|
|
)
|
|
)
|
|
)
|
|
sym_rows_hist = result.fetchall()
|
|
|
|
sym_rows_24h = [r for r in sym_rows_hist if r.published_at >= cutoff_24h]
|
|
sym_rows_6h = [r for r in sym_rows_24h if r.published_at >= cutoff_6h]
|
|
|
|
headline_count_24h = len(sym_rows_24h)
|
|
headline_count_6h = len(sym_rows_6h)
|
|
publishers = {r.publisher for r in sym_rows_24h if r.publisher}
|
|
publisher_breadth_24h = len(publishers)
|
|
|
|
# Build daily counts for z-score window
|
|
daily_counts: Dict[str, int] = {}
|
|
for row in sym_rows_hist:
|
|
key = _day_key(row.published_at)
|
|
daily_counts[key] = daily_counts.get(key, 0) + 1
|
|
|
|
hist_values = list(daily_counts.values())
|
|
headline_burst_z = compute_zscore(float(headline_count_24h), hist_values) if hist_values else None
|
|
|
|
return {
|
|
"headline_count_6h": headline_count_6h,
|
|
"headline_count_24h": headline_count_24h,
|
|
"publisher_breadth_24h": publisher_breadth_24h,
|
|
"headline_burst_z": headline_burst_z,
|
|
}
|
|
|
|
# ------------------------------------------------------------------
|
|
# YouTube features
|
|
# ------------------------------------------------------------------
|
|
|
|
async def build_youtube_features(
|
|
self, db: AsyncSession, symbol: str, as_of: datetime
|
|
) -> Dict:
|
|
cutoff_24h = as_of - timedelta(hours=24)
|
|
window_cutoff = as_of - timedelta(days=ZSCORE_WINDOW_DAYS)
|
|
|
|
symbol_json = cast([symbol], JSONB)
|
|
result = await db.execute(
|
|
select(
|
|
OverlayVideoEvent.view_count,
|
|
OverlayVideoEvent.channel_weight,
|
|
OverlayVideoEvent.published_at,
|
|
).where(
|
|
and_(
|
|
OverlayVideoEvent.published_at >= window_cutoff,
|
|
OverlayVideoEvent.published_at <= as_of,
|
|
cast(OverlayVideoEvent.matched_symbols, JSONB).op('@>')(symbol_json),
|
|
)
|
|
)
|
|
)
|
|
sym_rows = result.fetchall()
|
|
sym_rows_24h = [r for r in sym_rows if r.published_at >= cutoff_24h]
|
|
|
|
mentions_24h = len(sym_rows_24h)
|
|
weighted_views_24h = sum((r.view_count or 0) * (r.channel_weight or 0.5) for r in sym_rows_24h)
|
|
|
|
# Daily weighted views for z-score
|
|
daily_weighted: Dict[str, float] = {}
|
|
for row in sym_rows:
|
|
key = _day_key(row.published_at)
|
|
daily_weighted[key] = daily_weighted.get(key, 0.0) + (row.view_count or 0) * (row.channel_weight or 0.5)
|
|
|
|
hist_values = list(daily_weighted.values())
|
|
youtube_influence_z = compute_zscore(weighted_views_24h, hist_values) if hist_values else None
|
|
|
|
return {
|
|
"youtube_mentions_24h": mentions_24h,
|
|
"youtube_weighted_views_24h": round(weighted_views_24h, 2),
|
|
"youtube_influence_z": youtube_influence_z,
|
|
}
|
|
|
|
# ------------------------------------------------------------------
|
|
# Wiki features
|
|
# ------------------------------------------------------------------
|
|
|
|
async def build_wiki_features(
|
|
self, db: AsyncSession, symbol: str, as_of: datetime
|
|
) -> Dict:
|
|
cutoff_7d = as_of - timedelta(days=7)
|
|
cutoff_1d = as_of - timedelta(days=1)
|
|
window_cutoff = as_of - timedelta(days=ZSCORE_WINDOW_DAYS)
|
|
|
|
result = await db.execute(
|
|
select(
|
|
OverlayWikiPageview.views,
|
|
OverlayWikiPageview.date,
|
|
).where(
|
|
and_(
|
|
OverlayWikiPageview.mapped_symbol == symbol,
|
|
OverlayWikiPageview.date >= window_cutoff,
|
|
)
|
|
).order_by(OverlayWikiPageview.date.desc())
|
|
)
|
|
rows = result.fetchall()
|
|
|
|
if not rows:
|
|
return {"wiki_views_1d": None, "wiki_views_7d_avg": None, "wiki_attention_z": None}
|
|
|
|
# Latest day's views (most recent row, regardless of exact time)
|
|
views_1d = rows[0].views if rows else None
|
|
|
|
# 7-day average
|
|
rows_7d = [r for r in rows if r.date >= cutoff_7d]
|
|
views_7d_avg = sum(r.views for r in rows_7d) / len(rows_7d) if rows_7d else None
|
|
|
|
# Historical z-score
|
|
hist_views = [r.views for r in rows]
|
|
wiki_attention_z = None
|
|
if views_1d is not None and hist_views:
|
|
wiki_attention_z = compute_zscore(float(views_1d), hist_views)
|
|
|
|
return {
|
|
"wiki_views_1d": views_1d,
|
|
"wiki_views_7d_avg": round(views_7d_avg, 2) if views_7d_avg else None,
|
|
"wiki_attention_z": wiki_attention_z,
|
|
}
|
|
|
|
# ------------------------------------------------------------------
|
|
# Google Trends features
|
|
# ------------------------------------------------------------------
|
|
|
|
async def build_trends_features(
|
|
self, db: AsyncSession, symbol: str, as_of: datetime
|
|
) -> Dict:
|
|
window_cutoff = as_of - timedelta(days=ZSCORE_WINDOW_DAYS)
|
|
cutoff_1d = as_of - timedelta(days=1)
|
|
|
|
# Find topic IDs mapped to this symbol
|
|
topics_result = await db.execute(
|
|
select(ThemeTopicMap).where(ThemeTopicMap.active == True)
|
|
)
|
|
topics = topics_result.scalars().all()
|
|
topic_ids = [t.topic_id for t in topics if symbol in (t.mapped_symbols or [])]
|
|
|
|
if not topic_ids:
|
|
return {"theme_heat_z": None}
|
|
|
|
result = await db.execute(
|
|
select(
|
|
OverlayTrendObservation.interest_value,
|
|
OverlayTrendObservation.observed_at,
|
|
).where(
|
|
and_(
|
|
OverlayTrendObservation.topic_id.in_(topic_ids),
|
|
OverlayTrendObservation.observed_at >= window_cutoff,
|
|
)
|
|
).order_by(OverlayTrendObservation.observed_at)
|
|
)
|
|
rows = result.fetchall()
|
|
|
|
if not rows:
|
|
return {"theme_heat_z": None}
|
|
|
|
recent = [r for r in rows if r.observed_at >= cutoff_1d]
|
|
current_value = float(sum(r.interest_value for r in recent) / len(recent)) if recent else None
|
|
|
|
if current_value is None:
|
|
return {"theme_heat_z": None}
|
|
|
|
hist_values = [float(r.interest_value) for r in rows]
|
|
theme_heat_z = compute_zscore(current_value, hist_values) if len(hist_values) >= 2 else None
|
|
|
|
return {"theme_heat_z": theme_heat_z}
|
|
|
|
# ------------------------------------------------------------------
|
|
# FINRA crowding features
|
|
# ------------------------------------------------------------------
|
|
|
|
async def build_crowding_features(self, db: AsyncSession, symbol: str) -> Dict:
|
|
return await self.finra_loader.get_crowding_metrics(db, symbol)
|
|
|
|
# ------------------------------------------------------------------
|
|
# Build all features
|
|
# ------------------------------------------------------------------
|
|
|
|
async def build_all_features(
|
|
self, db: AsyncSession, symbol: str, as_of: Optional[datetime] = None
|
|
) -> Dict:
|
|
"""Build all features for a symbol and return a combined feature dict."""
|
|
if as_of is None:
|
|
as_of = datetime.now(timezone.utc)
|
|
|
|
headline = await self.build_headline_features(db, symbol, as_of)
|
|
youtube = await self.build_youtube_features(db, symbol, as_of)
|
|
wiki = await self.build_wiki_features(db, symbol, as_of)
|
|
trends = await self.build_trends_features(db, symbol, as_of)
|
|
crowding = await self.build_crowding_features(db, symbol)
|
|
|
|
return {
|
|
**headline,
|
|
**youtube,
|
|
**wiki,
|
|
**trends,
|
|
**crowding,
|
|
"as_of_ts": as_of,
|
|
}
|