feat: day_gainers 5분봉 수집 + News v2 ingest + overlay 정리
주요 변경사항: gainer snapshot (신규) - gainer_snapshots 테이블: 5분 단위 intraday top-100 day_gainers 스냅샷 - APScheduler CronTrigger (*/5 min, Mon-Fri ET) + 장중 guard (9:30~16:00) - alembic migration p7g8h9i0j1k2 /stocks/gainers 엔드포인트 (신규) - yfinance day_gainers preset, 실시간(캐시 없음) - yfinance_plus screen() wrapper — 모든 screen() 호출에 세션 풀 + 브라우저 지문 우회 적용 News v2 ingest (신규) - Alpaca News + StockTwits + Finnhub 수집 파이프라인 - news_headlines 테이블 + scheduler (5분 realtime poll, 일 1회 backfill) Overlay 정리 - 미사용 서브시스템 제거: wikimedia, youtube, google_trends, feature_builder, overlay_scorer - Yahoo RSS 어댑터 유지, 파이프라인 단순화 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>main
I Luk Kim
3 months ago
parent
27990ea908
commit
1d087b54a4
@ -0,0 +1,54 @@
|
||||
"""add news_headline table
|
||||
|
||||
Revision ID: n5d6e7f8h9i0
|
||||
Revises: m4e5f6g7h8i9
|
||||
Create Date: 2026-04-25
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
revision: str = "n5d6e7f8h9i0"
|
||||
down_revision: Union[str, Sequence[str], None] = "m4e5f6g7h8i9"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
|
||||
if not conn.dialect.has_table(conn, "news_headline"):
|
||||
op.create_table(
|
||||
"news_headline",
|
||||
sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True),
|
||||
sa.Column("source", sa.String(30), nullable=False),
|
||||
sa.Column("source_id", sa.String(100), nullable=False),
|
||||
sa.Column("ticker", sa.String(10), nullable=False),
|
||||
sa.Column("tickers_all", postgresql.ARRAY(sa.String(10)), nullable=True),
|
||||
sa.Column("published_at", postgresql.TIMESTAMP(timezone=True), nullable=False),
|
||||
sa.Column("headline", sa.Text(), nullable=False),
|
||||
sa.Column("summary", sa.Text(), nullable=True),
|
||||
sa.Column("url", sa.Text(), nullable=True),
|
||||
sa.Column("language", sa.String(8), server_default="en"),
|
||||
sa.Column("vendor_categories", postgresql.ARRAY(sa.String(50)), nullable=True),
|
||||
sa.Column("categories", postgresql.ARRAY(sa.String(50)), nullable=True),
|
||||
sa.Column("raw_sentiment", sa.Float(), nullable=True),
|
||||
sa.Column("is_primary", sa.Boolean(), nullable=False, server_default=sa.text("false")),
|
||||
sa.Column("ingested_at", postgresql.TIMESTAMP(timezone=True), nullable=False),
|
||||
sa.Column("created_at", postgresql.TIMESTAMP(timezone=True)),
|
||||
sa.Column("updated_at", postgresql.TIMESTAMP(timezone=True)),
|
||||
sa.UniqueConstraint(
|
||||
"source", "source_id", "ticker",
|
||||
name="uq_news_headline_source_ticker",
|
||||
),
|
||||
)
|
||||
op.create_index("idx_news_ticker_published", "news_headline", ["ticker", "published_at"])
|
||||
op.create_index("idx_news_published", "news_headline", ["published_at"])
|
||||
op.create_index("idx_news_source_published", "news_headline", ["source", "published_at"])
|
||||
op.create_index("idx_news_ingested", "news_headline", ["ingested_at"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("news_headline")
|
||||
@ -0,0 +1,43 @@
|
||||
"""drop unused overlay tables
|
||||
|
||||
Revision ID: o6e7f8g9j0a1
|
||||
Revises: n5d6e7f8h9i0
|
||||
Create Date: 2026-04-26
|
||||
|
||||
Removes overlay subsystem tables that are no longer fed (wikimedia / youtube /
|
||||
google_trends / feature_build / scoring were retired). Keeps:
|
||||
- overlay_headline_events
|
||||
- overlay_job_log
|
||||
- company_aliases
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "o6e7f8g9j0a1"
|
||||
down_revision: Union[str, Sequence[str], None] = "n5d6e7f8h9i0"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
_TABLES_TO_DROP = [
|
||||
"overlay_feature_records",
|
||||
"overlay_video_events",
|
||||
"overlay_wiki_pageviews",
|
||||
"overlay_trend_observations",
|
||||
"youtube_channel_registry",
|
||||
"wiki_page_map",
|
||||
"theme_topic_map",
|
||||
]
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
for tbl in _TABLES_TO_DROP:
|
||||
if conn.dialect.has_table(conn, tbl):
|
||||
op.drop_table(tbl)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# One-way drop; recreate would require restoring the deleted models.
|
||||
pass
|
||||
@ -0,0 +1,52 @@
|
||||
"""add gainer_snapshots table
|
||||
|
||||
Revision ID: p7g8h9i0j1k2
|
||||
Revises: o6e7f8g9j0a1
|
||||
Create Date: 2026-05-06
|
||||
|
||||
5-minute intraday snapshots of Yahoo Finance day_gainers for backtesting.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
revision: str = "p7g8h9i0j1k2"
|
||||
down_revision: Union[str, Sequence[str], None] = "o6e7f8g9j0a1"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"gainer_snapshots",
|
||||
sa.Column("id", postgresql.UUID(as_uuid=True), nullable=False),
|
||||
sa.Column("snapshot_at", sa.TIMESTAMP(timezone=True), nullable=False),
|
||||
sa.Column("rank", sa.Integer(), nullable=False),
|
||||
sa.Column("symbol", sa.String(10), nullable=False),
|
||||
sa.Column("name", sa.Text(), nullable=True),
|
||||
sa.Column("exchange", sa.String(20), nullable=True),
|
||||
sa.Column("price", sa.Float(), nullable=True),
|
||||
sa.Column("change_percent", sa.Float(), nullable=True),
|
||||
sa.Column("volume", sa.BigInteger(), nullable=True),
|
||||
sa.Column("avg_volume_3m", sa.BigInteger(), nullable=True),
|
||||
sa.Column("market_cap", sa.BigInteger(), nullable=True),
|
||||
sa.Column("pe_ratio", sa.Float(), nullable=True),
|
||||
sa.Column("forward_pe", sa.Float(), nullable=True),
|
||||
sa.Column("eps_ttm", sa.Float(), nullable=True),
|
||||
sa.Column("dividend_yield", sa.Float(), nullable=True),
|
||||
sa.Column("fifty_two_week_high", sa.Float(), nullable=True),
|
||||
sa.Column("fifty_two_week_low", sa.Float(), nullable=True),
|
||||
sa.Column("created_at", sa.TIMESTAMP(timezone=True), nullable=True),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
sa.UniqueConstraint("snapshot_at", "symbol", name="uq_gainer_snapshot_symbol"),
|
||||
)
|
||||
op.create_index("idx_gainer_snapshot_at", "gainer_snapshots", ["snapshot_at"])
|
||||
op.create_index("idx_gainer_symbol_snapshot", "gainer_snapshots", ["symbol", "snapshot_at"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("idx_gainer_symbol_snapshot", table_name="gainer_snapshots")
|
||||
op.drop_index("idx_gainer_snapshot_at", table_name="gainer_snapshots")
|
||||
op.drop_table("gainer_snapshots")
|
||||
@ -0,0 +1,392 @@
|
||||
"""
|
||||
News v2 endpoints — multi-source headline + session aggregate API.
|
||||
|
||||
Mounted at `/api/v1/news/v2/*` to avoid conflict with the legacy
|
||||
`GET /news/{ticker}` aggregator (which uses a wildcard path).
|
||||
|
||||
Endpoints:
|
||||
GET /headlines raw headline rows
|
||||
GET /session_aggregate single (ticker, session_date, window)
|
||||
POST /session_aggregate/batch many tickers in one shot
|
||||
GET /coverage per-source ingest depth probe
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import date, datetime, timezone
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, Body, Depends, HTTPException, Query
|
||||
from fastapi.responses import Response
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.database import get_db
|
||||
from app.models.news_headline import NewsHeadline
|
||||
from app.services.news.session_aggregator import (
|
||||
aggregate_session,
|
||||
aggregate_session_batch,
|
||||
)
|
||||
from app.services.news.session_window import session_window
|
||||
from app.utils.cache import with_cache
|
||||
|
||||
logger = logging.getLogger("app.api.v1.news_v2")
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
# Concurrency gates — prevent event-loop saturation under bulk scans
|
||||
_HEADLINES_SEMAPHORE = asyncio.Semaphore(8)
|
||||
_AGG_SEMAPHORE = asyncio.Semaphore(8)
|
||||
_SEMAPHORE_WAIT_TIMEOUT = 10.0
|
||||
|
||||
_VALID_SOURCES = {"alpaca_benzinga", "stocktwits", "finnhub", "gdelt"}
|
||||
_VALID_WINDOWS = {"premarket", "intraday", "post", "full_session"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Schemas
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class HeadlineItem(BaseModel):
|
||||
source: str
|
||||
source_id: str
|
||||
ticker: str
|
||||
tickers_all: list[str] | None = None
|
||||
published_at: str
|
||||
headline: str
|
||||
summary: str | None = None
|
||||
url: str | None = None
|
||||
language: str | None = None
|
||||
vendor_categories: list[str] | None = None
|
||||
categories: list[str] | None = None
|
||||
raw_sentiment: float | None = None
|
||||
is_primary: bool
|
||||
ingested_at: str
|
||||
|
||||
|
||||
class HeadlinesResponse(BaseModel):
|
||||
items: list[HeadlineItem]
|
||||
next_cursor: str | None = None
|
||||
|
||||
|
||||
class SocialStatsItem(BaseModel):
|
||||
message_count: int = 0
|
||||
bull_count: int = 0
|
||||
bear_count: int = 0
|
||||
bull_bear_ratio: float | None = None
|
||||
|
||||
|
||||
class SessionAggregateItem(BaseModel):
|
||||
ticker: str
|
||||
session_date: str
|
||||
window: str
|
||||
headline_count: int = 0
|
||||
primary_count: int = 0
|
||||
first_headline_at: str | None = None
|
||||
last_headline_at: str | None = None
|
||||
category_counts: dict[str, int] = Field(default_factory=dict)
|
||||
sentiment_mean: float | None = None
|
||||
sentiment_recency_weighted: float | None = None
|
||||
social: SocialStatsItem = Field(default_factory=SocialStatsItem)
|
||||
sources_present: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class SessionAggregateBatchRequest(BaseModel):
|
||||
session_date: date
|
||||
window: str
|
||||
symbols: list[str]
|
||||
sources: list[str] | None = None
|
||||
|
||||
@field_validator("window")
|
||||
@classmethod
|
||||
def _validate_window(cls, v: str) -> str:
|
||||
if v not in _VALID_WINDOWS:
|
||||
raise ValueError(f"window must be one of {sorted(_VALID_WINDOWS)}")
|
||||
return v
|
||||
|
||||
@field_validator("symbols")
|
||||
@classmethod
|
||||
def _validate_symbols(cls, v: list[str]) -> list[str]:
|
||||
if not v:
|
||||
raise ValueError("symbols must not be empty")
|
||||
if len(v) > 200:
|
||||
raise ValueError("symbols max 200 per request")
|
||||
return v
|
||||
|
||||
|
||||
class SessionAggregateBatchResponse(BaseModel):
|
||||
items: dict[str, SessionAggregateItem]
|
||||
|
||||
|
||||
class CoverageResponse(BaseModel):
|
||||
source: str
|
||||
symbol: str | None = None
|
||||
earliest: str | None = None
|
||||
latest: str | None = None
|
||||
ingested_count: int
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# A. /headlines — raw rows
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.get(
|
||||
"/headlines",
|
||||
response_model=HeadlinesResponse,
|
||||
summary="Raw multi-source news headlines",
|
||||
description=(
|
||||
"Multi-source raw headline rows. Filter by symbols, time window, and "
|
||||
"source. Sources: `alpaca_benzinga`, `stocktwits`, `finnhub`, `gdelt`."
|
||||
),
|
||||
)
|
||||
@with_cache(
|
||||
namespace="news_v2:headlines",
|
||||
ttl=300,
|
||||
key_params=["symbols", "start", "end", "sources", "limit", "cursor"],
|
||||
)
|
||||
async def get_headlines(
|
||||
response: Response,
|
||||
symbols: Optional[str] = Query(None, description="CSV ticker list, max 50 (e.g. AAPL,MSFT)"),
|
||||
start: Optional[datetime] = Query(None, description="Start time (UTC ISO)"),
|
||||
end: Optional[datetime] = Query(None, description="End time (UTC ISO)"),
|
||||
sources: Optional[str] = Query(None, description=f"CSV source filter, subset of {sorted(_VALID_SOURCES)}"),
|
||||
limit: int = Query(100, ge=1, le=500),
|
||||
cursor: Optional[str] = Query(None, description="published_at_lt cursor (ISO datetime)"),
|
||||
force_refresh: bool = Query(False),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
sym_list = _csv_to_list(symbols, max_items=50, kind="symbols")
|
||||
src_list = _csv_to_list(sources, max_items=10, kind="sources")
|
||||
if src_list:
|
||||
bad = [s for s in src_list if s not in _VALID_SOURCES]
|
||||
if bad:
|
||||
raise HTTPException(400, f"Unknown sources: {bad}")
|
||||
|
||||
cursor_dt = _parse_cursor(cursor)
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(_HEADLINES_SEMAPHORE.acquire(), timeout=_SEMAPHORE_WAIT_TIMEOUT)
|
||||
except asyncio.TimeoutError:
|
||||
raise HTTPException(429, "Server busy — try again later")
|
||||
try:
|
||||
stmt = select(NewsHeadline).order_by(NewsHeadline.published_at.desc()).limit(limit)
|
||||
if sym_list:
|
||||
stmt = stmt.where(NewsHeadline.ticker.in_([s.upper() for s in sym_list]))
|
||||
if start is not None:
|
||||
stmt = stmt.where(NewsHeadline.published_at >= _ensure_utc(start))
|
||||
if end is not None:
|
||||
stmt = stmt.where(NewsHeadline.published_at < _ensure_utc(end))
|
||||
if src_list:
|
||||
stmt = stmt.where(NewsHeadline.source.in_(src_list))
|
||||
if cursor_dt is not None:
|
||||
stmt = stmt.where(NewsHeadline.published_at < cursor_dt)
|
||||
|
||||
result = await db.execute(stmt)
|
||||
rows = result.scalars().all()
|
||||
finally:
|
||||
_HEADLINES_SEMAPHORE.release()
|
||||
|
||||
items = [_row_to_item(r) for r in rows]
|
||||
next_cursor = items[-1].published_at if len(items) == limit else None
|
||||
return HeadlinesResponse(items=items, next_cursor=next_cursor)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# B. /session_aggregate — single ticker
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.get(
|
||||
"/session_aggregate",
|
||||
response_model=SessionAggregateItem,
|
||||
summary="Session-aggregated news for one ticker",
|
||||
)
|
||||
@with_cache(
|
||||
namespace="news_v2:session_agg",
|
||||
ttl=600,
|
||||
key_params=["symbol", "session_date", "window", "sources"],
|
||||
)
|
||||
async def get_session_aggregate(
|
||||
response: Response,
|
||||
symbol: str = Query(..., description="Ticker symbol"),
|
||||
session_date: date = Query(..., description="ET session date (YYYY-MM-DD)"),
|
||||
window: str = Query("premarket", description=f"One of {sorted(_VALID_WINDOWS)}"),
|
||||
sources: Optional[str] = Query(None, description=f"CSV source filter, subset of {sorted(_VALID_SOURCES)}"),
|
||||
force_refresh: bool = Query(False),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
if window not in _VALID_WINDOWS:
|
||||
raise HTTPException(400, f"window must be one of {sorted(_VALID_WINDOWS)}")
|
||||
|
||||
src_list = _csv_to_list(sources, max_items=10, kind="sources")
|
||||
if src_list:
|
||||
bad = [s for s in src_list if s not in _VALID_SOURCES]
|
||||
if bad:
|
||||
raise HTTPException(400, f"Unknown sources: {bad}")
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(_AGG_SEMAPHORE.acquire(), timeout=_SEMAPHORE_WAIT_TIMEOUT)
|
||||
except asyncio.TimeoutError:
|
||||
raise HTTPException(429, "Server busy — try again later")
|
||||
try:
|
||||
try:
|
||||
agg = await aggregate_session(
|
||||
db=db,
|
||||
ticker=symbol,
|
||||
session_date=session_date,
|
||||
window=window, # type: ignore[arg-type]
|
||||
sources=src_list,
|
||||
)
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e))
|
||||
finally:
|
||||
_AGG_SEMAPHORE.release()
|
||||
|
||||
return SessionAggregateItem(**agg.to_dict())
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# C. POST /session_aggregate/batch — many tickers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.post(
|
||||
"/session_aggregate/batch",
|
||||
response_model=SessionAggregateBatchResponse,
|
||||
summary="Session-aggregated news for many tickers in one call",
|
||||
description=(
|
||||
"Batch variant. Caching is intentionally NOT applied at this layer — "
|
||||
"fithia2 maintains a client-side disk cache as the primary defense; "
|
||||
"Oracle absorbs only burst load. Use the GET single endpoint for "
|
||||
"Redis-cached single-ticker reads."
|
||||
),
|
||||
)
|
||||
async def post_session_aggregate_batch(
|
||||
payload: SessionAggregateBatchRequest = Body(...),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
src_list = payload.sources or None
|
||||
if src_list:
|
||||
bad = [s for s in src_list if s not in _VALID_SOURCES]
|
||||
if bad:
|
||||
raise HTTPException(400, f"Unknown sources: {bad}")
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(_AGG_SEMAPHORE.acquire(), timeout=_SEMAPHORE_WAIT_TIMEOUT)
|
||||
except asyncio.TimeoutError:
|
||||
raise HTTPException(429, "Server busy — try again later")
|
||||
try:
|
||||
try:
|
||||
results = await aggregate_session_batch(
|
||||
db=db,
|
||||
tickers=payload.symbols,
|
||||
session_date=payload.session_date,
|
||||
window=payload.window, # type: ignore[arg-type]
|
||||
sources=src_list,
|
||||
)
|
||||
except ValueError as e:
|
||||
raise HTTPException(400, str(e))
|
||||
finally:
|
||||
_AGG_SEMAPHORE.release()
|
||||
|
||||
return SessionAggregateBatchResponse(
|
||||
items={t: SessionAggregateItem(**a.to_dict()) for t, a in results.items()}
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# D. /coverage — per-source ingest depth
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.get(
|
||||
"/coverage",
|
||||
response_model=CoverageResponse,
|
||||
summary="Per-source ingest coverage probe",
|
||||
)
|
||||
@with_cache(
|
||||
namespace="news_v2:coverage",
|
||||
ttl=300,
|
||||
key_params=["source", "symbol"],
|
||||
)
|
||||
async def get_coverage(
|
||||
response: Response,
|
||||
source: str = Query(..., description=f"One of {sorted(_VALID_SOURCES)}"),
|
||||
symbol: Optional[str] = Query(None, description="Optional ticker filter"),
|
||||
force_refresh: bool = Query(False),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
if source not in _VALID_SOURCES:
|
||||
raise HTTPException(400, f"Unknown source: {source}")
|
||||
|
||||
stmt = select(
|
||||
func.min(NewsHeadline.published_at),
|
||||
func.max(NewsHeadline.published_at),
|
||||
func.count(NewsHeadline.id),
|
||||
).where(NewsHeadline.source == source)
|
||||
if symbol:
|
||||
stmt = stmt.where(NewsHeadline.ticker == symbol.strip().upper())
|
||||
|
||||
result = await db.execute(stmt)
|
||||
row = result.one()
|
||||
earliest, latest, count = row
|
||||
|
||||
return CoverageResponse(
|
||||
source=source,
|
||||
symbol=symbol.strip().upper() if symbol else None,
|
||||
earliest=earliest.isoformat() if earliest else None,
|
||||
latest=latest.isoformat() if latest else None,
|
||||
ingested_count=int(count or 0),
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _csv_to_list(s: str | None, *, max_items: int, kind: str) -> list[str]:
|
||||
if not s:
|
||||
return []
|
||||
items = [x.strip() for x in s.split(",") if x.strip()]
|
||||
if len(items) > max_items:
|
||||
raise HTTPException(400, f"{kind} max {max_items} per request")
|
||||
return items
|
||||
|
||||
|
||||
def _parse_cursor(cursor: str | None) -> datetime | None:
|
||||
if not cursor:
|
||||
return None
|
||||
try:
|
||||
if cursor.endswith("Z"):
|
||||
cursor = cursor[:-1] + "+00:00"
|
||||
dt = datetime.fromisoformat(cursor)
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
return dt
|
||||
except Exception:
|
||||
raise HTTPException(400, "cursor must be ISO datetime")
|
||||
|
||||
|
||||
def _ensure_utc(dt: datetime) -> datetime:
|
||||
if dt.tzinfo is None:
|
||||
return dt.replace(tzinfo=timezone.utc)
|
||||
return dt.astimezone(timezone.utc)
|
||||
|
||||
|
||||
def _row_to_item(r: NewsHeadline) -> HeadlineItem:
|
||||
return HeadlineItem(
|
||||
source=r.source,
|
||||
source_id=r.source_id,
|
||||
ticker=r.ticker,
|
||||
tickers_all=list(r.tickers_all) if r.tickers_all else None,
|
||||
published_at=_ensure_utc(r.published_at).isoformat(),
|
||||
headline=r.headline,
|
||||
summary=r.summary,
|
||||
url=r.url,
|
||||
language=r.language,
|
||||
vendor_categories=list(r.vendor_categories) if r.vendor_categories else None,
|
||||
categories=list(r.categories) if r.categories else None,
|
||||
raw_sentiment=r.raw_sentiment,
|
||||
is_primary=bool(r.is_primary),
|
||||
ingested_at=_ensure_utc(r.ingested_at).isoformat(),
|
||||
)
|
||||
@ -0,0 +1,43 @@
|
||||
"""
|
||||
GainerSnapshot — 5-min snapshots of Yahoo Finance day_gainers during market hours.
|
||||
Stored for backtesting: who was a top gainer at each intraday checkpoint.
|
||||
"""
|
||||
from datetime import datetime, timezone
|
||||
import uuid
|
||||
|
||||
from sqlalchemy import Column, String, Float, Integer, BigInteger, Text, Index, UniqueConstraint
|
||||
from sqlalchemy.dialects.postgresql import UUID, TIMESTAMP
|
||||
|
||||
from app.core.database import Base
|
||||
|
||||
|
||||
class GainerSnapshot(Base):
|
||||
__tablename__ = "gainer_snapshots"
|
||||
|
||||
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
snapshot_at = Column(TIMESTAMP(timezone=True), nullable=False)
|
||||
rank = Column(Integer, nullable=False)
|
||||
symbol = Column(String(10), nullable=False)
|
||||
name = Column(Text, nullable=True)
|
||||
exchange = Column(String(20), nullable=True)
|
||||
price = Column(Float, nullable=True)
|
||||
change_percent = Column(Float, nullable=True)
|
||||
volume = Column(BigInteger, nullable=True)
|
||||
avg_volume_3m = Column(BigInteger, nullable=True)
|
||||
market_cap = Column(BigInteger, nullable=True)
|
||||
pe_ratio = Column(Float, nullable=True)
|
||||
forward_pe = Column(Float, nullable=True)
|
||||
eps_ttm = Column(Float, nullable=True)
|
||||
dividend_yield = Column(Float, nullable=True)
|
||||
fifty_two_week_high = Column(Float, nullable=True)
|
||||
fifty_two_week_low = Column(Float, nullable=True)
|
||||
created_at = Column(
|
||||
TIMESTAMP(timezone=True), default=lambda: datetime.now(timezone.utc)
|
||||
)
|
||||
|
||||
__table_args__ = (
|
||||
# One symbol per snapshot timestamp (dedup on retry)
|
||||
UniqueConstraint("snapshot_at", "symbol", name="uq_gainer_snapshot_symbol"),
|
||||
Index("idx_gainer_snapshot_at", "snapshot_at"),
|
||||
Index("idx_gainer_symbol_snapshot", "symbol", "snapshot_at"),
|
||||
)
|
||||
@ -1,210 +0,0 @@
|
||||
"""
|
||||
Pydantic schemas for Overlay API responses
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, Optional
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class OverlayFeatures(BaseModel):
|
||||
headline_burst_z: Optional[float] = None
|
||||
youtube_influence_z: Optional[float] = None
|
||||
wiki_attention_z: Optional[float] = None
|
||||
theme_heat_z: Optional[float] = None
|
||||
crowding_stress_z: Optional[float] = None
|
||||
|
||||
|
||||
class OverlaySourcePresence(BaseModel):
|
||||
yahoo: bool = False
|
||||
youtube: bool = False
|
||||
wikimedia: bool = False
|
||||
google_trends: bool = False
|
||||
finra: bool = False
|
||||
|
||||
|
||||
class YahooSourceDetail(BaseModel):
|
||||
headline_count_6h: int = 0
|
||||
headline_count_24h: int = 0
|
||||
publisher_breadth_24h: int = 0
|
||||
|
||||
|
||||
class YouTubeSourceDetail(BaseModel):
|
||||
mentions_24h: int = 0
|
||||
weighted_views_24h: float = 0.0
|
||||
|
||||
|
||||
class WikiSourceDetail(BaseModel):
|
||||
page_views_1d: Optional[int] = None
|
||||
page_views_7d_avg: Optional[float] = None
|
||||
|
||||
|
||||
class FinraSourceDetail(BaseModel):
|
||||
short_volume_ratio: Optional[float] = None
|
||||
short_volume_spike_zscore: Optional[float] = None
|
||||
|
||||
|
||||
class OverlaySourceDetails(BaseModel):
|
||||
yahoo: Optional[YahooSourceDetail] = None
|
||||
youtube: Optional[YouTubeSourceDetail] = None
|
||||
wikimedia: Optional[WikiSourceDetail] = None
|
||||
finra: Optional[FinraSourceDetail] = None
|
||||
|
||||
|
||||
class OverlayMetadata(BaseModel):
|
||||
feature_version: str = "v1"
|
||||
data_freshness: Optional[datetime] = None
|
||||
next_update_expected: Optional[datetime] = None
|
||||
|
||||
|
||||
class OverlayScoreResponse(BaseModel):
|
||||
symbol: str
|
||||
as_of_ts: Optional[datetime] = None
|
||||
overlay_score: float = 0.0
|
||||
overlay_confidence: float = 0.0
|
||||
overlay_band: Optional[str] = None
|
||||
hold_extension_hint: Optional[str] = None
|
||||
add_on_eligibility: Optional[bool] = None
|
||||
features: OverlayFeatures = Field(default_factory=OverlayFeatures)
|
||||
source_presence: OverlaySourcePresence = Field(default_factory=OverlaySourcePresence)
|
||||
source_details: OverlaySourceDetails = Field(default_factory=OverlaySourceDetails)
|
||||
metadata: OverlayMetadata = Field(default_factory=OverlayMetadata)
|
||||
|
||||
|
||||
class BulkOverlayResponse(BaseModel):
|
||||
results: List[OverlayScoreResponse]
|
||||
total_count: int
|
||||
metadata: Dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class OverlayTopMover(BaseModel):
|
||||
symbol: str
|
||||
overlay_score: float
|
||||
overlay_band: Optional[str] = None
|
||||
as_of_ts: Optional[datetime] = None
|
||||
|
||||
|
||||
class TopMoversResponse(BaseModel):
|
||||
top_movers: List[OverlayTopMover]
|
||||
total_count: int
|
||||
metadata: Dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class HeadlineItem(BaseModel):
|
||||
title: str
|
||||
publisher: Optional[str] = None
|
||||
published_at: datetime
|
||||
article_guid: str
|
||||
|
||||
|
||||
class HeadlinesResponse(BaseModel):
|
||||
symbol: str
|
||||
headlines: List[HeadlineItem]
|
||||
headline_count_6h: int = 0
|
||||
headline_count_24h: int = 0
|
||||
publisher_breadth_24h: int = 0
|
||||
metadata: Dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class VideoItem(BaseModel):
|
||||
video_id: str
|
||||
channel_id: str
|
||||
title: str
|
||||
view_count: int = 0
|
||||
comment_count: int = 0
|
||||
published_at: Optional[datetime] = None
|
||||
channel_weight: float = 0.5
|
||||
|
||||
|
||||
class YouTubeResponse(BaseModel):
|
||||
symbol: str
|
||||
videos: List[VideoItem]
|
||||
mentions_24h: int = 0
|
||||
weighted_views_24h: float = 0.0
|
||||
metadata: Dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class WikiPageviewPoint(BaseModel):
|
||||
date: datetime
|
||||
views: int
|
||||
page_title: str
|
||||
|
||||
|
||||
class WikiResponse(BaseModel):
|
||||
symbol: str
|
||||
pageviews: List[WikiPageviewPoint]
|
||||
views_1d: Optional[int] = None
|
||||
views_7d_avg: Optional[float] = None
|
||||
metadata: Dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class CrowdingResponse(BaseModel):
|
||||
symbol: str
|
||||
short_volume_ratio: Optional[float] = None
|
||||
short_volume_spike_zscore: Optional[float] = None
|
||||
crowding_stress_z: Optional[float] = None
|
||||
metadata: Dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class TrendPoint(BaseModel):
|
||||
observed_at: datetime
|
||||
interest_value: int
|
||||
topic_id: str
|
||||
topic_label: Optional[str] = None
|
||||
|
||||
|
||||
class TrendsResponse(BaseModel):
|
||||
symbol: str
|
||||
trends: List[TrendPoint]
|
||||
theme_heat_z: Optional[float] = None
|
||||
metadata: Dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class OverlayHistoryPoint(BaseModel):
|
||||
as_of_ts: datetime
|
||||
overlay_score: float
|
||||
overlay_confidence: float
|
||||
overlay_band: Optional[str] = None
|
||||
|
||||
|
||||
class OverlayHistoryResponse(BaseModel):
|
||||
symbol: str
|
||||
history: List[OverlayHistoryPoint]
|
||||
metadata: Dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class SourceHealthItem(BaseModel):
|
||||
source: str
|
||||
last_collected_at: Optional[datetime] = None
|
||||
status: str = "unknown"
|
||||
success_rate_24h: Optional[float] = None
|
||||
records_24h: int = 0
|
||||
|
||||
|
||||
class AdminHealthResponse(BaseModel):
|
||||
overlay_enabled: bool
|
||||
sources: List[SourceHealthItem]
|
||||
last_pipeline_run: Optional[datetime] = None
|
||||
metadata: Dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class TriggerPipelineResponse(BaseModel):
|
||||
status: str
|
||||
message: str
|
||||
job_ids: List[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class JobLogEntry(BaseModel):
|
||||
id: str
|
||||
job_type: str
|
||||
status: str
|
||||
started_at: datetime
|
||||
completed_at: Optional[datetime] = None
|
||||
records_processed: int = 0
|
||||
error_message: Optional[str] = None
|
||||
|
||||
|
||||
class JobLogResponse(BaseModel):
|
||||
logs: List[JobLogEntry]
|
||||
total_count: int
|
||||
metadata: Dict[str, Any] = Field(default_factory=dict)
|
||||
@ -0,0 +1,59 @@
|
||||
"""
|
||||
Gainer snapshot collector — fetches top 100 day_gainers and stores per 5-min slot.
|
||||
"""
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _floor_to_5min(dt: datetime) -> datetime:
|
||||
"""Round down to the nearest 5-minute boundary."""
|
||||
return dt.replace(minute=dt.minute - (dt.minute % 5), second=0, microsecond=0)
|
||||
|
||||
|
||||
async def collect_gainer_snapshot() -> int:
|
||||
"""Fetch 100 day_gainers and bulk-insert into gainer_snapshots. Returns inserted count."""
|
||||
from app.services.screener_service import screener_service
|
||||
from app.core.database import AsyncSessionLocal
|
||||
from app.models.gainer_snapshot import GainerSnapshot
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
|
||||
snapshot_at = _floor_to_5min(datetime.now(timezone.utc))
|
||||
|
||||
result = await screener_service.screen_preset("day_gainers", page=1, page_size=100)
|
||||
stocks = result.get("stocks", [])
|
||||
if not stocks:
|
||||
logger.warning("[Gainers] No stocks returned from day_gainers preset")
|
||||
return 0
|
||||
|
||||
rows = [
|
||||
{
|
||||
"snapshot_at": snapshot_at,
|
||||
"rank": rank,
|
||||
"symbol": s["symbol"],
|
||||
"name": s.get("name"),
|
||||
"exchange": s.get("exchange"),
|
||||
"price": s.get("price"),
|
||||
"change_percent": s.get("change_percent"),
|
||||
"volume": s.get("volume"),
|
||||
"avg_volume_3m": s.get("avg_volume_3m"),
|
||||
"market_cap": s.get("market_cap"),
|
||||
"pe_ratio": s.get("pe_ratio"),
|
||||
"forward_pe": s.get("forward_pe"),
|
||||
"eps_ttm": s.get("eps_ttm"),
|
||||
"dividend_yield": s.get("dividend_yield"),
|
||||
"fifty_two_week_high": s.get("fifty_two_week_high"),
|
||||
"fifty_two_week_low": s.get("fifty_two_week_low"),
|
||||
}
|
||||
for rank, s in enumerate(stocks, start=1)
|
||||
]
|
||||
|
||||
async with AsyncSessionLocal() as db:
|
||||
stmt = pg_insert(GainerSnapshot).values(rows)
|
||||
stmt = stmt.on_conflict_do_nothing(constraint="uq_gainer_snapshot_symbol")
|
||||
await db.execute(stmt)
|
||||
await db.commit()
|
||||
|
||||
logger.info("[Gainers] snapshot %s — %d rows", snapshot_at.isoformat(), len(rows))
|
||||
return len(rows)
|
||||
@ -0,0 +1,208 @@
|
||||
"""
|
||||
Alpaca News (Benzinga backend) REST client.
|
||||
|
||||
Endpoint: GET /v1beta1/news on data.alpaca.markets
|
||||
Auth: same APCA-API-KEY-ID/APCA-API-SECRET-KEY as market data.
|
||||
Free-tier history: ~30 days. Cumulative archive must be self-built via daily
|
||||
ingest.
|
||||
|
||||
Rate limit handled via existing per-process token-bucket pattern (200 req/min
|
||||
shared with the market-data client where this matters; News calls are far
|
||||
less frequent so a small private bucket is sufficient).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, AsyncIterator
|
||||
|
||||
import httpx
|
||||
|
||||
from app.core.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class AlpacaNewsArticle:
|
||||
id: int
|
||||
headline: str
|
||||
summary: str | None
|
||||
url: str | None
|
||||
author: str | None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
source: str | None # vendor source tag (e.g. "benzinga")
|
||||
symbols: list[str]
|
||||
images: list[dict[str, Any]]
|
||||
content: str | None
|
||||
|
||||
@classmethod
|
||||
def from_payload(cls, payload: dict[str, Any]) -> "AlpacaNewsArticle":
|
||||
return cls(
|
||||
id=int(payload["id"]),
|
||||
headline=payload.get("headline") or "",
|
||||
summary=payload.get("summary"),
|
||||
url=payload.get("url"),
|
||||
author=payload.get("author"),
|
||||
created_at=_parse_iso(payload["created_at"]),
|
||||
updated_at=_parse_iso(payload.get("updated_at") or payload["created_at"]),
|
||||
source=payload.get("source"),
|
||||
symbols=list(payload.get("symbols") or []),
|
||||
images=list(payload.get("images") or []),
|
||||
content=payload.get("content"),
|
||||
)
|
||||
|
||||
|
||||
def _parse_iso(s: str) -> datetime:
|
||||
"""Parse Alpaca timestamps (RFC 3339, may end in 'Z')."""
|
||||
if s.endswith("Z"):
|
||||
s = s[:-1] + "+00:00"
|
||||
dt = datetime.fromisoformat(s)
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
return dt
|
||||
|
||||
|
||||
class AlpacaNewsClient:
|
||||
"""Alpaca News API client. Reuses ALPACA_API_KEY/SECRET."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
api_key: str | None = None,
|
||||
secret_key: str | None = None,
|
||||
base_url: str | None = None,
|
||||
max_requests_per_min: int = 60,
|
||||
):
|
||||
self.api_key = api_key or settings.ALPACA_API_KEY
|
||||
self.secret_key = secret_key or settings.ALPACA_SECRET_KEY
|
||||
self.base_url = (base_url or settings.ALPACA_NEWS_BASE_URL).rstrip("/")
|
||||
self._max_rpm = max_requests_per_min
|
||||
self._request_times: list[float] = []
|
||||
self._client: httpx.AsyncClient | None = None
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
def is_configured(self) -> bool:
|
||||
return bool(self.api_key) and bool(self.secret_key)
|
||||
|
||||
async def _get_client(self) -> httpx.AsyncClient:
|
||||
if self._client is None or self._client.is_closed:
|
||||
self._client = httpx.AsyncClient(
|
||||
base_url=self.base_url,
|
||||
headers={
|
||||
"APCA-API-KEY-ID": self.api_key,
|
||||
"APCA-API-SECRET-KEY": self.secret_key,
|
||||
},
|
||||
timeout=30.0,
|
||||
)
|
||||
return self._client
|
||||
|
||||
async def close(self) -> None:
|
||||
if self._client and not self._client.is_closed:
|
||||
await self._client.aclose()
|
||||
|
||||
async def _wait_for_rate_limit(self) -> None:
|
||||
async with self._lock:
|
||||
now = time.monotonic()
|
||||
self._request_times = [t for t in self._request_times if now - t < 60]
|
||||
if len(self._request_times) >= self._max_rpm:
|
||||
sleep_for = 60 - (now - self._request_times[0]) + 0.1
|
||||
if sleep_for > 0:
|
||||
logger.debug(f"AlpacaNews rate limit reached, sleeping {sleep_for:.1f}s")
|
||||
await asyncio.sleep(sleep_for)
|
||||
self._request_times.append(time.monotonic())
|
||||
|
||||
async def _request(
|
||||
self,
|
||||
path: str,
|
||||
params: dict[str, Any] | None = None,
|
||||
retries: int = 3,
|
||||
) -> dict[str, Any]:
|
||||
await self._wait_for_rate_limit()
|
||||
client = await self._get_client()
|
||||
|
||||
last_exc: Exception | None = None
|
||||
for attempt in range(retries):
|
||||
try:
|
||||
resp = await client.get(path, params=params)
|
||||
if resp.status_code == 429:
|
||||
wait = 2 ** attempt
|
||||
logger.warning(f"AlpacaNews 429 — retrying in {wait}s (attempt {attempt + 1})")
|
||||
await asyncio.sleep(wait)
|
||||
continue
|
||||
if resp.status_code >= 500:
|
||||
wait = 2 ** attempt
|
||||
logger.warning(f"AlpacaNews {resp.status_code} — retrying in {wait}s (attempt {attempt + 1})")
|
||||
await asyncio.sleep(wait)
|
||||
continue
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
except httpx.HTTPStatusError as exc:
|
||||
last_exc = exc
|
||||
if attempt < retries - 1 and exc.response.status_code in (429, 500, 502, 503, 504):
|
||||
await asyncio.sleep(2 ** attempt)
|
||||
continue
|
||||
raise
|
||||
except (httpx.ConnectError, httpx.ReadTimeout) as exc:
|
||||
last_exc = exc
|
||||
if attempt < retries - 1:
|
||||
await asyncio.sleep(2 ** attempt)
|
||||
continue
|
||||
raise
|
||||
raise last_exc # type: ignore[misc]
|
||||
|
||||
async def fetch_news(
|
||||
self,
|
||||
symbols: list[str] | None = None,
|
||||
start: datetime | None = None,
|
||||
end: datetime | None = None,
|
||||
include_content: bool = True,
|
||||
sort: str = "desc",
|
||||
page_limit: int = 50,
|
||||
) -> AsyncIterator[AlpacaNewsArticle]:
|
||||
"""
|
||||
Yield articles for the given symbols (or the whole stream if None) over
|
||||
[start, end]. Handles pagination via `page_token`.
|
||||
|
||||
Alpaca page_size max = 50.
|
||||
"""
|
||||
params: dict[str, Any] = {
|
||||
"limit": min(page_limit, 50),
|
||||
"sort": sort,
|
||||
"include_content": str(include_content).lower(),
|
||||
}
|
||||
if symbols:
|
||||
params["symbols"] = ",".join(s.strip().upper() for s in symbols if s)
|
||||
if start is not None:
|
||||
params["start"] = _to_rfc3339(start)
|
||||
if end is not None:
|
||||
params["end"] = _to_rfc3339(end)
|
||||
|
||||
page_token: str | None = None
|
||||
while True:
|
||||
if page_token:
|
||||
params["page_token"] = page_token
|
||||
else:
|
||||
params.pop("page_token", None)
|
||||
|
||||
payload = await self._request("/v1beta1/news", params=params)
|
||||
for raw in payload.get("news", []) or []:
|
||||
try:
|
||||
yield AlpacaNewsArticle.from_payload(raw)
|
||||
except Exception as e:
|
||||
logger.warning(f"AlpacaNews payload parse failed: {e} — {raw.get('id')}")
|
||||
|
||||
page_token = payload.get("next_page_token")
|
||||
if not page_token:
|
||||
return
|
||||
|
||||
|
||||
def _to_rfc3339(dt: datetime) -> str:
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
# Alpaca accepts RFC-3339 with "Z" or "+00:00"
|
||||
return dt.astimezone(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
@ -0,0 +1,190 @@
|
||||
"""
|
||||
Vendor news category → unified taxonomy normalizer.
|
||||
|
||||
Each vendor uses its own categorization scheme (Alpaca/Benzinga channels,
|
||||
Finnhub category codes, StockTwits sentiment-only). We map them all to a
|
||||
single 22-term vocabulary so downstream consumers (fithia2 ORB scoring) can
|
||||
treat sources interchangeably. Vendor-original categories are preserved on
|
||||
NewsHeadline.vendor_categories for future re-classification.
|
||||
|
||||
Headline-keyword regex passes provide override hints for category nuance the
|
||||
vendor labels can't express (e.g. Alpaca tags FDA news as "FDA" but doesn't
|
||||
distinguish approval vs. rejection).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Iterable
|
||||
|
||||
UNIFIED_CATEGORIES: frozenset[str] = frozenset({
|
||||
"analyst_rating_upgrade",
|
||||
"analyst_rating_downgrade",
|
||||
"analyst_rating_initiate",
|
||||
"earnings_release",
|
||||
"earnings_preannouncement",
|
||||
"guidance_update",
|
||||
"m_and_a",
|
||||
"partnership",
|
||||
"contract_award",
|
||||
"fda_approval",
|
||||
"fda_rejection",
|
||||
"clinical_trial",
|
||||
"litigation",
|
||||
"regulatory_action",
|
||||
"sec_filing",
|
||||
"insider_trading",
|
||||
"secondary_offering",
|
||||
"buyback",
|
||||
"management_change",
|
||||
"restructuring",
|
||||
"general",
|
||||
})
|
||||
|
||||
|
||||
# Alpaca News (Benzinga backend) channel taxonomy. Source:
|
||||
# https://docs.alpaca.markets/reference/news-3
|
||||
ALPACA_BENZINGA_MAP: dict[str, str] = {
|
||||
"Analyst Color": "general",
|
||||
"Upgrades": "analyst_rating_upgrade",
|
||||
"Downgrades": "analyst_rating_downgrade",
|
||||
"Price Target": "general",
|
||||
"Reiteration": "analyst_rating_initiate",
|
||||
"Initiation": "analyst_rating_initiate",
|
||||
"Earnings": "earnings_release",
|
||||
"Earnings Beats": "earnings_release",
|
||||
"Earnings Misses": "earnings_release",
|
||||
"Guidance": "guidance_update",
|
||||
"M&A": "m_and_a",
|
||||
"Mergers": "m_and_a",
|
||||
"Acquisitions": "m_and_a",
|
||||
"Partnership": "partnership",
|
||||
"Contract": "contract_award",
|
||||
"FDA": "fda_approval", # rejection split via headline regex below
|
||||
"Clinical Trials": "clinical_trial",
|
||||
"Lawsuit": "litigation",
|
||||
"Litigation": "litigation",
|
||||
"SEC Filings": "sec_filing",
|
||||
"Insider Trades": "insider_trading",
|
||||
"Offerings": "secondary_offering",
|
||||
"Buybacks": "buyback",
|
||||
"Management": "management_change",
|
||||
"Restructuring": "restructuring",
|
||||
"Spinoff": "restructuring",
|
||||
"Bankruptcy": "restructuring",
|
||||
"Press Releases": "general",
|
||||
"News": "general",
|
||||
}
|
||||
|
||||
# Finnhub /company-news category field. Source:
|
||||
# https://finnhub.io/docs/api/company-news
|
||||
FINNHUB_MAP: dict[str, str] = {
|
||||
"company": "general",
|
||||
"general": "general",
|
||||
"earnings": "earnings_release",
|
||||
"merger": "m_and_a",
|
||||
"ipo": "secondary_offering",
|
||||
"guidance": "guidance_update",
|
||||
"rating": "general", # split direction via headline regex
|
||||
"regulation": "regulatory_action",
|
||||
"press release": "general",
|
||||
}
|
||||
|
||||
# StockTwits messages have no vendor category — only sentiment tags.
|
||||
STOCKTWITS_DEFAULT: str = "general"
|
||||
|
||||
|
||||
# Headline regex overrides — applied AFTER vendor map. Matches earlier in this
|
||||
# list win when multiple match. Each tuple is (compiled regex, target category).
|
||||
_HEADLINE_OVERRIDES: list[tuple[re.Pattern[str], str]] = [
|
||||
# Analyst rating direction (more specific than vendor map)
|
||||
(re.compile(r"\b(downgrade[ds]?|cuts?\s+(price\s+target|rating)|lowers?\s+rating)\b", re.I),
|
||||
"analyst_rating_downgrade"),
|
||||
(re.compile(r"\b(upgrade[ds]?|raises?\s+(price\s+target|rating)|boosts?\s+rating)\b", re.I),
|
||||
"analyst_rating_upgrade"),
|
||||
(re.compile(r"\binitiates?\s+(coverage|with)\b", re.I), "analyst_rating_initiate"),
|
||||
# FDA outcome direction
|
||||
(re.compile(r"\bFDA\s+(approves?|approval|grants?\s+approval|clears?)\b", re.I),
|
||||
"fda_approval"),
|
||||
(re.compile(r"\bFDA\s+(rejects?|rejection|denies?|complete\s+response\s+letter|CRL)\b", re.I),
|
||||
"fda_rejection"),
|
||||
# Pre-announcements / guidance distinction
|
||||
(re.compile(r"\b(pre[\-\s]?announce[sd]?|preliminary\s+results?)\b", re.I),
|
||||
"earnings_preannouncement"),
|
||||
(re.compile(r"\b(guidance|outlook|forecast)s?\b.*\b(raises?|lifts?|cuts?|lowers?|reaffirm[s]?)\b", re.I),
|
||||
"guidance_update"),
|
||||
# M&A specifics
|
||||
(re.compile(r"\b(acquires?|to\s+acquire|merger\s+with|takeover)\b", re.I), "m_and_a"),
|
||||
# Buybacks
|
||||
(re.compile(r"\b(share\s+repurchase|buyback|stock\s+repurchase)\b", re.I), "buyback"),
|
||||
# Offerings
|
||||
(re.compile(r"\b(secondary\s+offering|public\s+offering|equity\s+offering|prices?\s+offering)\b", re.I),
|
||||
"secondary_offering"),
|
||||
# Mgmt change
|
||||
(re.compile(r"\b(CEO|CFO|COO|CTO|chair(man)?|president)\b.*\b(steps?\s+down|resigns?|appointed?|named)\b", re.I),
|
||||
"management_change"),
|
||||
# Restructuring
|
||||
(re.compile(r"\b(layoffs?|restructur(ing|e)|chapter\s+11|bankruptcy|spinoff)\b", re.I), "restructuring"),
|
||||
# Litigation
|
||||
(re.compile(r"\b(lawsuit|sued|class\s+action|settles?\s+(suit|claim))\b", re.I), "litigation"),
|
||||
# Contract awards
|
||||
(re.compile(r"\b(awarded\s+contract|wins?\s+contract|secures?\s+contract)\b", re.I), "contract_award"),
|
||||
# Partnership
|
||||
(re.compile(r"\b(partner(ship|s\s+with)|collaborates?\s+with|joint\s+venture)\b", re.I), "partnership"),
|
||||
]
|
||||
|
||||
|
||||
def normalize_alpaca(channels: Iterable[str] | None, headline: str) -> list[str]:
|
||||
"""Map Alpaca/Benzinga channels + headline keywords → unified categories."""
|
||||
cats = _from_vendor_map(channels, ALPACA_BENZINGA_MAP)
|
||||
cats.update(_from_headline(headline))
|
||||
return _finalize(cats)
|
||||
|
||||
|
||||
def normalize_finnhub(category: str | None, headline: str) -> list[str]:
|
||||
"""Map Finnhub category + headline keywords → unified categories."""
|
||||
cats: set[str] = set()
|
||||
if category:
|
||||
mapped = FINNHUB_MAP.get(category.strip().lower())
|
||||
if mapped:
|
||||
cats.add(mapped)
|
||||
cats.update(_from_headline(headline))
|
||||
return _finalize(cats)
|
||||
|
||||
|
||||
def normalize_stocktwits(headline: str) -> list[str]:
|
||||
"""StockTwits has no category — only headline regex applies."""
|
||||
cats = _from_headline(headline)
|
||||
return _finalize(cats)
|
||||
|
||||
|
||||
def _from_vendor_map(channels: Iterable[str] | None, mapping: dict[str, str]) -> set[str]:
|
||||
if not channels:
|
||||
return set()
|
||||
out: set[str] = set()
|
||||
for ch in channels:
|
||||
if not ch:
|
||||
continue
|
||||
mapped = mapping.get(ch.strip())
|
||||
if mapped:
|
||||
out.add(mapped)
|
||||
return out
|
||||
|
||||
|
||||
def _from_headline(headline: str) -> set[str]:
|
||||
if not headline:
|
||||
return set()
|
||||
out: set[str] = set()
|
||||
for pattern, cat in _HEADLINE_OVERRIDES:
|
||||
if pattern.search(headline):
|
||||
out.add(cat)
|
||||
return out
|
||||
|
||||
|
||||
def _finalize(cats: set[str]) -> list[str]:
|
||||
if not cats:
|
||||
return ["general"]
|
||||
# Drop "general" if a specific category was matched.
|
||||
if len(cats) > 1:
|
||||
cats.discard("general")
|
||||
return sorted(cats)
|
||||
@ -0,0 +1,111 @@
|
||||
"""
|
||||
Finnhub /company-news client (free tier).
|
||||
|
||||
Rate limit: 60 calls/min on the free plan. We enforce 1 second between calls
|
||||
via an asyncio.Semaphore + sleep, which is the simplest correct shape since
|
||||
the entire ingest pipeline is single-process.
|
||||
|
||||
Endpoint shape: /company-news?symbol=AAPL&from=2026-04-01&to=2026-04-25
|
||||
Free-tier history: ~12 months. Cumulative archive must be self-built by
|
||||
running `scripts/news_backfill.py` once at install time, then daily.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import date
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from app.core.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class FinnhubClient:
|
||||
def __init__(
|
||||
self,
|
||||
api_key: str | None = None,
|
||||
base_url: str | None = None,
|
||||
request_interval_sec: float = 1.05,
|
||||
):
|
||||
self.api_key = api_key or settings.FINNHUB_API_KEY
|
||||
self.base_url = (base_url or settings.FINNHUB_BASE_URL).rstrip("/")
|
||||
self._client: httpx.AsyncClient | None = None
|
||||
# Single-slot semaphore + sleep enforces 60-calls/min ceiling
|
||||
self._gate = asyncio.Semaphore(1)
|
||||
self._interval = request_interval_sec
|
||||
|
||||
def is_configured(self) -> bool:
|
||||
return bool(self.api_key)
|
||||
|
||||
async def _get_client(self) -> httpx.AsyncClient:
|
||||
if self._client is None or self._client.is_closed:
|
||||
self._client = httpx.AsyncClient(base_url=self.base_url, timeout=30.0)
|
||||
return self._client
|
||||
|
||||
async def close(self) -> None:
|
||||
if self._client and not self._client.is_closed:
|
||||
await self._client.aclose()
|
||||
|
||||
async def _request(self, path: str, params: dict[str, Any], retries: int = 3) -> Any:
|
||||
client = await self._get_client()
|
||||
params = {**params, "token": self.api_key}
|
||||
|
||||
async with self._gate:
|
||||
last_exc: Exception | None = None
|
||||
for attempt in range(retries):
|
||||
try:
|
||||
resp = await client.get(path, params=params)
|
||||
if resp.status_code == 429:
|
||||
wait = 2 ** attempt
|
||||
logger.warning(f"Finnhub 429 — sleeping {wait}s (attempt {attempt + 1})")
|
||||
await asyncio.sleep(wait)
|
||||
continue
|
||||
if resp.status_code >= 500:
|
||||
wait = 2 ** attempt
|
||||
logger.warning(f"Finnhub {resp.status_code} — retry in {wait}s")
|
||||
await asyncio.sleep(wait)
|
||||
continue
|
||||
resp.raise_for_status()
|
||||
await asyncio.sleep(self._interval)
|
||||
return resp.json()
|
||||
except httpx.HTTPStatusError as exc:
|
||||
last_exc = exc
|
||||
if attempt < retries - 1 and exc.response.status_code in (429, 500, 502, 503, 504):
|
||||
await asyncio.sleep(2 ** attempt)
|
||||
continue
|
||||
raise
|
||||
except (httpx.ConnectError, httpx.ReadTimeout) as exc:
|
||||
last_exc = exc
|
||||
if attempt < retries - 1:
|
||||
await asyncio.sleep(2 ** attempt)
|
||||
continue
|
||||
raise
|
||||
raise last_exc # type: ignore[misc]
|
||||
|
||||
async def fetch_company_news(
|
||||
self,
|
||||
symbol: str,
|
||||
from_date: date,
|
||||
to_date: date,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Fetch company-news for [from_date, to_date] inclusive.
|
||||
|
||||
One call covers the full range, so split by month at the caller side
|
||||
for backfill granularity.
|
||||
"""
|
||||
payload = await self._request(
|
||||
"/company-news",
|
||||
params={
|
||||
"symbol": symbol.strip().upper(),
|
||||
"from": from_date.isoformat(),
|
||||
"to": to_date.isoformat(),
|
||||
},
|
||||
)
|
||||
if not isinstance(payload, list):
|
||||
logger.warning(f"Finnhub /company-news returned non-list for {symbol}")
|
||||
return []
|
||||
return payload
|
||||
@ -0,0 +1,222 @@
|
||||
"""
|
||||
Multi-source headline ingest — vendor payload → NewsHeadline rows.
|
||||
|
||||
For each article, one DB row per (source, source_id, ticker) is emitted so
|
||||
ticker-indexed queries are O(log n). The full vendor symbol set is preserved
|
||||
on `tickers_all`.
|
||||
|
||||
Dedup is enforced by `uq_news_headline_source_ticker` + ON CONFLICT DO NOTHING,
|
||||
making re-ingest idempotent (daily backfill rerunning over an overlapping
|
||||
window is safe).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from typing import Iterable
|
||||
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.database import AsyncSessionLocal
|
||||
from app.models.news_headline import NewsHeadline
|
||||
from app.services.news.alpaca_news_client import AlpacaNewsArticle
|
||||
from app.services.news.category_normalizer import (
|
||||
normalize_alpaca,
|
||||
normalize_finnhub,
|
||||
normalize_stocktwits,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# NewsHeadline has ~17 insertable columns. asyncpg param ceiling 32,767.
|
||||
# 32767 // 17 ≈ 1927; round down for safety + future column adds.
|
||||
_CHUNK = 1500
|
||||
|
||||
|
||||
async def insert_headline_rows(rows: list[dict], db: AsyncSession | None = None) -> int:
|
||||
"""Insert pre-built rows with chunked ON CONFLICT DO NOTHING.
|
||||
|
||||
If `db` is provided, it's used directly and the caller commits.
|
||||
Otherwise a short-lived session is opened and committed here.
|
||||
"""
|
||||
if not rows:
|
||||
return 0
|
||||
|
||||
if db is not None:
|
||||
return await _insert_chunked(db, rows)
|
||||
|
||||
async with AsyncSessionLocal() as sess:
|
||||
inserted = await _insert_chunked(sess, rows)
|
||||
await sess.commit()
|
||||
return inserted
|
||||
|
||||
|
||||
async def _insert_chunked(sess: AsyncSession, rows: list[dict]) -> int:
|
||||
inserted = 0
|
||||
for i in range(0, len(rows), _CHUNK):
|
||||
stmt = pg_insert(NewsHeadline).values(rows[i : i + _CHUNK])
|
||||
stmt = stmt.on_conflict_do_nothing(constraint="uq_news_headline_source_ticker")
|
||||
result = await sess.execute(stmt)
|
||||
inserted += result.rowcount or 0
|
||||
await asyncio.sleep(0)
|
||||
return inserted
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Vendor → row builders
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def alpaca_articles_to_rows(
|
||||
articles: Iterable[AlpacaNewsArticle],
|
||||
ingested_at: datetime | None = None,
|
||||
) -> list[dict]:
|
||||
"""Expand each Alpaca article into one row per ticker."""
|
||||
now = ingested_at or datetime.now(timezone.utc)
|
||||
out: list[dict] = []
|
||||
for art in articles:
|
||||
if not art.symbols:
|
||||
continue
|
||||
vendor_cats = [art.source] if art.source else []
|
||||
unified = normalize_alpaca(vendor_cats, art.headline)
|
||||
# Alpaca's first symbol is conventionally primary
|
||||
primary = art.symbols[0].upper()
|
||||
for sym in art.symbols:
|
||||
sym_u = sym.strip().upper()
|
||||
if not sym_u:
|
||||
continue
|
||||
out.append({
|
||||
"source": "alpaca_benzinga",
|
||||
"source_id": str(art.id),
|
||||
"ticker": sym_u,
|
||||
"tickers_all": [s.upper() for s in art.symbols if s],
|
||||
"published_at": art.created_at,
|
||||
"headline": art.headline,
|
||||
"summary": art.summary,
|
||||
"url": art.url,
|
||||
"language": "en",
|
||||
"vendor_categories": vendor_cats,
|
||||
"categories": unified,
|
||||
"raw_sentiment": None, # Alpaca free tier does not include sentiment
|
||||
"is_primary": sym_u == primary,
|
||||
"ingested_at": now,
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def finnhub_articles_to_rows(
|
||||
ticker: str,
|
||||
articles: Iterable[dict],
|
||||
ingested_at: datetime | None = None,
|
||||
) -> list[dict]:
|
||||
"""Convert Finnhub /company-news payloads. Single-ticker per article."""
|
||||
now = ingested_at or datetime.now(timezone.utc)
|
||||
out: list[dict] = []
|
||||
sym = ticker.strip().upper()
|
||||
for art in articles:
|
||||
if not art.get("id"):
|
||||
continue
|
||||
ts = art.get("datetime")
|
||||
if ts is None:
|
||||
continue
|
||||
try:
|
||||
published_at = datetime.fromtimestamp(int(ts), tz=timezone.utc)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
headline = art.get("headline") or ""
|
||||
category = art.get("category")
|
||||
unified = normalize_finnhub(category, headline)
|
||||
out.append({
|
||||
"source": "finnhub",
|
||||
"source_id": str(art["id"]),
|
||||
"ticker": sym,
|
||||
"tickers_all": [sym],
|
||||
"published_at": published_at,
|
||||
"headline": headline,
|
||||
"summary": art.get("summary"),
|
||||
"url": art.get("url"),
|
||||
"language": "en",
|
||||
"vendor_categories": [category] if category else [],
|
||||
"categories": unified,
|
||||
"raw_sentiment": None,
|
||||
"is_primary": True,
|
||||
"ingested_at": now,
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def stocktwits_messages_to_rows(
|
||||
ticker: str,
|
||||
messages: Iterable[dict],
|
||||
ingested_at: datetime | None = None,
|
||||
) -> list[dict]:
|
||||
"""
|
||||
Convert StockTwits stream messages (streams/symbol/{ticker}.json).
|
||||
|
||||
Sentiment tag is the message's entities.sentiment.basic in {"Bullish", "Bearish"}.
|
||||
Mapped to raw_sentiment ∈ {+1, -1}; absent tag → NULL (neutral).
|
||||
"""
|
||||
now = ingested_at or datetime.now(timezone.utc)
|
||||
out: list[dict] = []
|
||||
sym = ticker.strip().upper()
|
||||
for msg in messages:
|
||||
if not msg.get("id"):
|
||||
continue
|
||||
ts = msg.get("created_at")
|
||||
if not ts:
|
||||
continue
|
||||
try:
|
||||
published_at = _parse_st_iso(ts)
|
||||
except Exception:
|
||||
continue
|
||||
body = (msg.get("body") or "").strip()
|
||||
if not body:
|
||||
continue
|
||||
sentiment = _extract_st_sentiment(msg)
|
||||
unified = normalize_stocktwits(body)
|
||||
# Collect all symbol tags for tickers_all
|
||||
all_syms = [s.get("symbol", "").upper() for s in (msg.get("symbols") or []) if s.get("symbol")]
|
||||
if sym not in all_syms:
|
||||
all_syms.insert(0, sym)
|
||||
out.append({
|
||||
"source": "stocktwits",
|
||||
"source_id": str(msg["id"]),
|
||||
"ticker": sym,
|
||||
"tickers_all": all_syms,
|
||||
"published_at": published_at,
|
||||
"headline": body[:300], # truncate for headline; full body in summary
|
||||
"summary": body,
|
||||
"url": f"https://stocktwits.com/message/{msg['id']}",
|
||||
"language": "en",
|
||||
"vendor_categories": [sentiment] if sentiment else [],
|
||||
"categories": unified,
|
||||
"raw_sentiment": _sentiment_to_score(sentiment),
|
||||
"is_primary": all_syms[0] == sym if all_syms else True,
|
||||
"ingested_at": now,
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def _parse_st_iso(s: str) -> datetime:
|
||||
if s.endswith("Z"):
|
||||
s = s[:-1] + "+00:00"
|
||||
dt = datetime.fromisoformat(s)
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
return dt
|
||||
|
||||
|
||||
def _extract_st_sentiment(msg: dict) -> str | None:
|
||||
entities = msg.get("entities") or {}
|
||||
sentiment = entities.get("sentiment") or {}
|
||||
return sentiment.get("basic") # "Bullish" | "Bearish" | None
|
||||
|
||||
|
||||
def _sentiment_to_score(tag: str | None) -> float | None:
|
||||
if tag == "Bullish":
|
||||
return 1.0
|
||||
if tag == "Bearish":
|
||||
return -1.0
|
||||
return None
|
||||
@ -0,0 +1,310 @@
|
||||
"""
|
||||
APScheduler entry point for News v2 ingest.
|
||||
|
||||
Jobs:
|
||||
alpaca_news_realtime_poll every 5 min: latest Alpaca News stream
|
||||
alpaca_news_daily_backfill 04:30 ET: prior 26h Alpaca News refill
|
||||
stocktwits_universe_refresh 09:00 ET: rebuild dynamic poll list
|
||||
stocktwits_poll every 5 min: per-ticker stream pull
|
||||
finnhub_daily_backfill 05:00 ET: prior day Finnhub /company-news
|
||||
|
||||
Activation gate:
|
||||
NEWS_INGEST_ENABLED must be true. Per-source key presence determines which
|
||||
jobs are registered — Alpaca needs ALPACA_API_KEY/SECRET, Finnhub needs
|
||||
FINNHUB_API_KEY, StockTwits needs nothing. If a source's key is missing,
|
||||
its jobs are skipped with a WARNING. If NO sources are configured the
|
||||
scheduler aborts (fail-fast) so the misconfiguration is loud.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from app.core.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_scheduler = None
|
||||
|
||||
|
||||
def _get_scheduler():
|
||||
global _scheduler
|
||||
if _scheduler is None:
|
||||
try:
|
||||
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
||||
_scheduler = AsyncIOScheduler(timezone=settings.NEWS_INGEST_TIMEZONE)
|
||||
except ImportError:
|
||||
logger.warning("apscheduler not installed; news ingest scheduling disabled")
|
||||
return None
|
||||
return _scheduler
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Job bodies
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def _run_alpaca_realtime_poll() -> None:
|
||||
"""Poll the latest 5 minutes of Alpaca News (no symbol filter)."""
|
||||
from app.services.news.alpaca_news_client import AlpacaNewsClient
|
||||
from app.services.news.headline_ingest_service import (
|
||||
alpaca_articles_to_rows,
|
||||
insert_headline_rows,
|
||||
)
|
||||
|
||||
client = AlpacaNewsClient()
|
||||
if not client.is_configured():
|
||||
return
|
||||
|
||||
end = datetime.now(timezone.utc)
|
||||
start = end - timedelta(minutes=10) # 5-min poll + 5-min overlap for safety
|
||||
|
||||
try:
|
||||
articles = []
|
||||
async for art in client.fetch_news(symbols=None, start=start, end=end, page_limit=50):
|
||||
articles.append(art)
|
||||
rows = alpaca_articles_to_rows(articles)
|
||||
if rows:
|
||||
inserted = await insert_headline_rows(rows)
|
||||
if inserted:
|
||||
logger.info(f"[News] Alpaca realtime: +{inserted} new ({len(articles)} articles)")
|
||||
except Exception as e:
|
||||
logger.error(f"[News] Alpaca realtime poll failed: {e}")
|
||||
finally:
|
||||
await client.close()
|
||||
|
||||
|
||||
async def _run_alpaca_daily_backfill() -> None:
|
||||
"""Refill prior 26h Alpaca News stream to catch any gaps."""
|
||||
from app.services.news.alpaca_news_client import AlpacaNewsClient
|
||||
from app.services.news.headline_ingest_service import (
|
||||
alpaca_articles_to_rows,
|
||||
insert_headline_rows,
|
||||
)
|
||||
|
||||
client = AlpacaNewsClient()
|
||||
if not client.is_configured():
|
||||
return
|
||||
|
||||
end = datetime.now(timezone.utc)
|
||||
start = end - timedelta(hours=26)
|
||||
|
||||
try:
|
||||
articles = []
|
||||
async for art in client.fetch_news(symbols=None, start=start, end=end, page_limit=50):
|
||||
articles.append(art)
|
||||
rows = alpaca_articles_to_rows(articles)
|
||||
inserted = await insert_headline_rows(rows) if rows else 0
|
||||
logger.info(f"[News] Alpaca daily backfill: +{inserted} new ({len(articles)} articles, 26h)")
|
||||
except Exception as e:
|
||||
logger.error(f"[News] Alpaca daily backfill failed: {e}")
|
||||
finally:
|
||||
await client.close()
|
||||
|
||||
|
||||
async def _run_stocktwits_universe_refresh() -> None:
|
||||
from app.services.news.stocktwits_universe import compute_stocktwits_universe
|
||||
try:
|
||||
await compute_stocktwits_universe()
|
||||
except Exception as e:
|
||||
logger.error(f"[News] StockTwits universe refresh failed: {e}")
|
||||
|
||||
|
||||
async def _run_stocktwits_poll() -> None:
|
||||
"""Pull recent messages for every ticker in the cached universe."""
|
||||
from app.services.news.stocktwits_client import StocktwitsClient
|
||||
from app.services.news.stocktwits_universe import get_cached_universe
|
||||
from app.services.news.headline_ingest_service import (
|
||||
stocktwits_messages_to_rows,
|
||||
insert_headline_rows,
|
||||
)
|
||||
|
||||
universe = await get_cached_universe()
|
||||
if not universe:
|
||||
# On first start there's no cached universe — compute on the fly so
|
||||
# the very first poll has something to do.
|
||||
from app.services.news.stocktwits_universe import compute_stocktwits_universe
|
||||
try:
|
||||
universe = await compute_stocktwits_universe()
|
||||
except Exception as e:
|
||||
logger.warning(f"[News] StockTwits initial universe build failed: {e}")
|
||||
return
|
||||
if not universe:
|
||||
return
|
||||
|
||||
client = StocktwitsClient()
|
||||
total_inserted = 0
|
||||
try:
|
||||
for ticker in universe:
|
||||
try:
|
||||
payload = await client.fetch_symbol_stream(ticker, max_results=30)
|
||||
except Exception as e:
|
||||
logger.warning(f"[News] StockTwits {ticker} failed: {e}")
|
||||
continue
|
||||
messages = payload.get("messages") or []
|
||||
if not messages:
|
||||
continue
|
||||
rows = stocktwits_messages_to_rows(ticker, messages)
|
||||
if rows:
|
||||
total_inserted += await insert_headline_rows(rows)
|
||||
finally:
|
||||
await client.close()
|
||||
|
||||
if total_inserted:
|
||||
logger.info(f"[News] StockTwits poll: +{total_inserted} new across {len(universe)} tickers")
|
||||
|
||||
|
||||
async def _run_finnhub_daily_backfill() -> None:
|
||||
"""Pull yesterday's Finnhub /company-news for the active V49 universe."""
|
||||
from app.services.news.finnhub_client import FinnhubClient
|
||||
from app.services.news.headline_ingest_service import (
|
||||
finnhub_articles_to_rows,
|
||||
insert_headline_rows,
|
||||
)
|
||||
from sqlalchemy import select
|
||||
from app.core.database import AsyncSessionLocal
|
||||
from app.models.universe_snapshot import UniverseTickerRegistry
|
||||
|
||||
client = FinnhubClient()
|
||||
if not client.is_configured():
|
||||
return
|
||||
|
||||
today = datetime.now(timezone.utc).date()
|
||||
yesterday = today - timedelta(days=1)
|
||||
|
||||
# Pull active universe tickers
|
||||
async with AsyncSessionLocal() as db:
|
||||
rows = (await db.execute(
|
||||
select(UniverseTickerRegistry.ticker).where(UniverseTickerRegistry.is_active.is_(True))
|
||||
)).scalars().all()
|
||||
tickers = [t.upper() for t in rows if t]
|
||||
if not tickers:
|
||||
logger.info("[News] Finnhub daily: no active universe tickers")
|
||||
return
|
||||
|
||||
total_inserted = 0
|
||||
try:
|
||||
for ticker in tickers:
|
||||
try:
|
||||
payload = await client.fetch_company_news(ticker, yesterday, today)
|
||||
except Exception as e:
|
||||
logger.warning(f"[News] Finnhub {ticker} failed: {e}")
|
||||
continue
|
||||
if not payload:
|
||||
continue
|
||||
news_rows = finnhub_articles_to_rows(ticker, payload)
|
||||
if news_rows:
|
||||
total_inserted += await insert_headline_rows(news_rows)
|
||||
finally:
|
||||
await client.close()
|
||||
|
||||
logger.info(f"[News] Finnhub daily: +{total_inserted} new across {len(tickers)} tickers")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Lifecycle
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def start_news_ingest_scheduler() -> None:
|
||||
"""Wire jobs into the AsyncIOScheduler. Idempotent."""
|
||||
if not settings.NEWS_INGEST_ENABLED:
|
||||
logger.info("[News] NEWS_INGEST_ENABLED=false — scheduler not started")
|
||||
return
|
||||
|
||||
sched = _get_scheduler()
|
||||
if sched is None:
|
||||
return
|
||||
|
||||
# Source key presence checks (fail-fast on no-keys-at-all)
|
||||
alpaca_ok = bool(settings.ALPACA_API_KEY and settings.ALPACA_SECRET_KEY)
|
||||
finnhub_ok = bool(settings.FINNHUB_API_KEY)
|
||||
stocktwits_ok = True # public API
|
||||
enabled_sources = []
|
||||
if alpaca_ok:
|
||||
enabled_sources.append("alpaca_benzinga")
|
||||
else:
|
||||
logger.warning("[News] ALPACA_API_KEY/SECRET missing — Alpaca News jobs skipped")
|
||||
if finnhub_ok:
|
||||
enabled_sources.append("finnhub")
|
||||
else:
|
||||
logger.warning("[News] FINNHUB_API_KEY missing — Finnhub jobs skipped")
|
||||
if stocktwits_ok:
|
||||
enabled_sources.append("stocktwits")
|
||||
|
||||
# Fail-fast: if neither Alpaca nor Finnhub is configured, the only source
|
||||
# is StockTwits which is low-signal on its own. Abort instead of silently
|
||||
# running a degenerate ingest.
|
||||
if not (alpaca_ok or finnhub_ok):
|
||||
logger.error(
|
||||
"[News] NEWS_INGEST_ENABLED=true but no premium-source keys configured "
|
||||
"(need at least ALPACA_API_KEY/SECRET or FINNHUB_API_KEY) — aborting scheduler"
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
from apscheduler.triggers.cron import CronTrigger
|
||||
from apscheduler.triggers.interval import IntervalTrigger
|
||||
|
||||
if alpaca_ok:
|
||||
sched.add_job(
|
||||
_run_alpaca_realtime_poll,
|
||||
trigger=IntervalTrigger(minutes=5),
|
||||
id="alpaca_news_realtime_poll",
|
||||
replace_existing=True,
|
||||
max_instances=1,
|
||||
misfire_grace_time=120,
|
||||
coalesce=True,
|
||||
)
|
||||
sched.add_job(
|
||||
_run_alpaca_daily_backfill,
|
||||
trigger=CronTrigger(hour=4, minute=30),
|
||||
id="alpaca_news_daily_backfill",
|
||||
replace_existing=True,
|
||||
max_instances=1,
|
||||
misfire_grace_time=600,
|
||||
coalesce=True,
|
||||
)
|
||||
|
||||
if stocktwits_ok:
|
||||
sched.add_job(
|
||||
_run_stocktwits_universe_refresh,
|
||||
trigger=CronTrigger(day_of_week="mon-fri", hour=9, minute=0),
|
||||
id="stocktwits_universe_refresh",
|
||||
replace_existing=True,
|
||||
max_instances=1,
|
||||
misfire_grace_time=600,
|
||||
coalesce=True,
|
||||
)
|
||||
sched.add_job(
|
||||
_run_stocktwits_poll,
|
||||
trigger=IntervalTrigger(minutes=5),
|
||||
id="stocktwits_poll",
|
||||
replace_existing=True,
|
||||
max_instances=1,
|
||||
misfire_grace_time=120,
|
||||
coalesce=True,
|
||||
)
|
||||
|
||||
if finnhub_ok:
|
||||
sched.add_job(
|
||||
_run_finnhub_daily_backfill,
|
||||
trigger=CronTrigger(hour=5, minute=0),
|
||||
id="finnhub_daily_backfill",
|
||||
replace_existing=True,
|
||||
max_instances=1,
|
||||
misfire_grace_time=600,
|
||||
coalesce=True,
|
||||
)
|
||||
|
||||
if not sched.running:
|
||||
sched.start()
|
||||
logger.info(f"[News] ingest scheduler started — sources: {enabled_sources}")
|
||||
except Exception as e:
|
||||
logger.error(f"[News] ingest scheduler start failed: {e}")
|
||||
|
||||
|
||||
def stop_news_ingest_scheduler() -> None:
|
||||
sched = _get_scheduler()
|
||||
if sched and sched.running:
|
||||
sched.shutdown(wait=False)
|
||||
logger.info("[News] ingest scheduler stopped")
|
||||
@ -0,0 +1,227 @@
|
||||
"""
|
||||
Session-aggregated news/social signal.
|
||||
|
||||
Computes per-(ticker, session_date, window) aggregates directly from raw
|
||||
NewsHeadline rows via SQL — no materialized table. Redis caching at the
|
||||
endpoint layer absorbs repeated reads.
|
||||
|
||||
Aggregate fields:
|
||||
headline_count, primary_count, first/last_headline_at,
|
||||
category_counts (JSON), sentiment_mean, sentiment_recency_weighted,
|
||||
social: {message_count, bull_count, bear_count, bull_bear_ratio},
|
||||
sources_present
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import math
|
||||
from collections import Counter
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from datetime import date, datetime, timezone
|
||||
from typing import Iterable
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.news_headline import NewsHeadline
|
||||
from app.services.news.session_window import WindowName, session_window
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Recency-weight half-life: 1 hour close to open contributes weight 1.0,
|
||||
# 6 hours earlier ≈ 0.55. Matches plan (decay=0.1 per hour).
|
||||
_RECENCY_DECAY_PER_MIN = 0.1 / 60.0
|
||||
|
||||
|
||||
@dataclass
|
||||
class SocialStats:
|
||||
message_count: int = 0
|
||||
bull_count: int = 0
|
||||
bear_count: int = 0
|
||||
bull_bear_ratio: float | None = None # bull/(bull+bear); None if no directional msgs
|
||||
|
||||
|
||||
@dataclass
|
||||
class SessionAggregate:
|
||||
ticker: str
|
||||
session_date: str # ISO date (ET)
|
||||
window: str
|
||||
headline_count: int = 0
|
||||
primary_count: int = 0
|
||||
first_headline_at: str | None = None
|
||||
last_headline_at: str | None = None
|
||||
category_counts: dict[str, int] = field(default_factory=dict)
|
||||
sentiment_mean: float | None = None
|
||||
sentiment_recency_weighted: float | None = None
|
||||
social: SocialStats = field(default_factory=SocialStats)
|
||||
sources_present: list[str] = field(default_factory=list)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
async def aggregate_session(
|
||||
db: AsyncSession,
|
||||
ticker: str,
|
||||
session_date: date,
|
||||
window: WindowName,
|
||||
sources: list[str] | None = None,
|
||||
pit_cutoff: datetime | None = None,
|
||||
) -> SessionAggregate:
|
||||
"""
|
||||
Aggregate news headlines for one (ticker, session_date, window).
|
||||
|
||||
`pit_cutoff` (optional) excludes rows whose `ingested_at > cutoff` to
|
||||
enable lookahead-safe backtest queries. If omitted, defaults to the
|
||||
session window's end_utc (so we never see headlines that arrived after
|
||||
the window closed in real time).
|
||||
"""
|
||||
ticker_u = ticker.strip().upper()
|
||||
start_utc, end_utc = session_window(session_date, window)
|
||||
cutoff = pit_cutoff or end_utc
|
||||
|
||||
stmt = select(NewsHeadline).where(
|
||||
NewsHeadline.ticker == ticker_u,
|
||||
NewsHeadline.published_at >= start_utc,
|
||||
NewsHeadline.published_at < end_utc,
|
||||
NewsHeadline.ingested_at <= cutoff,
|
||||
)
|
||||
if sources:
|
||||
stmt = stmt.where(NewsHeadline.source.in_([s.strip() for s in sources if s]))
|
||||
|
||||
result = await db.execute(stmt)
|
||||
rows = result.scalars().all()
|
||||
|
||||
return _build_aggregate(
|
||||
rows=rows,
|
||||
ticker=ticker_u,
|
||||
session_date=session_date,
|
||||
window=window,
|
||||
window_end_utc=end_utc,
|
||||
)
|
||||
|
||||
|
||||
async def aggregate_session_batch(
|
||||
db: AsyncSession,
|
||||
tickers: list[str],
|
||||
session_date: date,
|
||||
window: WindowName,
|
||||
sources: list[str] | None = None,
|
||||
pit_cutoff: datetime | None = None,
|
||||
) -> dict[str, SessionAggregate]:
|
||||
"""Aggregate for many tickers in one query. Missing tickers → empty agg."""
|
||||
tickers_u = [t.strip().upper() for t in tickers if t and t.strip()]
|
||||
if not tickers_u:
|
||||
return {}
|
||||
|
||||
start_utc, end_utc = session_window(session_date, window)
|
||||
cutoff = pit_cutoff or end_utc
|
||||
|
||||
stmt = select(NewsHeadline).where(
|
||||
NewsHeadline.ticker.in_(tickers_u),
|
||||
NewsHeadline.published_at >= start_utc,
|
||||
NewsHeadline.published_at < end_utc,
|
||||
NewsHeadline.ingested_at <= cutoff,
|
||||
)
|
||||
if sources:
|
||||
stmt = stmt.where(NewsHeadline.source.in_([s.strip() for s in sources if s]))
|
||||
|
||||
result = await db.execute(stmt)
|
||||
rows = result.scalars().all()
|
||||
|
||||
by_ticker: dict[str, list[NewsHeadline]] = {t: [] for t in tickers_u}
|
||||
for r in rows:
|
||||
by_ticker.setdefault(r.ticker, []).append(r)
|
||||
|
||||
out: dict[str, SessionAggregate] = {}
|
||||
for t in tickers_u:
|
||||
out[t] = _build_aggregate(
|
||||
rows=by_ticker.get(t, []),
|
||||
ticker=t,
|
||||
session_date=session_date,
|
||||
window=window,
|
||||
window_end_utc=end_utc,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Aggregate construction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _build_aggregate(
|
||||
rows: Iterable[NewsHeadline],
|
||||
ticker: str,
|
||||
session_date: date,
|
||||
window: str,
|
||||
window_end_utc: datetime,
|
||||
) -> SessionAggregate:
|
||||
rows = list(rows)
|
||||
agg = SessionAggregate(
|
||||
ticker=ticker,
|
||||
session_date=session_date.isoformat(),
|
||||
window=window,
|
||||
)
|
||||
if not rows:
|
||||
return agg
|
||||
|
||||
cat_counter: Counter[str] = Counter()
|
||||
sentiments: list[float] = []
|
||||
weighted_num = 0.0
|
||||
weighted_den = 0.0
|
||||
sources: set[str] = set()
|
||||
social_msgs = 0
|
||||
social_bull = 0
|
||||
social_bear = 0
|
||||
first_at: datetime | None = None
|
||||
last_at: datetime | None = None
|
||||
primary_count = 0
|
||||
|
||||
for r in rows:
|
||||
sources.add(r.source)
|
||||
if r.is_primary:
|
||||
primary_count += 1
|
||||
|
||||
for c in (r.categories or []):
|
||||
cat_counter[c] += 1
|
||||
|
||||
pub = r.published_at
|
||||
if pub.tzinfo is None:
|
||||
pub = pub.replace(tzinfo=timezone.utc)
|
||||
if first_at is None or pub < first_at:
|
||||
first_at = pub
|
||||
if last_at is None or pub > last_at:
|
||||
last_at = pub
|
||||
|
||||
if r.raw_sentiment is not None:
|
||||
sentiments.append(float(r.raw_sentiment))
|
||||
# Recency weight: closer to window end_utc → higher weight
|
||||
mins_before_end = max(0.0, (window_end_utc - pub).total_seconds() / 60.0)
|
||||
w = math.exp(-_RECENCY_DECAY_PER_MIN * mins_before_end)
|
||||
weighted_num += w * float(r.raw_sentiment)
|
||||
weighted_den += w
|
||||
|
||||
if r.source == "stocktwits":
|
||||
social_msgs += 1
|
||||
if r.raw_sentiment == 1.0:
|
||||
social_bull += 1
|
||||
elif r.raw_sentiment == -1.0:
|
||||
social_bear += 1
|
||||
|
||||
agg.headline_count = len(rows)
|
||||
agg.primary_count = primary_count
|
||||
agg.first_headline_at = first_at.isoformat() if first_at else None
|
||||
agg.last_headline_at = last_at.isoformat() if last_at else None
|
||||
agg.category_counts = dict(cat_counter)
|
||||
agg.sentiment_mean = (sum(sentiments) / len(sentiments)) if sentiments else None
|
||||
agg.sentiment_recency_weighted = (weighted_num / weighted_den) if weighted_den > 0 else None
|
||||
directional = social_bull + social_bear
|
||||
agg.social = SocialStats(
|
||||
message_count=social_msgs,
|
||||
bull_count=social_bull,
|
||||
bear_count=social_bear,
|
||||
bull_bear_ratio=(social_bull / directional) if directional > 0 else None,
|
||||
)
|
||||
agg.sources_present = sorted(sources)
|
||||
return agg
|
||||
@ -0,0 +1,150 @@
|
||||
"""
|
||||
Trading-session window math (NYSE / XNYS).
|
||||
|
||||
`session_window(date, "premarket"|"intraday"|"post"|"full_session")` returns
|
||||
the (start_utc, end_utc) bounds the news aggregator filters on. NYSE holidays
|
||||
and short-day closes (1pm ET on day-after-Thanksgiving etc.) are handled by
|
||||
pandas_market_calendars when available; falls back to weekday-only logic if
|
||||
the dependency is missing.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import date, datetime, time, timedelta, timezone
|
||||
from functools import lru_cache
|
||||
from typing import Literal
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
ET = ZoneInfo("America/New_York")
|
||||
|
||||
WindowName = Literal["premarket", "intraday", "post", "full_session"]
|
||||
|
||||
# Default ET clock-time bounds. Short days (early close) override regular_close
|
||||
# via the market-calendar lookup below. `post` window ends at the *next*
|
||||
# trading day's premarket start (04:00 ET) to avoid overlap.
|
||||
_DEFAULT_PREMARKET_START_T = time(4, 0)
|
||||
_DEFAULT_REGULAR_OPEN_T = time(9, 30)
|
||||
_DEFAULT_REGULAR_CLOSE_T = time(16, 0)
|
||||
|
||||
|
||||
def session_window(session_date: date, window: WindowName) -> tuple[datetime, datetime]:
|
||||
"""Return UTC bounds for the given window on the given ET session date."""
|
||||
if not _is_trading_day(session_date):
|
||||
raise ValueError(
|
||||
f"{session_date} is not a NYSE trading day; choose the next/prev session"
|
||||
)
|
||||
|
||||
open_dt_et, close_dt_et = _session_bounds_et(session_date)
|
||||
prev_close_dt_et = _prev_session_close_et(session_date)
|
||||
next_premarket_start_et = _next_session_premarket_start_et(session_date)
|
||||
|
||||
if window == "premarket":
|
||||
start_et = prev_close_dt_et
|
||||
end_et = open_dt_et
|
||||
elif window == "intraday":
|
||||
start_et = open_dt_et
|
||||
end_et = close_dt_et
|
||||
elif window == "post":
|
||||
# Post-market ends at next session's premarket start (04:00 ET) to avoid
|
||||
# overlap with the next session's `premarket` window.
|
||||
start_et = close_dt_et
|
||||
end_et = next_premarket_start_et
|
||||
elif window == "full_session":
|
||||
start_et = prev_close_dt_et
|
||||
end_et = next_premarket_start_et
|
||||
else:
|
||||
raise ValueError(f"Unknown window: {window}")
|
||||
|
||||
return _to_utc(start_et), _to_utc(end_et)
|
||||
|
||||
|
||||
def _to_utc(dt: datetime) -> datetime:
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=ET)
|
||||
return dt.astimezone(timezone.utc)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Market-calendar lookups (cached)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _calendar():
|
||||
try:
|
||||
import pandas_market_calendars as mcal # type: ignore
|
||||
return mcal.get_calendar("XNYS")
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"pandas_market_calendars not available (%s); falling back to weekday-only session logic",
|
||||
e,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _is_trading_day(d: date) -> bool:
|
||||
cal = _calendar()
|
||||
if cal is None:
|
||||
return d.weekday() < 5
|
||||
schedule = cal.schedule(start_date=d, end_date=d)
|
||||
return not schedule.empty
|
||||
|
||||
|
||||
def _session_bounds_et(d: date) -> tuple[datetime, datetime]:
|
||||
"""Regular open/close in ET. Honors short days when calendar is available."""
|
||||
cal = _calendar()
|
||||
if cal is not None:
|
||||
schedule = cal.schedule(start_date=d, end_date=d)
|
||||
if not schedule.empty:
|
||||
row = schedule.iloc[0]
|
||||
open_utc = row["market_open"].to_pydatetime()
|
||||
close_utc = row["market_close"].to_pydatetime()
|
||||
return open_utc.astimezone(ET), close_utc.astimezone(ET)
|
||||
return (
|
||||
datetime.combine(d, _DEFAULT_REGULAR_OPEN_T, tzinfo=ET),
|
||||
datetime.combine(d, _DEFAULT_REGULAR_CLOSE_T, tzinfo=ET),
|
||||
)
|
||||
|
||||
|
||||
def _prev_session_close_et(d: date) -> datetime:
|
||||
prev = _prev_trading_day(d)
|
||||
_, close_dt_et = _session_bounds_et(prev)
|
||||
return close_dt_et
|
||||
|
||||
|
||||
def _next_session_open_et(d: date) -> datetime:
|
||||
nxt = _next_trading_day(d)
|
||||
open_dt_et, _ = _session_bounds_et(nxt)
|
||||
return open_dt_et
|
||||
|
||||
|
||||
def _next_session_premarket_start_et(d: date) -> datetime:
|
||||
"""Next trading day's premarket window start (04:00 ET)."""
|
||||
nxt = _next_trading_day(d)
|
||||
return datetime.combine(nxt, _DEFAULT_PREMARKET_START_T, tzinfo=ET)
|
||||
|
||||
|
||||
def _prev_trading_day(d: date, max_lookback: int = 10) -> date:
|
||||
cal = _calendar()
|
||||
if cal is not None:
|
||||
schedule = cal.schedule(start_date=d - timedelta(days=max_lookback), end_date=d - timedelta(days=1))
|
||||
if not schedule.empty:
|
||||
return schedule.index[-1].date()
|
||||
candidate = d - timedelta(days=1)
|
||||
while candidate.weekday() >= 5:
|
||||
candidate -= timedelta(days=1)
|
||||
return candidate
|
||||
|
||||
|
||||
def _next_trading_day(d: date, max_lookahead: int = 10) -> date:
|
||||
cal = _calendar()
|
||||
if cal is not None:
|
||||
schedule = cal.schedule(start_date=d + timedelta(days=1), end_date=d + timedelta(days=max_lookahead))
|
||||
if not schedule.empty:
|
||||
return schedule.index[0].date()
|
||||
candidate = d + timedelta(days=1)
|
||||
while candidate.weekday() >= 5:
|
||||
candidate += timedelta(days=1)
|
||||
return candidate
|
||||
@ -0,0 +1,96 @@
|
||||
"""
|
||||
StockTwits public API client.
|
||||
|
||||
Endpoint: GET /streams/symbol/{ticker}.json on api.stocktwits.com/api/2
|
||||
Auth: none required for public streams.
|
||||
Rate limit: ~200 req/hr per IP. We enforce a token bucket of one request per
|
||||
3.6 seconds (60 / hour buffer below the 200/hr ceiling) — well under the
|
||||
documented limit even with multiple ingestor processes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from app.core.config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class StocktwitsClient:
|
||||
def __init__(
|
||||
self,
|
||||
base_url: str | None = None,
|
||||
request_interval_sec: float = 3.6,
|
||||
):
|
||||
self.base_url = (base_url or settings.STOCKTWITS_BASE_URL).rstrip("/")
|
||||
self._client: httpx.AsyncClient | None = None
|
||||
self._gate = asyncio.Semaphore(1)
|
||||
self._interval = request_interval_sec
|
||||
self._last_request_at: float = 0.0
|
||||
|
||||
def is_configured(self) -> bool:
|
||||
return True # public API
|
||||
|
||||
async def _get_client(self) -> httpx.AsyncClient:
|
||||
if self._client is None or self._client.is_closed:
|
||||
self._client = httpx.AsyncClient(base_url=self.base_url, timeout=20.0)
|
||||
return self._client
|
||||
|
||||
async def close(self) -> None:
|
||||
if self._client and not self._client.is_closed:
|
||||
await self._client.aclose()
|
||||
|
||||
async def fetch_symbol_stream(
|
||||
self,
|
||||
symbol: str,
|
||||
since_id: int | None = None,
|
||||
max_results: int = 30,
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Fetch the recent message stream for one symbol.
|
||||
|
||||
`since_id` (StockTwits parameter `since`) yields only messages whose ID
|
||||
is greater than the given value. Pass the highest message ID seen on
|
||||
the previous poll to stay incremental and avoid re-ingesting duplicates
|
||||
(dedup is also enforced at the DB layer via uq_news_headline_*).
|
||||
"""
|
||||
params: dict[str, Any] = {"limit": min(max_results, 30)}
|
||||
if since_id is not None:
|
||||
params["since"] = since_id
|
||||
|
||||
sym = symbol.strip().upper()
|
||||
path = f"/streams/symbol/{sym}.json"
|
||||
|
||||
async with self._gate:
|
||||
now = time.monotonic()
|
||||
wait = self._interval - (now - self._last_request_at)
|
||||
if wait > 0:
|
||||
await asyncio.sleep(wait)
|
||||
|
||||
client = await self._get_client()
|
||||
try:
|
||||
resp = await client.get(path, params=params)
|
||||
self._last_request_at = time.monotonic()
|
||||
except (httpx.ConnectError, httpx.ReadTimeout) as e:
|
||||
logger.warning(f"StockTwits fetch failed {sym}: {e}")
|
||||
return {"messages": []}
|
||||
|
||||
if resp.status_code == 429:
|
||||
logger.warning(f"StockTwits 429 on {sym} — caller should back off")
|
||||
return {"messages": []}
|
||||
if resp.status_code == 404:
|
||||
# Symbol not found on StockTwits — treat as empty rather than error
|
||||
return {"messages": []}
|
||||
try:
|
||||
resp.raise_for_status()
|
||||
except httpx.HTTPStatusError as e:
|
||||
logger.warning(f"StockTwits {resp.status_code} on {sym}: {e}")
|
||||
return {"messages": []}
|
||||
|
||||
return resp.json()
|
||||
@ -1,285 +0,0 @@
|
||||
"""
|
||||
Feature builder - aggregate raw events into normalized (z-scored) feature dicts.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import statistics
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select, and_, func, cast
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
|
||||
from app.models.overlay_raw_event import (
|
||||
OverlayHeadlineEvent,
|
||||
OverlayVideoEvent,
|
||||
OverlayWikiPageview,
|
||||
OverlayTrendObservation,
|
||||
)
|
||||
from app.models.overlay_registry import ThemeTopicMap
|
||||
from app.services.overlay.finra_overlay_loader import FinraOverlayLoader
|
||||
from app.core.overlay_config import ZSCORE_WINDOW_DAYS, WINSOR_LOWER, WINSOR_UPPER
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def winsorize(value: float, lower: float = WINSOR_LOWER, upper: float = WINSOR_UPPER) -> float:
|
||||
return max(lower, min(upper, value))
|
||||
|
||||
|
||||
|
||||
def compute_zscore(value: float, values: List[float]) -> Optional[float]:
|
||||
"""Compute z-score of *value* within *values* (requires ≥2 data points)."""
|
||||
if len(values) < 2:
|
||||
return None
|
||||
mean = statistics.mean(values)
|
||||
stdev = statistics.pstdev(values) # population stdev for stability
|
||||
if stdev == 0:
|
||||
return 0.0
|
||||
z = (value - mean) / stdev
|
||||
return winsorize(z)
|
||||
|
||||
|
||||
def _day_key(ts) -> str:
|
||||
"""Return YYYY-MM-DD string from a datetime or date object."""
|
||||
if hasattr(ts, "date"):
|
||||
return ts.date().isoformat()
|
||||
return str(ts)[:10]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Feature builder
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class FeatureBuilder:
|
||||
"""Build overlay feature records from raw event tables."""
|
||||
|
||||
def __init__(self):
|
||||
self.finra_loader = FinraOverlayLoader()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Headline features
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def build_headline_features(
|
||||
self, db: AsyncSession, symbol: str, as_of: datetime
|
||||
) -> Dict:
|
||||
cutoff_24h = as_of - timedelta(hours=24)
|
||||
cutoff_6h = as_of - timedelta(hours=6)
|
||||
window_cutoff = as_of - timedelta(days=ZSCORE_WINDOW_DAYS)
|
||||
|
||||
# Fetch headlines matching symbol in z-score window (DB-level JSON filter)
|
||||
symbol_json = cast([symbol], JSONB)
|
||||
result = await db.execute(
|
||||
select(
|
||||
OverlayHeadlineEvent.publisher,
|
||||
OverlayHeadlineEvent.published_at,
|
||||
).where(
|
||||
and_(
|
||||
OverlayHeadlineEvent.published_at >= window_cutoff,
|
||||
OverlayHeadlineEvent.published_at <= as_of,
|
||||
cast(OverlayHeadlineEvent.matched_symbols, JSONB).op('@>')(symbol_json),
|
||||
)
|
||||
)
|
||||
)
|
||||
sym_rows_hist = result.fetchall()
|
||||
|
||||
sym_rows_24h = [r for r in sym_rows_hist if r.published_at >= cutoff_24h]
|
||||
sym_rows_6h = [r for r in sym_rows_24h if r.published_at >= cutoff_6h]
|
||||
|
||||
headline_count_24h = len(sym_rows_24h)
|
||||
headline_count_6h = len(sym_rows_6h)
|
||||
publishers = {r.publisher for r in sym_rows_24h if r.publisher}
|
||||
publisher_breadth_24h = len(publishers)
|
||||
|
||||
# Build daily counts for z-score window
|
||||
daily_counts: Dict[str, int] = {}
|
||||
for row in sym_rows_hist:
|
||||
key = _day_key(row.published_at)
|
||||
daily_counts[key] = daily_counts.get(key, 0) + 1
|
||||
|
||||
hist_values = list(daily_counts.values())
|
||||
headline_burst_z = compute_zscore(float(headline_count_24h), hist_values) if hist_values else None
|
||||
|
||||
return {
|
||||
"headline_count_6h": headline_count_6h,
|
||||
"headline_count_24h": headline_count_24h,
|
||||
"publisher_breadth_24h": publisher_breadth_24h,
|
||||
"headline_burst_z": headline_burst_z,
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# YouTube features
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def build_youtube_features(
|
||||
self, db: AsyncSession, symbol: str, as_of: datetime
|
||||
) -> Dict:
|
||||
cutoff_24h = as_of - timedelta(hours=24)
|
||||
window_cutoff = as_of - timedelta(days=ZSCORE_WINDOW_DAYS)
|
||||
|
||||
symbol_json = cast([symbol], JSONB)
|
||||
result = await db.execute(
|
||||
select(
|
||||
OverlayVideoEvent.view_count,
|
||||
OverlayVideoEvent.channel_weight,
|
||||
OverlayVideoEvent.published_at,
|
||||
).where(
|
||||
and_(
|
||||
OverlayVideoEvent.published_at >= window_cutoff,
|
||||
OverlayVideoEvent.published_at <= as_of,
|
||||
cast(OverlayVideoEvent.matched_symbols, JSONB).op('@>')(symbol_json),
|
||||
)
|
||||
)
|
||||
)
|
||||
sym_rows = result.fetchall()
|
||||
sym_rows_24h = [r for r in sym_rows if r.published_at >= cutoff_24h]
|
||||
|
||||
mentions_24h = len(sym_rows_24h)
|
||||
weighted_views_24h = sum((r.view_count or 0) * (r.channel_weight or 0.5) for r in sym_rows_24h)
|
||||
|
||||
# Daily weighted views for z-score
|
||||
daily_weighted: Dict[str, float] = {}
|
||||
for row in sym_rows:
|
||||
key = _day_key(row.published_at)
|
||||
daily_weighted[key] = daily_weighted.get(key, 0.0) + (row.view_count or 0) * (row.channel_weight or 0.5)
|
||||
|
||||
hist_values = list(daily_weighted.values())
|
||||
youtube_influence_z = compute_zscore(weighted_views_24h, hist_values) if hist_values else None
|
||||
|
||||
return {
|
||||
"youtube_mentions_24h": mentions_24h,
|
||||
"youtube_weighted_views_24h": round(weighted_views_24h, 2),
|
||||
"youtube_influence_z": youtube_influence_z,
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Wiki features
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def build_wiki_features(
|
||||
self, db: AsyncSession, symbol: str, as_of: datetime
|
||||
) -> Dict:
|
||||
cutoff_7d = as_of - timedelta(days=7)
|
||||
cutoff_1d = as_of - timedelta(days=1)
|
||||
window_cutoff = as_of - timedelta(days=ZSCORE_WINDOW_DAYS)
|
||||
|
||||
result = await db.execute(
|
||||
select(
|
||||
OverlayWikiPageview.views,
|
||||
OverlayWikiPageview.date,
|
||||
).where(
|
||||
and_(
|
||||
OverlayWikiPageview.mapped_symbol == symbol,
|
||||
OverlayWikiPageview.date >= window_cutoff,
|
||||
)
|
||||
).order_by(OverlayWikiPageview.date.desc())
|
||||
)
|
||||
rows = result.fetchall()
|
||||
|
||||
if not rows:
|
||||
return {"wiki_views_1d": None, "wiki_views_7d_avg": None, "wiki_attention_z": None}
|
||||
|
||||
# Latest day's views (most recent row, regardless of exact time)
|
||||
views_1d = rows[0].views if rows else None
|
||||
|
||||
# 7-day average
|
||||
rows_7d = [r for r in rows if r.date >= cutoff_7d]
|
||||
views_7d_avg = sum(r.views for r in rows_7d) / len(rows_7d) if rows_7d else None
|
||||
|
||||
# Historical z-score
|
||||
hist_views = [r.views for r in rows]
|
||||
wiki_attention_z = None
|
||||
if views_1d is not None and hist_views:
|
||||
wiki_attention_z = compute_zscore(float(views_1d), hist_views)
|
||||
|
||||
return {
|
||||
"wiki_views_1d": views_1d,
|
||||
"wiki_views_7d_avg": round(views_7d_avg, 2) if views_7d_avg else None,
|
||||
"wiki_attention_z": wiki_attention_z,
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Google Trends features
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def build_trends_features(
|
||||
self, db: AsyncSession, symbol: str, as_of: datetime
|
||||
) -> Dict:
|
||||
window_cutoff = as_of - timedelta(days=ZSCORE_WINDOW_DAYS)
|
||||
cutoff_1d = as_of - timedelta(days=1)
|
||||
|
||||
# Find topic IDs mapped to this symbol
|
||||
topics_result = await db.execute(
|
||||
select(ThemeTopicMap).where(ThemeTopicMap.active == True)
|
||||
)
|
||||
topics = topics_result.scalars().all()
|
||||
topic_ids = [t.topic_id for t in topics if symbol in (t.mapped_symbols or [])]
|
||||
|
||||
if not topic_ids:
|
||||
return {"theme_heat_z": None}
|
||||
|
||||
result = await db.execute(
|
||||
select(
|
||||
OverlayTrendObservation.interest_value,
|
||||
OverlayTrendObservation.observed_at,
|
||||
).where(
|
||||
and_(
|
||||
OverlayTrendObservation.topic_id.in_(topic_ids),
|
||||
OverlayTrendObservation.observed_at >= window_cutoff,
|
||||
)
|
||||
).order_by(OverlayTrendObservation.observed_at)
|
||||
)
|
||||
rows = result.fetchall()
|
||||
|
||||
if not rows:
|
||||
return {"theme_heat_z": None}
|
||||
|
||||
recent = [r for r in rows if r.observed_at >= cutoff_1d]
|
||||
current_value = float(sum(r.interest_value for r in recent) / len(recent)) if recent else None
|
||||
|
||||
if current_value is None:
|
||||
return {"theme_heat_z": None}
|
||||
|
||||
hist_values = [float(r.interest_value) for r in rows]
|
||||
theme_heat_z = compute_zscore(current_value, hist_values) if len(hist_values) >= 2 else None
|
||||
|
||||
return {"theme_heat_z": theme_heat_z}
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# FINRA crowding features
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def build_crowding_features(self, db: AsyncSession, symbol: str) -> Dict:
|
||||
return await self.finra_loader.get_crowding_metrics(db, symbol)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Build all features
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def build_all_features(
|
||||
self, db: AsyncSession, symbol: str, as_of: Optional[datetime] = None
|
||||
) -> Dict:
|
||||
"""Build all features for a symbol and return a combined feature dict."""
|
||||
if as_of is None:
|
||||
as_of = datetime.now(timezone.utc)
|
||||
|
||||
headline = await self.build_headline_features(db, symbol, as_of)
|
||||
youtube = await self.build_youtube_features(db, symbol, as_of)
|
||||
wiki = await self.build_wiki_features(db, symbol, as_of)
|
||||
trends = await self.build_trends_features(db, symbol, as_of)
|
||||
crowding = await self.build_crowding_features(db, symbol)
|
||||
|
||||
return {
|
||||
**headline,
|
||||
**youtube,
|
||||
**wiki,
|
||||
**trends,
|
||||
**crowding,
|
||||
"as_of_ts": as_of,
|
||||
}
|
||||
@ -1,85 +0,0 @@
|
||||
"""
|
||||
FINRA overlay loader - derive crowding/stress features from the existing
|
||||
finra_short_volume table without duplicating data.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import statistics
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Dict
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select, and_, func
|
||||
|
||||
from app.models.finra_short_volume import FinraShortVolume
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class FinraOverlayLoader:
|
||||
"""Calculate crowding stress metrics from FINRA short volume data."""
|
||||
|
||||
async def get_crowding_metrics(
|
||||
self, db: AsyncSession, symbol: str, days: int = 30
|
||||
) -> Dict:
|
||||
"""
|
||||
Derive crowding stress metrics for a symbol over the last *days* days.
|
||||
|
||||
Returns a dict with:
|
||||
short_volume_ratio - latest daily short/total ratio
|
||||
short_volume_spike_zscore - how far above the rolling mean
|
||||
crowding_stress_z - negative spike z-score (high → more stress)
|
||||
|
||||
Returns {} if no FINRA data is available.
|
||||
"""
|
||||
symbol = symbol.upper()
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(days=days)
|
||||
|
||||
result = await db.execute(
|
||||
select(
|
||||
FinraShortVolume.date,
|
||||
func.sum(FinraShortVolume.short_volume).label("short_volume"),
|
||||
func.sum(FinraShortVolume.total_volume).label("total_volume"),
|
||||
)
|
||||
.where(
|
||||
and_(
|
||||
FinraShortVolume.symbol == symbol,
|
||||
FinraShortVolume.date >= cutoff,
|
||||
)
|
||||
)
|
||||
.group_by(FinraShortVolume.date)
|
||||
.order_by(FinraShortVolume.date)
|
||||
)
|
||||
|
||||
rows = result.fetchall()
|
||||
if not rows:
|
||||
return {}
|
||||
|
||||
# Build daily short ratios
|
||||
ratios = []
|
||||
for row in rows:
|
||||
_, sv, tv = row
|
||||
if tv and tv > 0:
|
||||
ratios.append(sv / tv)
|
||||
|
||||
if not ratios:
|
||||
return {}
|
||||
|
||||
latest_ratio = ratios[-1]
|
||||
|
||||
# z-score of latest vs rolling window
|
||||
if len(ratios) >= 2:
|
||||
mean_r = statistics.mean(ratios)
|
||||
stdev_r = statistics.stdev(ratios)
|
||||
spike_z = (latest_ratio - mean_r) / stdev_r if stdev_r > 0 else 0.0
|
||||
else:
|
||||
spike_z = 0.0
|
||||
|
||||
# crowding_stress_z: higher short-volume spike → negative stress on price
|
||||
crowding_stress_z = round(-spike_z, 4)
|
||||
|
||||
return {
|
||||
"short_volume_ratio": round(latest_ratio, 6),
|
||||
"short_volume_spike_zscore": round(spike_z, 4),
|
||||
"crowding_stress_z": crowding_stress_z,
|
||||
}
|
||||
@ -1,96 +0,0 @@
|
||||
"""
|
||||
Google Trends adapter (experimental, feature-flagged).
|
||||
Disabled by default; enable via GOOGLE_TRENDS_ENABLED=true env var.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from typing import List
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select, and_
|
||||
|
||||
from app.core.config import settings
|
||||
from app.models.overlay_registry import ThemeTopicMap
|
||||
from app.models.overlay_raw_event import OverlayTrendObservation
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class GoogleTrendsAdapter:
|
||||
"""Collect Google Trends data (experimental, disabled by default)."""
|
||||
|
||||
@property
|
||||
def enabled(self) -> bool:
|
||||
return getattr(settings, "GOOGLE_TRENDS_ENABLED", False)
|
||||
|
||||
async def collect(self, db: AsyncSession) -> int:
|
||||
"""
|
||||
Collect Google Trends interest data for active topics.
|
||||
|
||||
Returns 0 if feature flag is disabled or pytrends is not installed.
|
||||
Returns number of new records inserted.
|
||||
"""
|
||||
if not self.enabled:
|
||||
logger.debug("Google Trends adapter: feature flag disabled, skipping")
|
||||
return 0
|
||||
|
||||
try:
|
||||
from pytrends.request import TrendReq
|
||||
except ImportError:
|
||||
logger.warning("pytrends not installed; Google Trends adapter inactive")
|
||||
return 0
|
||||
|
||||
result = await db.execute(
|
||||
select(ThemeTopicMap).where(ThemeTopicMap.active == True)
|
||||
)
|
||||
topics = result.scalars().all()
|
||||
if not topics:
|
||||
return 0
|
||||
|
||||
inserted = 0
|
||||
|
||||
try:
|
||||
pytrends = TrendReq(hl="en-US", tz=360)
|
||||
# pytrends limits to 5 keywords at a time
|
||||
kw_list = [t.topic_label for t in topics[:5]]
|
||||
pytrends.build_payload(kw_list, timeframe="now 7-d", geo="US")
|
||||
interest_df = pytrends.interest_over_time()
|
||||
except Exception as e:
|
||||
logger.error(f"Google Trends API error: {e}")
|
||||
return 0
|
||||
|
||||
for topic in topics:
|
||||
if topic.topic_label not in interest_df.columns:
|
||||
continue
|
||||
series = interest_df[topic.topic_label]
|
||||
for ts, val in series.items():
|
||||
observed_at = ts.to_pydatetime().replace(tzinfo=timezone.utc)
|
||||
|
||||
# Check duplicate
|
||||
existing = await db.execute(
|
||||
select(OverlayTrendObservation.id).where(
|
||||
and_(
|
||||
OverlayTrendObservation.topic_id == topic.topic_id,
|
||||
OverlayTrendObservation.observed_at == observed_at,
|
||||
OverlayTrendObservation.geography == "US",
|
||||
)
|
||||
)
|
||||
)
|
||||
if existing.first():
|
||||
continue
|
||||
|
||||
record = OverlayTrendObservation(
|
||||
topic_id=topic.topic_id,
|
||||
observed_at=observed_at,
|
||||
geography="US",
|
||||
interest_value=int(val),
|
||||
)
|
||||
db.add(record)
|
||||
inserted += 1
|
||||
|
||||
if inserted:
|
||||
await db.commit()
|
||||
logger.info(f"Google Trends: inserted {inserted} observations")
|
||||
|
||||
return inserted
|
||||
@ -1,104 +0,0 @@
|
||||
"""
|
||||
Overlay scorer - compute final overlay_score, band, confidence, and decision hints
|
||||
from a feature dict produced by FeatureBuilder.
|
||||
"""
|
||||
|
||||
import math
|
||||
import logging
|
||||
from typing import Dict, Optional
|
||||
|
||||
from app.core.overlay_config import (
|
||||
SOURCE_WEIGHTS,
|
||||
BAND_THRESHOLDS,
|
||||
CONFIDENCE_PER_SOURCE,
|
||||
HOLD_EXTENSION_EXTEND_THRESHOLD,
|
||||
HOLD_EXTENSION_TRIM_THRESHOLD,
|
||||
ADD_ON_ELIGIBILITY_THRESHOLD,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _sigmoid(x: float) -> float:
|
||||
"""Sigmoid function mapping any real to (0, 1)."""
|
||||
return 1.0 / (1.0 + math.exp(-x))
|
||||
|
||||
|
||||
def _zscore_to_01(z: Optional[float]) -> Optional[float]:
|
||||
"""Convert z-score to 0~1 via sigmoid (z=0 → 0.5)."""
|
||||
if z is None:
|
||||
return None
|
||||
return _sigmoid(z)
|
||||
|
||||
|
||||
class OverlayScorer:
|
||||
"""Compute final overlay score and derived metrics from a feature dict."""
|
||||
|
||||
def score(self, features: Dict) -> Dict:
|
||||
"""
|
||||
Given a features dict (output of FeatureBuilder.build_all_features),
|
||||
compute overlay_score, overlay_confidence, overlay_band, and decision hints.
|
||||
|
||||
Returns a dict suitable for storing in OverlayFeatureRecord.
|
||||
"""
|
||||
# Map each z-score to 0~1
|
||||
normalized = {
|
||||
"yahoo": _zscore_to_01(features.get("headline_burst_z")),
|
||||
"youtube": _zscore_to_01(features.get("youtube_influence_z")),
|
||||
"wikimedia": _zscore_to_01(features.get("wiki_attention_z")),
|
||||
"google_trends": _zscore_to_01(features.get("theme_heat_z")),
|
||||
"finra": _zscore_to_01(features.get("crowding_stress_z")),
|
||||
}
|
||||
|
||||
# Source presence: based on actual raw data, not z-score availability.
|
||||
# Z-scores require 2+ days of history; a source is "present" if it has any data at all.
|
||||
source_presence_mask = {
|
||||
"yahoo": (features.get("headline_count_24h") or 0) > 0,
|
||||
"youtube": (features.get("youtube_mentions_24h") or 0) > 0,
|
||||
"wikimedia": features.get("wiki_views_1d") is not None,
|
||||
"google_trends": normalized.get("google_trends") is not None,
|
||||
"finra": features.get("short_volume_ratio") is not None,
|
||||
}
|
||||
|
||||
# Weighted average across present sources
|
||||
total_weight = 0.0
|
||||
weighted_sum = 0.0
|
||||
present_count = 0
|
||||
for src, val in normalized.items():
|
||||
if val is not None:
|
||||
w = SOURCE_WEIGHTS.get(src, 0.0)
|
||||
weighted_sum += val * w
|
||||
total_weight += w
|
||||
present_count += 1
|
||||
|
||||
overlay_score = weighted_sum / total_weight if total_weight > 0 else 0.0
|
||||
|
||||
# Confidence based on number of active sources
|
||||
overlay_confidence = min(1.0, present_count * CONFIDENCE_PER_SOURCE)
|
||||
|
||||
# Band assignment (evaluate thresholds from high to low)
|
||||
overlay_band = "silent"
|
||||
for band_name, threshold in sorted(BAND_THRESHOLDS.items(), key=lambda kv: -kv[1]):
|
||||
if overlay_score >= threshold:
|
||||
overlay_band = band_name
|
||||
break
|
||||
|
||||
# Hold-extension hint
|
||||
if overlay_score >= HOLD_EXTENSION_EXTEND_THRESHOLD:
|
||||
hold_extension_hint = "extend"
|
||||
elif overlay_score <= HOLD_EXTENSION_TRIM_THRESHOLD:
|
||||
hold_extension_hint = "trim"
|
||||
else:
|
||||
hold_extension_hint = "neutral"
|
||||
|
||||
# Add-on eligibility
|
||||
add_on_eligibility = overlay_score >= ADD_ON_ELIGIBILITY_THRESHOLD
|
||||
|
||||
return {
|
||||
"overlay_score": round(overlay_score, 4),
|
||||
"overlay_confidence": round(overlay_confidence, 4),
|
||||
"overlay_band": overlay_band,
|
||||
"source_presence_mask": source_presence_mask,
|
||||
"hold_extension_hint": hold_extension_hint,
|
||||
"add_on_eligibility": add_on_eligibility,
|
||||
}
|
||||
@ -1,107 +0,0 @@
|
||||
"""
|
||||
Wikimedia REST API adapter - collect daily page view counts for watched pages.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import List, Optional
|
||||
|
||||
import aiohttp
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select, and_
|
||||
|
||||
from app.models.overlay_raw_event import OverlayWikiPageview
|
||||
from app.models.overlay_registry import WikiPageMap
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
WIKIMEDIA_BASE = "https://wikimedia.org/api/rest_v1/metrics/pageviews/per-article"
|
||||
USER_AGENT = "StockOracle/1.0 (github.com/stockoracle; contact@stockoracle.com)"
|
||||
|
||||
|
||||
class WikimediaAdapter:
|
||||
"""Collect Wikimedia page view data for pages mapped to tickers."""
|
||||
|
||||
async def _get_watched_pages(self, db: AsyncSession) -> List[WikiPageMap]:
|
||||
result = await db.execute(
|
||||
select(WikiPageMap).where(WikiPageMap.active == True)
|
||||
)
|
||||
return result.scalars().all()
|
||||
|
||||
async def fetch_pageviews(
|
||||
self, page_title: str, date: datetime, project: str = "en.wikipedia"
|
||||
) -> Optional[int]:
|
||||
"""Fetch daily pageviews for a specific page and date."""
|
||||
date_str = date.strftime("%Y%m%d")
|
||||
encoded_title = page_title.replace(" ", "_")
|
||||
url = (
|
||||
f"{WIKIMEDIA_BASE}/{project}/all-access/all-agents"
|
||||
f"/{encoded_title}/daily/{date_str}/{date_str}"
|
||||
)
|
||||
try:
|
||||
headers = {"User-Agent": USER_AGENT}
|
||||
async with aiohttp.ClientSession(headers=headers) as session:
|
||||
async with session.get(url, timeout=aiohttp.ClientTimeout(total=30)) as resp:
|
||||
if resp.status == 404:
|
||||
return None
|
||||
resp.raise_for_status()
|
||||
data = await resp.json()
|
||||
items = data.get("items", [])
|
||||
if items:
|
||||
return items[0].get("views", 0)
|
||||
return 0
|
||||
except Exception as e:
|
||||
logger.debug(f"Wikimedia fetch error for '{page_title}' on {date_str}: {e}")
|
||||
return None
|
||||
|
||||
async def collect(self, db: AsyncSession, days_back: int = 3) -> int:
|
||||
"""
|
||||
Collect pageviews for all active wiki page mappings.
|
||||
|
||||
Returns number of new records inserted.
|
||||
"""
|
||||
pages = await self._get_watched_pages(db)
|
||||
if not pages:
|
||||
logger.info("Wikimedia: no active wiki page mappings found")
|
||||
return 0
|
||||
|
||||
inserted = 0
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
for page in pages:
|
||||
for days_ago in range(1, days_back + 1):
|
||||
target_date = now - timedelta(days=days_ago)
|
||||
target_date = target_date.replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
|
||||
# Check duplicate
|
||||
existing = await db.execute(
|
||||
select(OverlayWikiPageview.id).where(
|
||||
and_(
|
||||
OverlayWikiPageview.page_title == page.wiki_page_title,
|
||||
OverlayWikiPageview.date == target_date,
|
||||
OverlayWikiPageview.project == "en.wikipedia",
|
||||
)
|
||||
)
|
||||
)
|
||||
if existing.first():
|
||||
continue
|
||||
|
||||
views = await self.fetch_pageviews(page.wiki_page_title, target_date)
|
||||
if views is None:
|
||||
continue
|
||||
|
||||
record = OverlayWikiPageview(
|
||||
page_title=page.wiki_page_title,
|
||||
date=target_date,
|
||||
project="en.wikipedia",
|
||||
views=views,
|
||||
mapped_symbol=page.symbol,
|
||||
)
|
||||
db.add(record)
|
||||
inserted += 1
|
||||
|
||||
if inserted:
|
||||
await db.commit()
|
||||
logger.info(f"Wikimedia: inserted {inserted} pageview records")
|
||||
|
||||
return inserted
|
||||
@ -1,170 +0,0 @@
|
||||
"""
|
||||
YouTube Data API v3 adapter - collect video mentions from whitelisted channels.
|
||||
Gracefully skips if YOUTUBE_API_KEY is not configured.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
import aiohttp
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.http_client import get_http_session
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.core.config import settings
|
||||
from app.models.overlay_raw_event import OverlayVideoEvent
|
||||
from app.models.overlay_registry import YouTubeChannelRegistry
|
||||
from app.services.overlay.entity_resolver import EntityResolver
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
YOUTUBE_SEARCH_URL = "https://www.googleapis.com/youtube/v3/search"
|
||||
YOUTUBE_VIDEOS_URL = "https://www.googleapis.com/youtube/v3/videos"
|
||||
|
||||
|
||||
class YouTubeAdapter:
|
||||
"""Collect YouTube video data from whitelisted channels."""
|
||||
|
||||
def __init__(self):
|
||||
self.api_key: str = getattr(settings, "YOUTUBE_API_KEY", "")
|
||||
self.resolver = EntityResolver()
|
||||
|
||||
@property
|
||||
def enabled(self) -> bool:
|
||||
return bool(self.api_key)
|
||||
|
||||
async def _get_channels(self, db: AsyncSession) -> List[YouTubeChannelRegistry]:
|
||||
result = await db.execute(
|
||||
select(YouTubeChannelRegistry).where(YouTubeChannelRegistry.active == True)
|
||||
)
|
||||
return result.scalars().all()
|
||||
|
||||
async def _search_channel_videos(
|
||||
self, channel_id: str, published_after: datetime
|
||||
) -> List[dict]:
|
||||
"""Search for recent videos from a channel via YouTube Data API."""
|
||||
params = {
|
||||
"part": "snippet",
|
||||
"channelId": channel_id,
|
||||
"type": "video",
|
||||
"order": "date",
|
||||
"publishedAfter": published_after.strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||
"maxResults": 20,
|
||||
"key": self.api_key,
|
||||
}
|
||||
try:
|
||||
session = await get_http_session()
|
||||
async with session.get(
|
||||
YOUTUBE_SEARCH_URL, params=params, timeout=aiohttp.ClientTimeout(total=30)
|
||||
) as resp:
|
||||
if resp.status == 403:
|
||||
logger.warning("YouTube API: quota exceeded or invalid key")
|
||||
return []
|
||||
resp.raise_for_status()
|
||||
data = await resp.json()
|
||||
return data.get("items", [])
|
||||
except Exception as e:
|
||||
logger.error(f"YouTube search error for channel {channel_id}: {e}")
|
||||
return []
|
||||
|
||||
async def _get_video_stats(self, video_ids: List[str]) -> Dict[str, Dict]:
|
||||
"""Fetch view/comment counts for a list of video IDs."""
|
||||
if not video_ids:
|
||||
return {}
|
||||
params = {
|
||||
"part": "statistics",
|
||||
"id": ",".join(video_ids),
|
||||
"key": self.api_key,
|
||||
}
|
||||
try:
|
||||
session = await get_http_session()
|
||||
async with session.get(
|
||||
YOUTUBE_VIDEOS_URL, params=params, timeout=aiohttp.ClientTimeout(total=30)
|
||||
) as resp:
|
||||
resp.raise_for_status()
|
||||
data = await resp.json()
|
||||
stats = {}
|
||||
for item in data.get("items", []):
|
||||
vid_id = item["id"]
|
||||
s = item.get("statistics", {})
|
||||
stats[vid_id] = {
|
||||
"view_count": int(s.get("viewCount", 0)),
|
||||
"comment_count": int(s.get("commentCount", 0)),
|
||||
}
|
||||
return stats
|
||||
except Exception as e:
|
||||
logger.error(f"YouTube stats fetch error: {e}")
|
||||
return {}
|
||||
|
||||
async def collect(self, db: AsyncSession, days_back: int = 2) -> int:
|
||||
"""
|
||||
Collect YouTube videos from whitelisted channels.
|
||||
|
||||
Gracefully returns 0 if API key is not configured.
|
||||
Returns number of new records inserted.
|
||||
"""
|
||||
if not self.enabled:
|
||||
logger.info("YouTube adapter: API key not configured, skipping")
|
||||
return 0
|
||||
|
||||
await self.resolver.load_aliases(db)
|
||||
channels = await self._get_channels(db)
|
||||
if not channels:
|
||||
logger.info("YouTube: no active channels in registry")
|
||||
return 0
|
||||
|
||||
published_after = datetime.now(timezone.utc) - timedelta(days=days_back)
|
||||
inserted = 0
|
||||
|
||||
for channel in channels:
|
||||
items = await self._search_channel_videos(channel.channel_id, published_after)
|
||||
video_ids = [
|
||||
item["id"]["videoId"]
|
||||
for item in items
|
||||
if isinstance(item.get("id"), dict) and "videoId" in item["id"]
|
||||
]
|
||||
stats = await self._get_video_stats(video_ids)
|
||||
|
||||
for item in items:
|
||||
vid_id = item.get("id", {}).get("videoId") if isinstance(item.get("id"), dict) else None
|
||||
if not vid_id:
|
||||
continue
|
||||
|
||||
# Check duplicate
|
||||
existing = await db.execute(
|
||||
select(OverlayVideoEvent.id).where(OverlayVideoEvent.video_id == vid_id)
|
||||
)
|
||||
if existing.first():
|
||||
continue
|
||||
|
||||
snippet = item.get("snippet", {})
|
||||
title = snippet.get("title", "")
|
||||
pub_at_str = snippet.get("publishedAt", "")
|
||||
try:
|
||||
pub_at = datetime.fromisoformat(pub_at_str.replace("Z", "+00:00"))
|
||||
except Exception:
|
||||
pub_at = datetime.now(timezone.utc)
|
||||
|
||||
stat = stats.get(vid_id, {})
|
||||
matched = self.resolver.resolve_from_title(title)
|
||||
|
||||
event = OverlayVideoEvent(
|
||||
video_id=vid_id,
|
||||
channel_id=channel.channel_id,
|
||||
title=title,
|
||||
view_count=stat.get("view_count", 0),
|
||||
comment_count=stat.get("comment_count", 0),
|
||||
published_at=pub_at,
|
||||
matched_symbols=matched,
|
||||
channel_weight=channel.channel_weight,
|
||||
)
|
||||
db.add(event)
|
||||
inserted += 1
|
||||
|
||||
if inserted:
|
||||
await db.commit()
|
||||
logger.info(f"YouTube: inserted {inserted} video events")
|
||||
|
||||
return inserted
|
||||
File diff suppressed because one or more lines are too long
@ -0,0 +1,111 @@
|
||||
"""
|
||||
Seed the company_aliases table with canonical names + common short forms for
|
||||
the TOP_50 watchlist. Idempotent — re-running is safe.
|
||||
|
||||
Usage:
|
||||
docker exec stock_oracle_api python -m scripts.seed_company_aliases
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import sys
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.core.database import AsyncSessionLocal
|
||||
from app.models.overlay_registry import CompanyAlias
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# (symbol, [(alias, alias_type, confidence), ...])
|
||||
# Confidence ≥ 0.7 is required for the EntityResolver to apply Stage 3
|
||||
# (alias-based) matches.
|
||||
_SEEDS = {
|
||||
"AAPL": [("Apple", "short", 0.95), ("Apple Inc", "canonical", 1.0)],
|
||||
"MSFT": [("Microsoft", "short", 0.95), ("Microsoft Corporation", "canonical", 1.0)],
|
||||
"NVDA": [("Nvidia", "short", 0.95), ("NVIDIA", "alias", 0.95), ("NVIDIA Corporation", "canonical", 1.0)],
|
||||
"AMZN": [("Amazon", "short", 0.9), ("Amazon.com", "canonical", 1.0)],
|
||||
"GOOGL": [("Google", "short", 0.85), ("Alphabet", "alias", 0.9), ("Alphabet Inc", "canonical", 1.0)],
|
||||
"META": [("Meta", "short", 0.8), ("Meta Platforms", "canonical", 1.0), ("Facebook", "alias", 0.8)],
|
||||
"TSLA": [("Tesla", "short", 0.95), ("Tesla Inc", "canonical", 1.0), ("Tesla Motors", "alias", 0.9)],
|
||||
"BRK.B": [("Berkshire Hathaway", "canonical", 1.0), ("Berkshire", "short", 0.9)],
|
||||
"JPM": [("JPMorgan", "short", 0.9), ("JPMorgan Chase", "canonical", 1.0), ("JP Morgan", "alias", 0.9)],
|
||||
"JNJ": [("Johnson & Johnson", "canonical", 1.0), ("J&J", "short", 0.9)],
|
||||
"V": [("Visa", "short", 0.9), ("Visa Inc", "canonical", 1.0)],
|
||||
"UNH": [("UnitedHealth", "short", 0.9), ("UnitedHealth Group", "canonical", 1.0)],
|
||||
"XOM": [("Exxon", "short", 0.9), ("ExxonMobil", "alias", 0.95), ("Exxon Mobil", "canonical", 1.0)],
|
||||
"PG": [("Procter & Gamble", "canonical", 1.0), ("P&G", "short", 0.9)],
|
||||
"MA": [("Mastercard", "short", 0.95), ("Mastercard Inc", "canonical", 1.0)],
|
||||
"HD": [("Home Depot", "short", 0.9), ("The Home Depot", "canonical", 1.0)],
|
||||
"CVX": [("Chevron", "short", 0.95), ("Chevron Corporation", "canonical", 1.0)],
|
||||
"LLY": [("Eli Lilly", "short", 0.95), ("Eli Lilly and Company", "canonical", 1.0), ("Lilly", "alias", 0.75)],
|
||||
"ABBV": [("AbbVie", "short", 0.95), ("AbbVie Inc", "canonical", 1.0)],
|
||||
"BAC": [("Bank of America", "canonical", 1.0), ("BofA", "short", 0.9)],
|
||||
"KO": [("Coca-Cola", "short", 0.9), ("The Coca-Cola Company", "canonical", 1.0), ("Coca Cola", "alias", 0.85)],
|
||||
"PEP": [("Pepsi", "short", 0.85), ("PepsiCo", "canonical", 1.0)],
|
||||
"AVGO": [("Broadcom", "short", 0.95), ("Broadcom Inc", "canonical", 1.0)],
|
||||
"COST": [("Costco", "short", 0.95), ("Costco Wholesale", "canonical", 1.0)],
|
||||
"WMT": [("Walmart", "short", 0.95), ("Wal-Mart", "alias", 0.9)],
|
||||
"MRK": [("Merck", "short", 0.85), ("Merck & Co", "canonical", 1.0)],
|
||||
"TMO": [("Thermo Fisher", "short", 0.9), ("Thermo Fisher Scientific", "canonical", 1.0)],
|
||||
"DIS": [("Disney", "short", 0.9), ("Walt Disney", "alias", 0.9), ("The Walt Disney Company", "canonical", 1.0)],
|
||||
"ACN": [("Accenture", "short", 0.95), ("Accenture plc", "canonical", 1.0)],
|
||||
"ABT": [("Abbott", "short", 0.85), ("Abbott Laboratories", "canonical", 1.0)],
|
||||
"VZ": [("Verizon", "short", 0.95), ("Verizon Communications", "canonical", 1.0)],
|
||||
"ADBE": [("Adobe", "short", 0.9), ("Adobe Inc", "canonical", 1.0)],
|
||||
"CRM": [("Salesforce", "short", 0.95), ("Salesforce.com", "canonical", 1.0)],
|
||||
"NFLX": [("Netflix", "short", 0.95), ("Netflix Inc", "canonical", 1.0)],
|
||||
"CMCSA": [("Comcast", "short", 0.95), ("Comcast Corporation", "canonical", 1.0)],
|
||||
"TXN": [("Texas Instruments", "short", 0.95), ("Texas Instruments Incorporated", "canonical", 1.0)],
|
||||
"CSCO": [("Cisco", "short", 0.9), ("Cisco Systems", "canonical", 1.0)],
|
||||
"NKE": [("Nike", "short", 0.9), ("Nike Inc", "canonical", 1.0)],
|
||||
"NEE": [("NextEra", "short", 0.9), ("NextEra Energy", "canonical", 1.0)],
|
||||
"AMD": [("AMD", "short", 0.85), ("Advanced Micro Devices", "canonical", 1.0)],
|
||||
"DHR": [("Danaher", "short", 0.95), ("Danaher Corporation", "canonical", 1.0)],
|
||||
"BMY": [("Bristol-Myers Squibb", "canonical", 1.0), ("Bristol Myers Squibb", "alias", 0.95), ("Bristol-Myers", "short", 0.85)],
|
||||
"QCOM": [("Qualcomm", "short", 0.95), ("Qualcomm Incorporated", "canonical", 1.0)],
|
||||
"T": [("AT&T", "short", 0.95), ("AT&T Inc", "canonical", 1.0)],
|
||||
"LOW": [("Lowe's", "short", 0.9), ("Lowe's Companies", "canonical", 1.0)],
|
||||
"PM": [("Philip Morris", "short", 0.95), ("Philip Morris International", "canonical", 1.0)],
|
||||
"HON": [("Honeywell", "short", 0.95), ("Honeywell International", "canonical", 1.0)],
|
||||
"ORCL": [("Oracle", "short", 0.85), ("Oracle Corporation", "canonical", 1.0)],
|
||||
"RTX": [("RTX", "short", 0.85), ("Raytheon", "alias", 0.85), ("Raytheon Technologies", "canonical", 1.0)],
|
||||
"UPS": [("UPS", "short", 0.9), ("United Parcel Service", "canonical", 1.0)],
|
||||
}
|
||||
|
||||
|
||||
async def seed():
|
||||
inserted = 0
|
||||
skipped = 0
|
||||
async with AsyncSessionLocal() as db:
|
||||
for symbol, aliases in _SEEDS.items():
|
||||
for value, alias_type, confidence in aliases:
|
||||
existing = await db.execute(
|
||||
select(CompanyAlias.id).where(
|
||||
CompanyAlias.symbol == symbol,
|
||||
CompanyAlias.alias_value == value,
|
||||
)
|
||||
)
|
||||
if existing.first():
|
||||
skipped += 1
|
||||
continue
|
||||
db.add(
|
||||
CompanyAlias(
|
||||
symbol=symbol,
|
||||
alias_type=alias_type,
|
||||
alias_value=value,
|
||||
confidence=confidence,
|
||||
active=True,
|
||||
)
|
||||
)
|
||||
inserted += 1
|
||||
await db.commit()
|
||||
logger.info(f"company_aliases seed: inserted={inserted}, skipped={skipped}")
|
||||
return inserted
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
n = asyncio.run(seed())
|
||||
sys.exit(0 if n >= 0 else 1)
|
||||
@ -0,0 +1,70 @@
|
||||
"""
|
||||
Unit tests for app.services.news.category_normalizer.
|
||||
|
||||
Covers vendor-map fallback, headline regex overrides, and the "drop general
|
||||
when specific exists" finalize rule.
|
||||
"""
|
||||
from app.services.news.category_normalizer import (
|
||||
UNIFIED_CATEGORIES,
|
||||
normalize_alpaca,
|
||||
normalize_finnhub,
|
||||
normalize_stocktwits,
|
||||
)
|
||||
|
||||
|
||||
def test_alpaca_upgrade_via_vendor_map():
|
||||
cats = normalize_alpaca(["Upgrades"], "Goldman raises Apple to Buy")
|
||||
assert "analyst_rating_upgrade" in cats
|
||||
|
||||
|
||||
def test_alpaca_downgrade_overrides_via_headline():
|
||||
# Vendor said "Analyst Color" (general), but headline mentions downgrade
|
||||
cats = normalize_alpaca(["Analyst Color"], "Morgan Stanley downgrades NVDA to Hold")
|
||||
assert "analyst_rating_downgrade" in cats
|
||||
assert "general" not in cats # specific present → general dropped
|
||||
|
||||
|
||||
def test_alpaca_fda_approval_vs_rejection_split():
|
||||
approval = normalize_alpaca(["FDA"], "FDA approves new oncology drug")
|
||||
rejection = normalize_alpaca(["FDA"], "FDA rejects PDUFA application; CRL issued")
|
||||
assert "fda_approval" in approval
|
||||
assert "fda_rejection" in rejection
|
||||
|
||||
|
||||
def test_finnhub_earnings_passthrough():
|
||||
cats = normalize_finnhub("earnings", "Apple beats Q1 estimates")
|
||||
assert "earnings_release" in cats
|
||||
|
||||
|
||||
def test_finnhub_unknown_category_falls_back_to_general():
|
||||
cats = normalize_finnhub("nonexistent", "")
|
||||
assert cats == ["general"]
|
||||
|
||||
|
||||
def test_stocktwits_default_general_with_no_keywords():
|
||||
assert normalize_stocktwits("Just bought some AAPL today") == ["general"]
|
||||
|
||||
|
||||
def test_stocktwits_buyback_keyword_picks_up_category():
|
||||
cats = normalize_stocktwits("Apple announces $90B share repurchase program")
|
||||
assert "buyback" in cats
|
||||
|
||||
|
||||
def test_unified_categories_contains_all_mapped_targets():
|
||||
"""Every vendor map target value must be a known unified category."""
|
||||
from app.services.news.category_normalizer import (
|
||||
ALPACA_BENZINGA_MAP,
|
||||
FINNHUB_MAP,
|
||||
)
|
||||
targets = set(ALPACA_BENZINGA_MAP.values()) | set(FINNHUB_MAP.values())
|
||||
assert targets <= UNIFIED_CATEGORIES, f"Stray targets: {targets - UNIFIED_CATEGORIES}"
|
||||
|
||||
|
||||
def test_no_categories_yields_general_singleton():
|
||||
cats = normalize_alpaca(None, "")
|
||||
assert cats == ["general"]
|
||||
|
||||
|
||||
def test_management_change_pattern():
|
||||
cats = normalize_alpaca(["News"], "Acme Corp CEO Jane Smith steps down")
|
||||
assert "management_change" in cats
|
||||
@ -0,0 +1,134 @@
|
||||
"""
|
||||
Unit tests for vendor → row dict transforms in headline_ingest_service.
|
||||
|
||||
These exercise the data-shape contract without hitting the database.
|
||||
"""
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from app.services.news.alpaca_news_client import AlpacaNewsArticle
|
||||
from app.services.news.headline_ingest_service import (
|
||||
alpaca_articles_to_rows,
|
||||
finnhub_articles_to_rows,
|
||||
stocktwits_messages_to_rows,
|
||||
)
|
||||
|
||||
|
||||
def _alpaca_article(symbols, headline="Test", source="benzinga"):
|
||||
return AlpacaNewsArticle(
|
||||
id=12345,
|
||||
headline=headline,
|
||||
summary="summary",
|
||||
url="https://example.com/n/12345",
|
||||
author="Author",
|
||||
created_at=datetime(2026, 4, 25, 13, 30, tzinfo=timezone.utc),
|
||||
updated_at=datetime(2026, 4, 25, 13, 30, tzinfo=timezone.utc),
|
||||
source=source,
|
||||
symbols=symbols,
|
||||
images=[],
|
||||
content=None,
|
||||
)
|
||||
|
||||
|
||||
def test_alpaca_emits_one_row_per_ticker_with_primary_flag():
|
||||
art = _alpaca_article(["AAPL", "MSFT", "NVDA"], headline="Tech rally continues")
|
||||
rows = alpaca_articles_to_rows([art])
|
||||
assert len(rows) == 3
|
||||
tickers = [r["ticker"] for r in rows]
|
||||
assert tickers == ["AAPL", "MSFT", "NVDA"]
|
||||
primary_flags = [r["is_primary"] for r in rows]
|
||||
assert primary_flags == [True, False, False]
|
||||
# tickers_all preserves the full set on every row
|
||||
for r in rows:
|
||||
assert r["tickers_all"] == ["AAPL", "MSFT", "NVDA"]
|
||||
assert r["source"] == "alpaca_benzinga"
|
||||
assert r["source_id"] == "12345"
|
||||
|
||||
|
||||
def test_alpaca_skips_articles_with_no_symbols():
|
||||
art = _alpaca_article([])
|
||||
assert alpaca_articles_to_rows([art]) == []
|
||||
|
||||
|
||||
def test_alpaca_published_at_is_tz_aware_utc():
|
||||
rows = alpaca_articles_to_rows([_alpaca_article(["AAPL"])])
|
||||
assert rows[0]["published_at"].tzinfo is not None
|
||||
assert rows[0]["ingested_at"].tzinfo is not None
|
||||
|
||||
|
||||
def test_alpaca_normalizes_categories_with_headline_override():
|
||||
art = _alpaca_article(["AAPL"], headline="Goldman downgrades AAPL to Sell")
|
||||
rows = alpaca_articles_to_rows([art])
|
||||
assert "analyst_rating_downgrade" in rows[0]["categories"]
|
||||
|
||||
|
||||
def test_finnhub_payload_to_rows():
|
||||
payload = [{
|
||||
"id": 999,
|
||||
"datetime": 1745596800, # 2025-04-25 16:00 UTC
|
||||
"headline": "Apple posts strong earnings",
|
||||
"summary": "Beat on EPS and revenue",
|
||||
"url": "https://finn/news/999",
|
||||
"category": "earnings",
|
||||
}]
|
||||
rows = finnhub_articles_to_rows("AAPL", payload)
|
||||
assert len(rows) == 1
|
||||
r = rows[0]
|
||||
assert r["source"] == "finnhub"
|
||||
assert r["source_id"] == "999"
|
||||
assert r["ticker"] == "AAPL"
|
||||
assert r["is_primary"] is True
|
||||
assert "earnings_release" in r["categories"]
|
||||
assert r["published_at"].tzinfo is not None
|
||||
|
||||
|
||||
def test_finnhub_skips_rows_missing_id_or_timestamp():
|
||||
payload = [
|
||||
{"id": 1, "datetime": None, "headline": "x"}, # missing ts
|
||||
{"datetime": 100, "headline": "x"}, # missing id
|
||||
]
|
||||
assert finnhub_articles_to_rows("AAPL", payload) == []
|
||||
|
||||
|
||||
def test_stocktwits_bullish_score_and_dedup_key():
|
||||
msg = {
|
||||
"id": 555,
|
||||
"created_at": "2026-04-25T13:00:00Z",
|
||||
"body": "AAPL to the moon",
|
||||
"entities": {"sentiment": {"basic": "Bullish"}},
|
||||
"symbols": [{"symbol": "AAPL"}, {"symbol": "MSFT"}],
|
||||
}
|
||||
rows = stocktwits_messages_to_rows("AAPL", [msg])
|
||||
assert len(rows) == 1
|
||||
r = rows[0]
|
||||
assert r["source"] == "stocktwits"
|
||||
assert r["source_id"] == "555"
|
||||
assert r["raw_sentiment"] == 1.0
|
||||
assert "AAPL" in r["tickers_all"] and "MSFT" in r["tickers_all"]
|
||||
assert r["published_at"].tzinfo is not None
|
||||
|
||||
|
||||
def test_stocktwits_bearish_score_negative():
|
||||
msg = {
|
||||
"id": 1,
|
||||
"created_at": "2026-04-25T13:00:00Z",
|
||||
"body": "shorting this dog",
|
||||
"entities": {"sentiment": {"basic": "Bearish"}},
|
||||
}
|
||||
rows = stocktwits_messages_to_rows("AAPL", [msg])
|
||||
assert rows[0]["raw_sentiment"] == -1.0
|
||||
|
||||
|
||||
def test_stocktwits_neutral_returns_none_sentiment():
|
||||
msg = {
|
||||
"id": 2,
|
||||
"created_at": "2026-04-25T13:00:00Z",
|
||||
"body": "watching this stock today",
|
||||
"entities": {},
|
||||
}
|
||||
rows = stocktwits_messages_to_rows("AAPL", [msg])
|
||||
assert rows[0]["raw_sentiment"] is None
|
||||
|
||||
|
||||
def test_stocktwits_skips_empty_body():
|
||||
msg = {"id": 3, "created_at": "2026-04-25T13:00:00Z", "body": " "}
|
||||
assert stocktwits_messages_to_rows("AAPL", [msg]) == []
|
||||
@ -0,0 +1,95 @@
|
||||
"""
|
||||
Unit tests for app.services.news.session_window.
|
||||
|
||||
Critical invariant: `post` must end at the next trading day's premarket
|
||||
start (04:00 ET) — NOT next session's open (09:30 ET) — to avoid 4-hour
|
||||
overlap with the next session's `premarket` window.
|
||||
"""
|
||||
from datetime import date, datetime, time
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services.news.session_window import session_window
|
||||
|
||||
ET = ZoneInfo("America/New_York")
|
||||
UTC = ZoneInfo("UTC")
|
||||
|
||||
|
||||
def _et(d: date, t: time) -> datetime:
|
||||
return datetime.combine(d, t, tzinfo=ET).astimezone(UTC)
|
||||
|
||||
|
||||
def test_premarket_bounds_basic_weekday():
|
||||
# Wednesday 2026-04-22 (Tue 2026-04-21 prev day, both weekdays)
|
||||
target = date(2026, 4, 22)
|
||||
start, end = session_window(target, "premarket")
|
||||
# Premarket starts at the previous session's *close*, not 04:00
|
||||
expected_start = _et(date(2026, 4, 21), time(16, 0))
|
||||
expected_end = _et(target, time(9, 30))
|
||||
assert start == expected_start
|
||||
assert end == expected_end
|
||||
|
||||
|
||||
def test_intraday_bounds_basic_weekday():
|
||||
target = date(2026, 4, 22)
|
||||
start, end = session_window(target, "intraday")
|
||||
assert start == _et(target, time(9, 30))
|
||||
assert end == _et(target, time(16, 0))
|
||||
|
||||
|
||||
def test_post_ends_at_next_premarket_start_not_next_open():
|
||||
"""post must NOT extend to next session's 09:30 — that would overlap T+1 premarket."""
|
||||
target = date(2026, 4, 22) # Wed; next trading day Thu 2026-04-23
|
||||
_, end = session_window(target, "post")
|
||||
expected_end = _et(date(2026, 4, 23), time(4, 0))
|
||||
assert end == expected_end, f"post should end at next premarket start (04:00 ET), got {end.astimezone(ET)}"
|
||||
|
||||
|
||||
def test_post_and_next_premarket_are_disjoint():
|
||||
target = date(2026, 4, 22)
|
||||
_, post_end = session_window(target, "post")
|
||||
next_pm_start, _ = session_window(date(2026, 4, 23), "premarket")
|
||||
# premarket starts at *previous close* (16:00), so they don't share boundary;
|
||||
# the contract is post_end <= 04:00 ET while next premarket fully overlaps that
|
||||
# zone. We assert the no-double-count guarantee: a 04:30 ET headline lands in
|
||||
# premarket (Thu) only, not in post (Wed).
|
||||
sample_dt = _et(date(2026, 4, 23), time(4, 30))
|
||||
assert sample_dt >= post_end, "04:30 ET headline should be past post-window end"
|
||||
assert sample_dt > next_pm_start # premarket window includes 04:30
|
||||
|
||||
|
||||
def test_full_session_spans_prev_close_to_next_premarket_start():
|
||||
target = date(2026, 4, 22)
|
||||
start, end = session_window(target, "full_session")
|
||||
assert start == _et(date(2026, 4, 21), time(16, 0))
|
||||
assert end == _et(date(2026, 4, 23), time(4, 0))
|
||||
|
||||
|
||||
def test_premarket_after_weekend_uses_friday_close():
|
||||
"""Monday's premarket starts at the previous *Friday*'s close, not Sunday."""
|
||||
target = date(2026, 4, 27) # Monday
|
||||
start, _ = session_window(target, "premarket")
|
||||
expected_start = _et(date(2026, 4, 24), time(16, 0)) # Friday
|
||||
assert start == expected_start
|
||||
|
||||
|
||||
def test_friday_post_extends_to_monday_premarket_start():
|
||||
"""Friday's `post` window must extend across the weekend to Monday 04:00 ET."""
|
||||
target = date(2026, 4, 24) # Friday
|
||||
_, end = session_window(target, "post")
|
||||
expected_end = _et(date(2026, 4, 27), time(4, 0)) # Monday 04:00
|
||||
assert end == expected_end
|
||||
|
||||
|
||||
def test_unknown_window_raises():
|
||||
with pytest.raises(ValueError):
|
||||
session_window(date(2026, 4, 22), "lunch") # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_returned_datetimes_are_utc_aware():
|
||||
start, end = session_window(date(2026, 4, 22), "premarket")
|
||||
assert start.tzinfo is not None
|
||||
assert end.tzinfo is not None
|
||||
assert start.utcoffset().total_seconds() == 0 # type: ignore[union-attr]
|
||||
assert end.utcoffset().total_seconds() == 0 # type: ignore[union-attr]
|
||||
Loading…
Reference in New Issue