""" Attention API endpoints — event-centric attention data for backtesting. Static routes are registered before parameterized routes to avoid path shadowing. Main endpoint: GET /api/v1/attention/event/{ticker}?event_date=YYYY-MM-DD Admin endpoints: GET /api/v1/attention/entity/{ticker} POST /api/v1/attention/admin/resolve/{ticker} POST /api/v1/attention/admin/collect/wiki/{ticker}?event_date=... POST /api/v1/attention/admin/collect/gdelt/{ticker}?event_date=... """ import logging from datetime import date, timedelta from fastapi import APIRouter, Depends, HTTPException, Query from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncSession from app.core.database import get_db from app.models.attention import AttentionFeaturesDaily, CompanyEntityMap, GdeltArticleRaw from app.services.attention.gdelt_collector import GDELT_EARLIEST_DATE from app.schemas.attention import ( CollectionStatusResponse, EntityInfo, EntityResolveResponse, EventAttentionResponse, NewsFeatures, WikiFeatures, ) from app.services.attention.entity_resolver import resolve_entity from app.services.attention.feature_materializer import materialize_features from app.services.attention.gdelt_collector import collect_gdelt_articles from app.services.attention.wiki_collector import collect_wiki_pageviews logger = logging.getLogger(__name__) router = APIRouter() def _entity_to_info(entity: CompanyEntityMap) -> EntityInfo: return EntityInfo( ticker=entity.ticker, canonical_name=entity.canonical_name, wiki_title=entity.wiki_title, gdelt_query=entity.gdelt_query, aliases=entity.aliases_json or [], resolver_confidence=entity.resolver_confidence or 0.0, is_manual_override=entity.is_manual_override or False, ) def _features_to_response( ticker: str, event_date: date, entity: CompanyEntityMap, features: AttentionFeaturesDaily, gdelt_status: str = "not_collected", ) -> EventAttentionResponse: # Compute baseline_10d from spike (inverse: baseline = views / spike) baseline_10d = None if features.wiki_spike_10d and features.wiki_views and features.wiki_spike_10d > 0: baseline_10d = features.wiki_views / features.wiki_spike_10d return EventAttentionResponse( ticker=ticker, event_date=event_date, entity=_entity_to_info(entity), wiki=WikiFeatures( views=features.wiki_views, baseline_10d=round(baseline_10d, 2) if baseline_10d else None, spike_10d=round(features.wiki_spike_10d, 4) if features.wiki_spike_10d is not None else None, zscore_20d=round(features.wiki_zscore_20d, 4) if features.wiki_zscore_20d is not None else None, ), news=NewsFeatures( article_count_1d=features.gdelt_article_count_1d or 0, article_count_3d=features.gdelt_article_count_3d or 0, unique_domains_3d=features.gdelt_unique_domains_3d or 0, us_article_count_3d=features.gdelt_us_article_count_3d or 0, gdelt_status=gdelt_status, ), metadata={ "wiki_title": entity.wiki_title, "resolver_confidence": entity.resolver_confidence, }, ) # =========================================================================== # Admin / utility routes (static — must come before parameterized routes) # =========================================================================== @router.post( "/admin/resolve/{ticker}", response_model=EntityResolveResponse, summary="Resolve ticker → canonical entity", description=( "Maps a ticker symbol to a canonical company entity by looking up the company name, " "normalizing it, and validating against Wikipedia. Stores the result (canonical name, " "wiki_title, gdelt_query) in `company_entity_map`.\n\n" "Skips re-resolution if `is_manual_override` is set. " "If the company name in the DB is a placeholder (e.g. 'AMZN Corporation'), " "falls back to SEC company_tickers.json to fetch the real name and updates the DB." ), tags=["attention-admin"], ) async def admin_resolve_entity( ticker: str, db: AsyncSession = Depends(get_db), ) -> EntityResolveResponse: ticker = ticker.upper() # Check for existing manual override existing_result = await db.execute( select(CompanyEntityMap).where(CompanyEntityMap.ticker == ticker) ) existing = existing_result.scalars().first() if existing and existing.is_manual_override: return EntityResolveResponse( ticker=ticker, entity=_entity_to_info(existing), status="manual_override_skipped", message="Manual override is active — resolution skipped.", ) try: entity = await resolve_entity(db, ticker) status = "resolved" message = f"Entity resolved: wiki_title={entity.wiki_title!r} confidence={entity.resolver_confidence:.2f}" except ValueError as exc: raise HTTPException(status_code=404, detail=str(exc)) except Exception as exc: logger.error("Entity resolution failed for %s: %s", ticker, exc) raise HTTPException(status_code=500, detail=f"Resolution failed: {exc}") return EntityResolveResponse( ticker=ticker, entity=_entity_to_info(entity), status=status, message=message, ) @router.post( "/admin/collect/wiki/{ticker}", response_model=CollectionStatusResponse, summary="Collect Wikipedia pageviews for an event date", description=( "Fetches daily Wikipedia pageview counts for the ticker's canonical wiki_title, " "covering `event_date` and enough lookback days (≥20) to compute spike and z-score. " "Safe to call on-demand — Wikipedia API has no meaningful rate limit for this use.\n\n" "Requires entity resolution to have been run first (`wiki_title` must be set)." ), tags=["attention-admin"], ) async def admin_collect_wiki( ticker: str, event_date: date = Query(..., description="Event date in YYYY-MM-DD format"), db: AsyncSession = Depends(get_db), ) -> CollectionStatusResponse: ticker = ticker.upper() try: count = await collect_wiki_pageviews(db, ticker, event_date) except ValueError as exc: raise HTTPException(status_code=422, detail=str(exc)) except Exception as exc: logger.error("Wiki collection failed for %s: %s", ticker, exc) raise HTTPException(status_code=500, detail=f"Wiki collection failed: {exc}") return CollectionStatusResponse( ticker=ticker, source="wiki", records_collected=count, date_range={"event_date": str(event_date)}, status="success", ) @router.post( "/admin/collect/gdelt/{ticker}", response_model=CollectionStatusResponse, summary="Collect GDELT news articles for an event date", description=( "Fetches news articles from GDELT V2 DOC API for the window `event_date ± 1 day`.\n\n" "**Coverage**: 2017-01-01 onwards. Requests for earlier dates return 0 immediately.\n\n" "**Rate limit**: GDELT enforces a global per-IP quota. This endpoint is protected by " "a process-wide lock (10s minimum interval) and retries with exponential backoff " "(30s → 60s → 120s) on 429 responses.\n\n" "⚠️ **Call this endpoint from a scheduler only** — never trigger it in response to " "user requests. Concurrent or rapid calls will exhaust the IP quota and cause " "temporary bans. The main `/event/{ticker}` endpoint intentionally does NOT collect " "GDELT on-demand for this reason." ), tags=["attention-admin"], ) async def admin_collect_gdelt( ticker: str, event_date: date = Query(..., description="Event date in YYYY-MM-DD format"), db: AsyncSession = Depends(get_db), ) -> CollectionStatusResponse: ticker = ticker.upper() try: count = await collect_gdelt_articles(db, ticker, event_date) except ValueError as exc: raise HTTPException(status_code=422, detail=str(exc)) except Exception as exc: logger.error("GDELT collection failed for %s: %s", ticker, exc) raise HTTPException(status_code=500, detail=f"GDELT collection failed: {exc}") return CollectionStatusResponse( ticker=ticker, source="gdelt", records_collected=count, date_range={"event_date": str(event_date)}, status="success", ) # =========================================================================== # Parameterized entity route (before /event/ to avoid shadowing) # =========================================================================== @router.get( "/entity/{ticker}", response_model=EntityResolveResponse, summary="Get entity mapping for a ticker", description=( "Returns the stored entity mapping for a ticker: canonical name, Wikipedia title, " "GDELT query string, and resolver confidence score.\n\n" "Returns 404 if no mapping exists — run `POST /admin/resolve/{ticker}` first." ), ) async def get_entity( ticker: str, db: AsyncSession = Depends(get_db), ) -> EntityResolveResponse: ticker = ticker.upper() result = await db.execute( select(CompanyEntityMap).where(CompanyEntityMap.ticker == ticker) ) entity = result.scalars().first() if not entity: raise HTTPException( status_code=404, detail=f"No entity mapping found for {ticker}. POST /admin/resolve/{ticker} to create one.", ) return EntityResolveResponse( ticker=ticker, entity=_entity_to_info(entity), status="exists", message="Entity mapping retrieved from database.", ) # =========================================================================== # Main event attention endpoint # =========================================================================== @router.get( "/event/{ticker}", response_model=EventAttentionResponse, summary="Get attention features for a ticker on an event date", description=( "Returns Wikipedia pageview spike/z-score and GDELT news article counts " "for the given ticker centered on `event_date`.\n\n" "**Wikipedia data** is collected on-demand if not in DB. " "**GDELT data** is never collected on-demand — it must be pre-populated via " "`POST /admin/collect/gdelt/{ticker}` (scheduler). " "Check `news.gdelt_status` in the response to understand data availability:\n\n" "- `collected` — GDELT was collected; counts are accurate (0 means genuinely no articles)\n" "- `not_collected` — scheduler has not run for this date yet\n" "- `not_available` — event date is before GDELT V2 coverage (2017-01-01)\n\n" "If no entity mapping exists, resolution runs automatically before collection." ), ) async def get_event_attention( ticker: str, event_date: date = Query(..., description="Event date in YYYY-MM-DD format"), db: AsyncSession = Depends(get_db), ) -> EventAttentionResponse: ticker = ticker.upper() # 1. Get or resolve entity entity_result = await db.execute( select(CompanyEntityMap).where(CompanyEntityMap.ticker == ticker) ) entity = entity_result.scalars().first() if not entity: logger.info("No entity mapping for %s — auto-resolving", ticker) try: entity = await resolve_entity(db, ticker) except ValueError as exc: raise HTTPException(status_code=404, detail=str(exc)) except Exception as exc: logger.error("Auto entity resolution failed for %s: %s", ticker, exc) raise HTTPException(status_code=500, detail=f"Entity resolution failed: {exc}") # 2. Check if features already exist in DB features_result = await db.execute( select(AttentionFeaturesDaily).where( AttentionFeaturesDaily.ticker == ticker, AttentionFeaturesDaily.date == event_date, ) ) features = features_result.scalars().first() if features is None: # 3. On-demand collection + materialization # NOTE: GDELT is intentionally excluded here — it must be collected via # the scheduler (POST /admin/collect/gdelt/{ticker}) to avoid IP rate bans. # This endpoint only collects Wikipedia data on-demand. logger.info("No features for %s on %s — collecting wiki on-demand", ticker, event_date) if entity.wiki_title: try: await collect_wiki_pageviews(db, ticker, event_date) except Exception as exc: logger.warning("Wiki collection failed for %s: %s", ticker, exc) try: features = await materialize_features(db, ticker, event_date) except Exception as exc: logger.error("Feature materialization failed for %s: %s", ticker, exc) raise HTTPException(status_code=500, detail=f"Feature materialization failed: {exc}") # Re-fetch entity in case it was updated during resolution entity_result2 = await db.execute( select(CompanyEntityMap).where(CompanyEntityMap.ticker == ticker) ) entity = entity_result2.scalars().first() # Determine GDELT collection status for the client if event_date < GDELT_EARLIEST_DATE: gdelt_status = "not_available" else: window_start = event_date - timedelta(days=1) window_end = event_date + timedelta(days=1) collected_count = await db.execute( select(func.count()).where( GdeltArticleRaw.matched_ticker == ticker, func.date(GdeltArticleRaw.published_at) >= window_start, func.date(GdeltArticleRaw.published_at) <= window_end, ) ) gdelt_status = "collected" if (collected_count.scalar() or 0) > 0 else "not_collected" return _features_to_response(ticker, event_date, entity, features, gdelt_status)