stock-oracle/app/api/v1/endpoints/attention.py

"""
Attention API endpoints — event-centric attention data for backtesting.

Static routes are registered before parameterized routes to avoid path shadowing.

Main endpoint:
  GET /api/v1/attention/event/{ticker}?event_date=YYYY-MM-DD

Admin endpoints:
  GET  /api/v1/attention/entity/{ticker}
  POST /api/v1/attention/admin/resolve/{ticker}
  POST /api/v1/attention/admin/collect/wiki/{ticker}?event_date=...
  POST /api/v1/attention/admin/collect/gdelt/{ticker}?event_date=...
"""

import asyncio
import logging
from datetime import date, timedelta

from fastapi import APIRouter, Depends, HTTPException, Query, Response
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession

from app.core.database import get_db
from app.utils.cache import with_cache
from app.models.attention import AttentionFeaturesDaily, CompanyEntityMap, GdeltArticleRaw
from app.services.attention.gdelt_collector import GDELT_EARLIEST_DATE
from app.schemas.attention import (
    CollectionStatusResponse,
    EntityInfo,
    EntityResolveResponse,
    EventAttentionResponse,
    NewsFeatures,
    WikiFeatures,
)
from app.services.attention.entity_resolver import resolve_entity
from app.services.attention.feature_materializer import materialize_features
from app.services.attention.gdelt_collector import collect_gdelt_articles
from app.services.attention.wiki_collector import collect_wiki_pageviews

logger = logging.getLogger(__name__)
router = APIRouter()

# Fast-fail gate for /event/{ticker} — blocks bulk scan floods that triggered
# wiki 429 bursts and contributed to container memory pressure.
_EVENT_SEMAPHORE = asyncio.Semaphore(8)
_EVENT_SEMAPHORE_WAIT = 10


def _entity_to_info(entity: CompanyEntityMap) -> EntityInfo:
    return EntityInfo(
        ticker=entity.ticker,
        canonical_name=entity.canonical_name,
        wiki_title=entity.wiki_title,
        gdelt_query=entity.gdelt_query,
        aliases=entity.aliases_json or [],
        resolver_confidence=entity.resolver_confidence or 0.0,
        is_manual_override=entity.is_manual_override or False,
    )


def _features_to_response(
    ticker: str,
    event_date: date,
    entity: CompanyEntityMap,
    features: AttentionFeaturesDaily,
    gdelt_status: str = "not_collected",
) -> EventAttentionResponse:
    # Compute baseline_10d from spike (inverse: baseline = views / spike)
    baseline_10d = None
    if features.wiki_spike_10d and features.wiki_views and features.wiki_spike_10d > 0:
        baseline_10d = features.wiki_views / features.wiki_spike_10d

    return EventAttentionResponse(
        ticker=ticker,
        event_date=event_date,
        entity=_entity_to_info(entity),
        wiki=WikiFeatures(
            views=features.wiki_views,
            baseline_10d=round(baseline_10d, 2) if baseline_10d else None,
            spike_10d=round(features.wiki_spike_10d, 4) if features.wiki_spike_10d is not None else None,
            zscore_20d=round(features.wiki_zscore_20d, 4) if features.wiki_zscore_20d is not None else None,
        ),
        news=NewsFeatures(
            article_count_1d=features.gdelt_article_count_1d or 0,
            article_count_3d=features.gdelt_article_count_3d or 0,
            unique_domains_3d=features.gdelt_unique_domains_3d or 0,
            us_article_count_3d=features.gdelt_us_article_count_3d or 0,
            gdelt_status=gdelt_status,
        ),
        metadata={
            "wiki_title": entity.wiki_title,
            "resolver_confidence": entity.resolver_confidence,
        },
    )


# ===========================================================================
# Admin / utility routes (static — must come before parameterized routes)
# ===========================================================================


@router.post(
    "/admin/resolve/{ticker}",
    response_model=EntityResolveResponse,
    summary="Resolve ticker → canonical entity",
    description=(
        "Maps a ticker symbol to a canonical company entity by looking up the company name, "
        "normalizing it, and validating against Wikipedia. Stores the result (canonical name, "
        "wiki_title, gdelt_query) in `company_entity_map`.\n\n"
        "Skips re-resolution if `is_manual_override` is set. "
        "If the company name in the DB is a placeholder (e.g. 'AMZN Corporation'), "
        "falls back to SEC company_tickers.json to fetch the real name and updates the DB."
    ),
    tags=["attention-admin"],
)
async def admin_resolve_entity(
    ticker: str,
    db: AsyncSession = Depends(get_db),
) -> EntityResolveResponse:
    ticker = ticker.upper()

    # Check for existing manual override
    existing_result = await db.execute(
        select(CompanyEntityMap).where(CompanyEntityMap.ticker == ticker)
    )
    existing = existing_result.scalars().first()
    if existing and existing.is_manual_override:
        return EntityResolveResponse(
            ticker=ticker,
            entity=_entity_to_info(existing),
            status="manual_override_skipped",
            message="Manual override is active — resolution skipped.",
        )

    try:
        entity = await resolve_entity(db, ticker)
        status = "resolved"
        message = f"Entity resolved: wiki_title={entity.wiki_title!r} confidence={entity.resolver_confidence:.2f}"
    except ValueError as exc:
        raise HTTPException(status_code=404, detail=str(exc))
    except Exception as exc:
        logger.error("Entity resolution failed for %s: %s", ticker, exc)
        raise HTTPException(status_code=500, detail=f"Resolution failed: {exc}")

    return EntityResolveResponse(
        ticker=ticker,
        entity=_entity_to_info(entity),
        status=status,
        message=message,
    )


@router.post(
    "/admin/collect/wiki/{ticker}",
    response_model=CollectionStatusResponse,
    summary="Collect Wikipedia pageviews for an event date",
    description=(
        "Fetches daily Wikipedia pageview counts for the ticker's canonical wiki_title, "
        "covering `event_date` and enough lookback days (≥20) to compute spike and z-score. "
        "Safe to call on-demand — Wikipedia API has no meaningful rate limit for this use.\n\n"
        "Requires entity resolution to have been run first (`wiki_title` must be set)."
    ),
    tags=["attention-admin"],
)
async def admin_collect_wiki(
    ticker: str,
    event_date: date = Query(..., description="Event date in YYYY-MM-DD format"),
    db: AsyncSession = Depends(get_db),
) -> CollectionStatusResponse:
    ticker = ticker.upper()
    try:
        count = await collect_wiki_pageviews(db, ticker, event_date)
    except ValueError as exc:
        raise HTTPException(status_code=422, detail=str(exc))
    except Exception as exc:
        logger.error("Wiki collection failed for %s: %s", ticker, exc)
        raise HTTPException(status_code=500, detail=f"Wiki collection failed: {exc}")

    return CollectionStatusResponse(
        ticker=ticker,
        source="wiki",
        records_collected=count,
        date_range={"event_date": str(event_date)},
        status="success",
    )


@router.post(
    "/admin/collect/gdelt/{ticker}",
    response_model=CollectionStatusResponse,
    summary="Collect GDELT news articles for an event date",
    description=(
        "Fetches news articles from GDELT V2 DOC API for the window `event_date ± 1 day`.\n\n"
        "**Coverage**: 2017-01-01 onwards. Requests for earlier dates return 0 immediately.\n\n"
        "**Rate limit**: GDELT enforces a global per-IP quota. This endpoint is protected by "
        "a process-wide lock (10s minimum interval) and retries with exponential backoff "
        "(30s → 60s → 120s) on 429 responses.\n\n"
        "⚠️ **Call this endpoint from a scheduler only** — never trigger it in response to "
        "user requests. Concurrent or rapid calls will exhaust the IP quota and cause "
        "temporary bans. The main `/event/{ticker}` endpoint intentionally does NOT collect "
        "GDELT on-demand for this reason."
    ),
    tags=["attention-admin"],
)
async def admin_collect_gdelt(
    ticker: str,
    event_date: date = Query(..., description="Event date in YYYY-MM-DD format"),
    db: AsyncSession = Depends(get_db),
) -> CollectionStatusResponse:
    ticker = ticker.upper()
    try:
        count = await collect_gdelt_articles(db, ticker, event_date)
    except ValueError as exc:
        raise HTTPException(status_code=422, detail=str(exc))
    except Exception as exc:
        logger.error("GDELT collection failed for %s: %s", ticker, exc)
        raise HTTPException(status_code=500, detail=f"GDELT collection failed: {exc}")

    return CollectionStatusResponse(
        ticker=ticker,
        source="gdelt",
        records_collected=count,
        date_range={"event_date": str(event_date)},
        status="success",
    )


# ===========================================================================
# Parameterized entity route (before /event/ to avoid shadowing)
# ===========================================================================


@router.get(
    "/entity/{ticker}",
    response_model=EntityResolveResponse,
    summary="Get entity mapping for a ticker",
    description="""
    Returns the stored entity mapping for a ticker: canonical name, Wikipedia title,
    GDELT query string, and resolver confidence score.

    Returns **404** if no mapping exists — run `POST /admin/resolve/{ticker}` first.

    **Example**: `GET /attention/entity/AAPL`
    """,
)
@with_cache(namespace="attention:entity", ttl=3600, key_params=["ticker"])
async def get_entity(
    ticker: str,
    response: Response = None,
    db: AsyncSession = Depends(get_db),
) -> EntityResolveResponse:
    ticker = ticker.upper()
    result = await db.execute(
        select(CompanyEntityMap).where(CompanyEntityMap.ticker == ticker)
    )
    entity = result.scalars().first()
    if not entity:
        raise HTTPException(
            status_code=404,
            detail=f"No entity mapping found for {ticker}. POST /admin/resolve/{ticker} to create one.",
        )
    return EntityResolveResponse(
        ticker=ticker,
        entity=_entity_to_info(entity),
        status="exists",
        message="Entity mapping retrieved from database.",
    )


# ===========================================================================
# Main event attention endpoint
# ===========================================================================


@router.get(
    "/event/{ticker}",
    response_model=EventAttentionResponse,
    summary="Get attention features for a ticker on an event date",
    description="""
    Returns Wikipedia pageview spike/z-score and GDELT news volume for a ticker
    centered on a specific event date. Designed for event-driven backtesting.

    **Wikipedia signals** (collected on-demand):
    - `wiki.views` — raw pageview count on `event_date`
    - `wiki.spike_10d` — views / 10-day median baseline; >1 = above-average interest
    - `wiki.zscore_20d` — standard-deviation units above 20-day mean

    **GDELT news signals** (pre-populated by scheduler only):
    - `news.article_count_1d` — articles published on `event_date`
    - `news.article_count_3d` — articles in `event_date ± 1 day` window
    - `news.unique_domains_3d` — distinct publisher domains in that window
    - `news.gdelt_status` — data availability flag:
      - `collected` — scheduler ran; counts are accurate (0 = genuinely no articles)
      - `not_collected` — scheduler has not run yet; use `POST /admin/collect/gdelt/{ticker}`
      - `not_available` — event date is before GDELT V2 coverage (2017-01-01)

    **Auto-resolution**: if no entity mapping exists, resolution runs automatically first.

    **Examples**:
    - `GET /attention/event/AAPL?event_date=2024-02-01` — Q1 earnings day attention
    - `GET /attention/event/NVDA?event_date=2024-05-22` — post-earnings spike
    """,
    responses={
        404: {"description": "Ticker not found or entity resolution failed"},
        500: {"description": "Feature materialization or collection error"},
    },
)
@with_cache(namespace="attention:event", ttl=3600, key_params=["ticker", "event_date"])
async def get_event_attention(
    ticker: str,
    event_date: date = Query(..., description="Event date in YYYY-MM-DD format"),
    response: Response = None,
    db: AsyncSession = Depends(get_db),
) -> EventAttentionResponse:
    ticker = ticker.upper()

    try:
        await asyncio.wait_for(_EVENT_SEMAPHORE.acquire(), timeout=_EVENT_SEMAPHORE_WAIT)
    except asyncio.TimeoutError:
        raise HTTPException(
            status_code=429,
            detail="서버가 바빠서 요청을 처리할 수 없습니다. 잠시 후 다시 시도하세요.",
        )
    try:
        return await _get_event_attention_impl(ticker, event_date, db)
    finally:
        _EVENT_SEMAPHORE.release()


async def _get_event_attention_impl(
    ticker: str,
    event_date: date,
    db: AsyncSession,
) -> EventAttentionResponse:
    # 1. Get or resolve entity
    entity_result = await db.execute(
        select(CompanyEntityMap).where(CompanyEntityMap.ticker == ticker)
    )
    entity = entity_result.scalars().first()

    if not entity:
        logger.info("No entity mapping for %s — auto-resolving", ticker)
        try:
            entity = await resolve_entity(db, ticker)
        except ValueError as exc:
            raise HTTPException(status_code=404, detail=str(exc))
        except Exception as exc:
            logger.error("Auto entity resolution failed for %s: %s", ticker, exc)
            raise HTTPException(status_code=500, detail=f"Entity resolution failed: {exc}")

    # 2. Check if features already exist in DB
    features_result = await db.execute(
        select(AttentionFeaturesDaily).where(
            AttentionFeaturesDaily.ticker == ticker,
            AttentionFeaturesDaily.date == event_date,
        )
    )
    features = features_result.scalars().first()

    if features is None:
        # 3. On-demand collection + materialization
        # NOTE: GDELT is intentionally excluded here — it must be collected via
        # the scheduler (POST /admin/collect/gdelt/{ticker}) to avoid IP rate bans.
        # This endpoint only collects Wikipedia data on-demand.
        logger.info("No features for %s on %s — collecting wiki on-demand", ticker, event_date)

        if entity.wiki_title:
            try:
                await collect_wiki_pageviews(db, ticker, event_date)
            except Exception as exc:
                logger.warning("Wiki collection failed for %s: %s", ticker, exc)

        try:
            features = await materialize_features(db, ticker, event_date)
        except Exception as exc:
            logger.error("Feature materialization failed for %s: %s", ticker, exc)
            raise HTTPException(status_code=500, detail=f"Feature materialization failed: {exc}")

    # Re-fetch entity in case it was updated during resolution
    entity_result2 = await db.execute(
        select(CompanyEntityMap).where(CompanyEntityMap.ticker == ticker)
    )
    entity = entity_result2.scalars().first()

    # Determine GDELT collection status for the client
    if event_date < GDELT_EARLIEST_DATE:
        gdelt_status = "not_available"
    else:
        window_start = event_date - timedelta(days=1)
        window_end = event_date + timedelta(days=1)
        collected_count = await db.execute(
            select(func.count()).where(
                GdeltArticleRaw.matched_ticker == ticker,
                func.date(GdeltArticleRaw.published_at) >= window_start,
                func.date(GdeltArticleRaw.published_at) <= window_end,
            )
        )
        gdelt_status = "collected" if (collected_count.scalar() or 0) > 0 else "not_collected"

    return _features_to_response(ticker, event_date, entity, features, gdelt_status)