stock-oracle/app/api/v1/endpoints/attention.py

"""
Attention API endpoints — event-centric attention data for backtesting.

Static routes are registered before parameterized routes to avoid path shadowing.

Main endpoint:
  GET /api/v1/attention/event/{ticker}?event_date=YYYY-MM-DD

Admin endpoints:
  GET  /api/v1/attention/entity/{ticker}
  POST /api/v1/attention/admin/resolve/{ticker}
  POST /api/v1/attention/admin/collect/wiki/{ticker}?event_date=...
  POST /api/v1/attention/admin/collect/gdelt/{ticker}?event_date=...
"""

import asyncio
import logging
from datetime import date, datetime, timedelta, timezone

from fastapi import APIRouter, Depends, HTTPException, Query, Response
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession

from app.core.database import get_db
from app.utils.cache import build_cache_key, get_cached_response, set_cached_response, with_cache
from app.models.attention import AttentionFeaturesDaily, CompanyEntityMap, GdeltArticleRaw
from app.services.attention.gdelt_collector import GDELT_EARLIEST_DATE
from app.schemas.attention import (
    CollectionStatusResponse,
    EntityInfo,
    EntityOverrideResponse,
    EntityResolveResponse,
    EventAttentionResponse,
    NewsFeatures,
    WikiFeatures,
)
from app.services.attention.entity_resolver import resolve_entity
from app.services.attention.entity_resolver import _build_gdelt_query, _normalize_name
from app.services.attention.feature_materializer import materialize_features
from app.services.attention.gdelt_collector import collect_gdelt_articles
from app.services.attention.wiki_collector import collect_wiki_pageviews

logger = logging.getLogger(__name__)
router = APIRouter()

# Fast-fail gate for /event/{ticker} — blocks bulk scan floods that triggered
# wiki 429 bursts and contributed to container memory pressure.
_EVENT_SEMAPHORE = asyncio.Semaphore(8)
_EVENT_SEMAPHORE_WAIT = 10

# Re-resolve a present-but-unresolved entity at most once per this window.
# Bounds the Wikipedia query rate when fithia2 re-runs its bulk upcoming-
# earnings scan (at semaphore=8) against still-NULL rows — prevents the 429
# storm that originally left 1423/1697 mappings broken.
_RESOLVE_RETRY_WINDOW = timedelta(hours=1)


def _resolve_retry_due(updated_at) -> bool:
    """True if enough time has elapsed since the last resolution attempt to
    retry a still-unresolved entity. A missing timestamp counts as due."""
    if updated_at is None:
        return True
    if updated_at.tzinfo is None:
        updated_at = updated_at.replace(tzinfo=timezone.utc)
    return datetime.now(timezone.utc) - updated_at >= _RESOLVE_RETRY_WINDOW


def _entity_to_info(entity: CompanyEntityMap) -> EntityInfo:
    return EntityInfo(
        ticker=entity.ticker,
        canonical_name=entity.canonical_name,
        wiki_title=entity.wiki_title,
        gdelt_query=entity.gdelt_query,
        aliases=entity.aliases_json or [],
        resolver_confidence=entity.resolver_confidence or 0.0,
        is_manual_override=entity.is_manual_override or False,
    )


def _features_to_response(
    ticker: str,
    event_date: date,
    entity: CompanyEntityMap,
    features: AttentionFeaturesDaily,
    gdelt_status: str = "not_collected",
) -> EventAttentionResponse:
    # Compute baseline_10d from spike (inverse: baseline = views / spike)
    baseline_10d = None
    if features.wiki_spike_10d and features.wiki_views and features.wiki_spike_10d > 0:
        baseline_10d = features.wiki_views / features.wiki_spike_10d

    return EventAttentionResponse(
        ticker=ticker,
        event_date=event_date,
        entity=_entity_to_info(entity),
        wiki=WikiFeatures(
            views=features.wiki_views,
            baseline_10d=round(baseline_10d, 2) if baseline_10d else None,
            spike_10d=round(features.wiki_spike_10d, 4) if features.wiki_spike_10d is not None else None,
            zscore_20d=round(features.wiki_zscore_20d, 4) if features.wiki_zscore_20d is not None else None,
        ),
        news=NewsFeatures(
            article_count_1d=features.gdelt_article_count_1d or 0,
            article_count_3d=features.gdelt_article_count_3d or 0,
            unique_domains_3d=features.gdelt_unique_domains_3d or 0,
            us_article_count_3d=features.gdelt_us_article_count_3d or 0,
            gdelt_status=gdelt_status,
        ),
        metadata={
            "wiki_title": entity.wiki_title,
            "resolver_confidence": entity.resolver_confidence,
        },
    )


# ===========================================================================
# Admin / utility routes (static — must come before parameterized routes)
# ===========================================================================


@router.post(
    "/admin/resolve/{ticker}",
    response_model=EntityResolveResponse,
    summary="Resolve ticker → canonical entity",
    description=(
        "Maps a ticker symbol to a canonical company entity by looking up the company name, "
        "normalizing it, and validating against Wikipedia. Stores the result (canonical name, "
        "wiki_title, gdelt_query) in `company_entity_map`.\n\n"
        "Skips re-resolution if `is_manual_override` is set. "
        "If the company name in the DB is a placeholder (e.g. 'AMZN Corporation'), "
        "falls back to SEC company_tickers.json to fetch the real name and updates the DB."
    ),
    tags=["attention-admin"],
)
async def admin_resolve_entity(
    ticker: str,
    db: AsyncSession = Depends(get_db),
) -> EntityResolveResponse:
    ticker = ticker.upper()

    # Check for existing manual override
    existing_result = await db.execute(
        select(CompanyEntityMap).where(CompanyEntityMap.ticker == ticker)
    )
    existing = existing_result.scalars().first()
    if existing and existing.is_manual_override:
        return EntityResolveResponse(
            ticker=ticker,
            entity=_entity_to_info(existing),
            status="manual_override_skipped",
            message="Manual override is active — resolution skipped.",
        )

    try:
        entity = await resolve_entity(db, ticker)
        status = "resolved"
        message = f"Entity resolved: wiki_title={entity.wiki_title!r} confidence={entity.resolver_confidence:.2f}"
    except ValueError as exc:
        raise HTTPException(status_code=404, detail=str(exc))
    except Exception as exc:
        logger.error("Entity resolution failed for %s: %s", ticker, exc)
        raise HTTPException(status_code=500, detail=f"Resolution failed: {exc}")

    return EntityResolveResponse(
        ticker=ticker,
        entity=_entity_to_info(entity),
        status=status,
        message=message,
    )


@router.post(
    "/admin/collect/wiki/{ticker}",
    response_model=CollectionStatusResponse,
    summary="Collect Wikipedia pageviews for an event date",
    description=(
        "Fetches daily Wikipedia pageview counts for the ticker's canonical wiki_title, "
        "covering `event_date` and enough lookback days (≥20) to compute spike and z-score. "
        "Safe to call on-demand — Wikipedia API has no meaningful rate limit for this use.\n\n"
        "Requires entity resolution to have been run first (`wiki_title` must be set)."
    ),
    tags=["attention-admin"],
)
async def admin_collect_wiki(
    ticker: str,
    event_date: date = Query(..., description="Event date in YYYY-MM-DD format"),
    db: AsyncSession = Depends(get_db),
) -> CollectionStatusResponse:
    ticker = ticker.upper()
    try:
        count = await collect_wiki_pageviews(db, ticker, event_date)
    except ValueError as exc:
        raise HTTPException(status_code=422, detail=str(exc))
    except Exception as exc:
        logger.error("Wiki collection failed for %s: %s", ticker, exc)
        raise HTTPException(status_code=500, detail=f"Wiki collection failed: {exc}")

    return CollectionStatusResponse(
        ticker=ticker,
        source="wiki",
        records_collected=count,
        date_range={"event_date": str(event_date)},
        status="success",
    )


@router.post(
    "/admin/collect/gdelt/{ticker}",
    response_model=CollectionStatusResponse,
    summary="Collect GDELT news articles for an event date",
    description=(
        "Fetches news articles from GDELT V2 DOC API for the window `event_date ± 1 day`.\n\n"
        "**Coverage**: 2017-01-01 onwards. Requests for earlier dates return 0 immediately.\n\n"
        "**Rate limit**: GDELT enforces a global per-IP quota. This endpoint is protected by "
        "a process-wide lock (10s minimum interval) and retries with exponential backoff "
        "(30s → 60s → 120s) on 429 responses.\n\n"
        "⚠️ **Call this endpoint from a scheduler only** — never trigger it in response to "
        "user requests. Concurrent or rapid calls will exhaust the IP quota and cause "
        "temporary bans. The main `/event/{ticker}` endpoint intentionally does NOT collect "
        "GDELT on-demand for this reason."
    ),
    tags=["attention-admin"],
)
async def admin_collect_gdelt(
    ticker: str,
    event_date: date = Query(..., description="Event date in YYYY-MM-DD format"),
    db: AsyncSession = Depends(get_db),
) -> CollectionStatusResponse:
    ticker = ticker.upper()
    try:
        count = await collect_gdelt_articles(db, ticker, event_date)
    except ValueError as exc:
        raise HTTPException(status_code=422, detail=str(exc))
    except Exception as exc:
        logger.error("GDELT collection failed for %s: %s", ticker, exc)
        raise HTTPException(status_code=500, detail=f"GDELT collection failed: {exc}")

    return CollectionStatusResponse(
        ticker=ticker,
        source="gdelt",
        records_collected=count,
        date_range={"event_date": str(event_date)},
        status="success",
    )


@router.post(
    "/admin/entity/{ticker}/override",
    response_model=EntityOverrideResponse,
    summary="Manually set wiki_title for a ticker (override automatic resolver)",
    description=(
        "Directly sets the Wikipedia article title for a ticker, bypassing the automatic resolver. "
        "Sets `is_manual_override=True` so the resolver will never overwrite this mapping.\n\n"
        "Use when the resolver persistently picks the wrong article "
        "(e.g. a lawsuit page or an acquired subsidiary instead of the company itself).\n\n"
        "**Example**: `POST /admin/entity/CSCO/override?wiki_title=Cisco%20Systems`"
    ),
    tags=["attention-admin"],
)
async def admin_override_entity(
    ticker: str,
    wiki_title: str = Query(..., description="Exact Wikipedia article title to use"),
    db: AsyncSession = Depends(get_db),
) -> EntityOverrideResponse:
    ticker = ticker.upper()

    entity_result = await db.execute(
        select(CompanyEntityMap).where(CompanyEntityMap.ticker == ticker)
    )
    entity = entity_result.scalars().first()
    if not entity:
        raise HTTPException(
            status_code=404,
            detail=f"No entity mapping found for {ticker}. POST /admin/resolve/{ticker} first.",
        )

    canonical_name = entity.canonical_name
    aliases = entity.aliases_json or []
    gdelt_query = _build_gdelt_query(canonical_name, aliases)

    from sqlalchemy import update as sa_update
    await db.execute(
        sa_update(CompanyEntityMap)
        .where(CompanyEntityMap.ticker == ticker)
        .values(
            wiki_title=wiki_title,
            gdelt_query=gdelt_query,
            is_manual_override=True,
            resolver_confidence=1.0,
        )
    )
    await db.commit()
    logger.info("Manual override applied: %s → wiki_title=%r", ticker, wiki_title)

    return EntityOverrideResponse(
        ticker=ticker,
        wiki_title=wiki_title,
        gdelt_query=gdelt_query,
        message=f"Manual override set: {ticker} → {wiki_title!r}. is_manual_override=True.",
    )


# ===========================================================================
# Parameterized entity route (before /event/ to avoid shadowing)
# ===========================================================================


@router.get(
    "/entity/{ticker}",
    response_model=EntityResolveResponse,
    summary="Get entity mapping for a ticker",
    description="""
    Returns the stored entity mapping for a ticker: canonical name, Wikipedia title,
    GDELT query string, and resolver confidence score.

    Returns **404** if no mapping exists — run `POST /admin/resolve/{ticker}` first.

    **Example**: `GET /attention/entity/AAPL`
    """,
)
@with_cache(namespace="attention:entity", ttl=3600, key_params=["ticker"])
async def get_entity(
    ticker: str,
    response: Response = None,
    db: AsyncSession = Depends(get_db),
) -> EntityResolveResponse:
    ticker = ticker.upper()
    result = await db.execute(
        select(CompanyEntityMap).where(CompanyEntityMap.ticker == ticker)
    )
    entity = result.scalars().first()
    if not entity:
        raise HTTPException(
            status_code=404,
            detail=f"No entity mapping found for {ticker}. POST /admin/resolve/{ticker} to create one.",
        )
    return EntityResolveResponse(
        ticker=ticker,
        entity=_entity_to_info(entity),
        status="exists",
        message="Entity mapping retrieved from database.",
    )


# ===========================================================================
# Main event attention endpoint
# ===========================================================================


_EVENT_CACHE_TTL_COMPLETE = 3600   # full response with wiki data → 1 h
_EVENT_CACHE_TTL_INCOMPLETE = 300  # wiki_views still null → 5 min, retry sooner


@router.get(
    "/event/{ticker}",
    response_model=EventAttentionResponse,
    summary="Get attention features for a ticker on an event date",
    description="""
    Returns Wikipedia pageview spike/z-score and GDELT news volume for a ticker
    centered on a specific event date. Designed for event-driven backtesting.

    **Wikipedia signals** (collected on-demand):
    - `wiki.views` — raw pageview count on `event_date`
    - `wiki.spike_10d` — views / 10-day median baseline; >1 = above-average interest
    - `wiki.zscore_20d` — standard-deviation units above 20-day mean

    **GDELT news signals** (pre-populated by scheduler only):
    - `news.article_count_1d` — articles published on `event_date`
    - `news.article_count_3d` — articles in `event_date ± 1 day` window
    - `news.unique_domains_3d` — distinct publisher domains in that window
    - `news.gdelt_status` — data availability flag:
      - `collected` — scheduler ran; counts are accurate (0 = genuinely no articles)
      - `not_collected` — scheduler has not run yet; use `POST /admin/collect/gdelt/{ticker}`
      - `not_available` — event date is before GDELT V2 coverage (2017-01-01)

    **Auto-resolution**: if no entity mapping exists, resolution runs automatically first.

    **Examples**:
    - `GET /attention/event/AAPL?event_date=2024-02-01` — Q1 earnings day attention
    - `GET /attention/event/NVDA?event_date=2024-05-22` — post-earnings spike
    """,
    responses={
        404: {"description": "Ticker not found or entity resolution failed"},
        500: {"description": "Feature materialization or collection error"},
    },
)
async def get_event_attention(
    ticker: str,
    event_date: date = Query(..., description="Event date in YYYY-MM-DD format"),
    response: Response = None,
    db: AsyncSession = Depends(get_db),
) -> EventAttentionResponse:
    ticker = ticker.upper()
    cache_key = build_cache_key("attention:event", ticker, str(event_date))

    cached = await get_cached_response(cache_key)
    if cached:
        cached_body, etag = cached
        if response is not None:
            response.headers["X-Cache"] = "HIT"
            response.headers["Cache-Control"] = f"public, max-age={_EVENT_CACHE_TTL_COMPLETE}"
            response.headers["ETag"] = etag
            response.headers["X-Data-Source"] = "redis-cache"
        return cached_body

    try:
        await asyncio.wait_for(_EVENT_SEMAPHORE.acquire(), timeout=_EVENT_SEMAPHORE_WAIT)
    except asyncio.TimeoutError:
        raise HTTPException(
            status_code=429,
            detail="서버가 바빠서 요청을 처리할 수 없습니다. 잠시 후 다시 시도하세요.",
        )
    try:
        result = await _get_event_attention_impl(ticker, event_date, db)
    finally:
        _EVENT_SEMAPHORE.release()

    # Incomplete responses (wiki_views still null) cache for only 5 min so they
    # re-trigger collection sooner rather than serving stale nulls for up to 1 h.
    wiki_complete = result.wiki.views is not None
    ttl = _EVENT_CACHE_TTL_COMPLETE if wiki_complete else _EVENT_CACHE_TTL_INCOMPLETE
    etag = await set_cached_response(cache_key, result.model_dump(), ttl_seconds=ttl)
    if response is not None:
        response.headers["X-Cache"] = "MISS"
        response.headers["Cache-Control"] = f"public, max-age={ttl}"
        response.headers["ETag"] = etag

    return result


async def _get_event_attention_impl(
    ticker: str,
    event_date: date,
    db: AsyncSession,
) -> EventAttentionResponse:
    # 1. Get or resolve entity
    entity_result = await db.execute(
        select(CompanyEntityMap).where(CompanyEntityMap.ticker == ticker)
    )
    entity = entity_result.scalars().first()

    if not entity:
        logger.info("No entity mapping for %s — auto-resolving", ticker)
        try:
            entity = await resolve_entity(db, ticker)
        except ValueError as exc:
            raise HTTPException(status_code=404, detail=str(exc))
        except Exception as exc:
            logger.error("Auto entity resolution failed for %s: %s", ticker, exc)
            raise HTTPException(status_code=500, detail=f"Entity resolution failed: {exc}")
    elif (
        not entity.wiki_title
        and (entity.resolver_confidence or 0.0) < 0.5
        and not entity.is_manual_override
        and _resolve_retry_due(entity.updated_at)
    ):
        # Present-but-unresolved row — e.g. a transient Wikipedia 429 during a
        # prior bulk run persisted wiki_title=NULL. Retry resolution, but at
        # most once per _RESOLVE_RETRY_WINDOW so a bulk scan over still-NULL
        # tickers can't re-create the 429 storm. Best-effort: on failure keep
        # serving the (empty) row rather than 500-ing. The resolver's
        # no-downgrade guard ensures a failed retry can't worsen the row.
        logger.info("Entity %s unresolved — attempting on-demand re-resolve", ticker)
        try:
            entity = await resolve_entity(db, ticker)
        except Exception as exc:  # noqa: BLE001
            logger.warning("On-demand re-resolve failed for %s: %s", ticker, exc)

    # 2. Check if features already exist in DB
    features_result = await db.execute(
        select(AttentionFeaturesDaily).where(
            AttentionFeaturesDaily.ticker == ticker,
            AttentionFeaturesDaily.date == event_date,
        )
    )
    features = features_result.scalars().first()

    needs_wiki = entity.wiki_title and (
        features is None
        or features.wiki_views is None
        or features.wiki_zscore_20d is None
    )
    if features is None or needs_wiki:
        # 3. On-demand collection + materialization.
        # Re-triggers when features are missing OR wiki_views / wiki_zscore_20d is null:
        # - wiki_views null: wiki_title was unset at collection time, or API failed
        # - wiki_zscore_20d null: insufficient lookback data when previously materialized
        # NOTE: GDELT is intentionally excluded here — it must be collected via
        # the scheduler (POST /admin/collect/gdelt/{ticker}) to avoid IP rate bans.
        logger.info("No features (or incomplete wiki) for %s on %s — collecting wiki on-demand", ticker, event_date)

        if entity.wiki_title:
            try:
                await collect_wiki_pageviews(db, ticker, event_date)
            except Exception as exc:
                logger.warning("Wiki collection failed for %s: %s", ticker, exc)

        try:
            features = await materialize_features(db, ticker, event_date)
        except Exception as exc:
            logger.error("Feature materialization failed for %s: %s", ticker, exc)
            raise HTTPException(status_code=500, detail=f"Feature materialization failed: {exc}")

    # Re-fetch entity in case it was updated during resolution
    entity_result2 = await db.execute(
        select(CompanyEntityMap).where(CompanyEntityMap.ticker == ticker)
    )
    entity = entity_result2.scalars().first()

    # Determine GDELT collection status for the client
    if event_date < GDELT_EARLIEST_DATE:
        gdelt_status = "not_available"
    else:
        window_start = event_date - timedelta(days=1)
        window_end = event_date + timedelta(days=1)
        collected_count = await db.execute(
            select(func.count()).where(
                GdeltArticleRaw.matched_ticker == ticker,
                func.date(GdeltArticleRaw.published_at) >= window_start,
                func.date(GdeltArticleRaw.published_at) <= window_end,
            )
        )
        gdelt_status = "collected" if (collected_count.scalar() or 0) > 0 else "not_collected"

    return _features_to_response(ticker, event_date, entity, features, gdelt_status)