diff --git a/app/api/v1/endpoints/attention.py b/app/api/v1/endpoints/attention.py
index f30bb31..9736f4c 100644
--- a/app/api/v1/endpoints/attention.py
+++ b/app/api/v1/endpoints/attention.py
@@ -15,25 +15,27 @@ Admin endpoints:
import asyncio
import logging
-from datetime import date, timedelta
+from datetime import date, datetime, timedelta, timezone
from fastapi import APIRouter, Depends, HTTPException, Query, Response
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.core.database import get_db
-from app.utils.cache import with_cache
+from app.utils.cache import build_cache_key, get_cached_response, set_cached_response, with_cache
from app.models.attention import AttentionFeaturesDaily, CompanyEntityMap, GdeltArticleRaw
from app.services.attention.gdelt_collector import GDELT_EARLIEST_DATE
from app.schemas.attention import (
CollectionStatusResponse,
EntityInfo,
+ EntityOverrideResponse,
EntityResolveResponse,
EventAttentionResponse,
NewsFeatures,
WikiFeatures,
)
from app.services.attention.entity_resolver import resolve_entity
+from app.services.attention.entity_resolver import _build_gdelt_query, _normalize_name
from app.services.attention.feature_materializer import materialize_features
from app.services.attention.gdelt_collector import collect_gdelt_articles
from app.services.attention.wiki_collector import collect_wiki_pageviews
@@ -46,6 +48,22 @@ router = APIRouter()
_EVENT_SEMAPHORE = asyncio.Semaphore(8)
_EVENT_SEMAPHORE_WAIT = 10
+# Re-resolve a present-but-unresolved entity at most once per this window.
+# Bounds the Wikipedia query rate when fithia2 re-runs its bulk upcoming-
+# earnings scan (at semaphore=8) against still-NULL rows — prevents the 429
+# storm that originally left 1423/1697 mappings broken.
+_RESOLVE_RETRY_WINDOW = timedelta(hours=1)
+
+
+def _resolve_retry_due(updated_at) -> bool:
+ """True if enough time has elapsed since the last resolution attempt to
+ retry a still-unresolved entity. A missing timestamp counts as due."""
+ if updated_at is None:
+ return True
+ if updated_at.tzinfo is None:
+ updated_at = updated_at.replace(tzinfo=timezone.utc)
+ return datetime.now(timezone.utc) - updated_at >= _RESOLVE_RETRY_WINDOW
+
def _entity_to_info(entity: CompanyEntityMap) -> EntityInfo:
return EntityInfo(
@@ -226,6 +244,62 @@ async def admin_collect_gdelt(
)
+@router.post(
+ "/admin/entity/{ticker}/override",
+ response_model=EntityOverrideResponse,
+ summary="Manually set wiki_title for a ticker (override automatic resolver)",
+ description=(
+ "Directly sets the Wikipedia article title for a ticker, bypassing the automatic resolver. "
+ "Sets `is_manual_override=True` so the resolver will never overwrite this mapping.\n\n"
+ "Use when the resolver persistently picks the wrong article "
+ "(e.g. a lawsuit page or an acquired subsidiary instead of the company itself).\n\n"
+ "**Example**: `POST /admin/entity/CSCO/override?wiki_title=Cisco%20Systems`"
+ ),
+ tags=["attention-admin"],
+)
+async def admin_override_entity(
+ ticker: str,
+ wiki_title: str = Query(..., description="Exact Wikipedia article title to use"),
+ db: AsyncSession = Depends(get_db),
+) -> EntityOverrideResponse:
+ ticker = ticker.upper()
+
+ entity_result = await db.execute(
+ select(CompanyEntityMap).where(CompanyEntityMap.ticker == ticker)
+ )
+ entity = entity_result.scalars().first()
+ if not entity:
+ raise HTTPException(
+ status_code=404,
+ detail=f"No entity mapping found for {ticker}. POST /admin/resolve/{ticker} first.",
+ )
+
+ canonical_name = entity.canonical_name
+ aliases = entity.aliases_json or []
+ gdelt_query = _build_gdelt_query(canonical_name, aliases)
+
+ from sqlalchemy import update as sa_update
+ await db.execute(
+ sa_update(CompanyEntityMap)
+ .where(CompanyEntityMap.ticker == ticker)
+ .values(
+ wiki_title=wiki_title,
+ gdelt_query=gdelt_query,
+ is_manual_override=True,
+ resolver_confidence=1.0,
+ )
+ )
+ await db.commit()
+ logger.info("Manual override applied: %s → wiki_title=%r", ticker, wiki_title)
+
+ return EntityOverrideResponse(
+ ticker=ticker,
+ wiki_title=wiki_title,
+ gdelt_query=gdelt_query,
+ message=f"Manual override set: {ticker} → {wiki_title!r}. is_manual_override=True.",
+ )
+
+
# ===========================================================================
# Parameterized entity route (before /event/ to avoid shadowing)
# ===========================================================================
@@ -273,6 +347,10 @@ async def get_entity(
# ===========================================================================
+_EVENT_CACHE_TTL_COMPLETE = 3600 # full response with wiki data → 1 h
+_EVENT_CACHE_TTL_INCOMPLETE = 300 # wiki_views still null → 5 min, retry sooner
+
+
@router.get(
"/event/{ticker}",
response_model=EventAttentionResponse,
@@ -306,7 +384,6 @@ async def get_entity(
500: {"description": "Feature materialization or collection error"},
},
)
-@with_cache(namespace="attention:event", ttl=3600, key_params=["ticker", "event_date"])
async def get_event_attention(
ticker: str,
event_date: date = Query(..., description="Event date in YYYY-MM-DD format"),
@@ -314,6 +391,17 @@ async def get_event_attention(
db: AsyncSession = Depends(get_db),
) -> EventAttentionResponse:
ticker = ticker.upper()
+ cache_key = build_cache_key("attention:event", ticker, str(event_date))
+
+ cached = await get_cached_response(cache_key)
+ if cached:
+ cached_body, etag = cached
+ if response is not None:
+ response.headers["X-Cache"] = "HIT"
+ response.headers["Cache-Control"] = f"public, max-age={_EVENT_CACHE_TTL_COMPLETE}"
+ response.headers["ETag"] = etag
+ response.headers["X-Data-Source"] = "redis-cache"
+ return cached_body
try:
await asyncio.wait_for(_EVENT_SEMAPHORE.acquire(), timeout=_EVENT_SEMAPHORE_WAIT)
@@ -323,10 +411,22 @@ async def get_event_attention(
detail="서버가 바빠서 요청을 처리할 수 없습니다. 잠시 후 다시 시도하세요.",
)
try:
- return await _get_event_attention_impl(ticker, event_date, db)
+ result = await _get_event_attention_impl(ticker, event_date, db)
finally:
_EVENT_SEMAPHORE.release()
+ # Incomplete responses (wiki_views still null) cache for only 5 min so they
+ # re-trigger collection sooner rather than serving stale nulls for up to 1 h.
+ wiki_complete = result.wiki.views is not None
+ ttl = _EVENT_CACHE_TTL_COMPLETE if wiki_complete else _EVENT_CACHE_TTL_INCOMPLETE
+ etag = await set_cached_response(cache_key, result.model_dump(), ttl_seconds=ttl)
+ if response is not None:
+ response.headers["X-Cache"] = "MISS"
+ response.headers["Cache-Control"] = f"public, max-age={ttl}"
+ response.headers["ETag"] = etag
+
+ return result
+
async def _get_event_attention_impl(
ticker: str,
@@ -348,6 +448,23 @@ async def _get_event_attention_impl(
except Exception as exc:
logger.error("Auto entity resolution failed for %s: %s", ticker, exc)
raise HTTPException(status_code=500, detail=f"Entity resolution failed: {exc}")
+ elif (
+ not entity.wiki_title
+ and (entity.resolver_confidence or 0.0) < 0.5
+ and not entity.is_manual_override
+ and _resolve_retry_due(entity.updated_at)
+ ):
+ # Present-but-unresolved row — e.g. a transient Wikipedia 429 during a
+ # prior bulk run persisted wiki_title=NULL. Retry resolution, but at
+ # most once per _RESOLVE_RETRY_WINDOW so a bulk scan over still-NULL
+ # tickers can't re-create the 429 storm. Best-effort: on failure keep
+ # serving the (empty) row rather than 500-ing. The resolver's
+ # no-downgrade guard ensures a failed retry can't worsen the row.
+ logger.info("Entity %s unresolved — attempting on-demand re-resolve", ticker)
+ try:
+ entity = await resolve_entity(db, ticker)
+ except Exception as exc: # noqa: BLE001
+ logger.warning("On-demand re-resolve failed for %s: %s", ticker, exc)
# 2. Check if features already exist in DB
features_result = await db.execute(
@@ -358,12 +475,19 @@ async def _get_event_attention_impl(
)
features = features_result.scalars().first()
- if features is None:
- # 3. On-demand collection + materialization
+ needs_wiki = entity.wiki_title and (
+ features is None
+ or features.wiki_views is None
+ or features.wiki_zscore_20d is None
+ )
+ if features is None or needs_wiki:
+ # 3. On-demand collection + materialization.
+ # Re-triggers when features are missing OR wiki_views / wiki_zscore_20d is null:
+ # - wiki_views null: wiki_title was unset at collection time, or API failed
+ # - wiki_zscore_20d null: insufficient lookback data when previously materialized
# NOTE: GDELT is intentionally excluded here — it must be collected via
# the scheduler (POST /admin/collect/gdelt/{ticker}) to avoid IP rate bans.
- # This endpoint only collects Wikipedia data on-demand.
- logger.info("No features for %s on %s — collecting wiki on-demand", ticker, event_date)
+ logger.info("No features (or incomplete wiki) for %s on %s — collecting wiki on-demand", ticker, event_date)
if entity.wiki_title:
try:
diff --git a/app/schemas/attention.py b/app/schemas/attention.py
index 89e1ff3..32fbb26 100644
--- a/app/schemas/attention.py
+++ b/app/schemas/attention.py
@@ -153,3 +153,10 @@ class CollectionStatusResponse(BaseModel):
records_collected: int
date_range: Dict[str, Any] = Field(default_factory=dict)
status: str
+
+
+class EntityOverrideResponse(BaseModel):
+ ticker: str
+ wiki_title: str
+ gdelt_query: Optional[str]
+ message: str
diff --git a/app/services/attention/entity_resolver.py b/app/services/attention/entity_resolver.py
index bbe83a5..2b64da7 100644
--- a/app/services/attention/entity_resolver.py
+++ b/app/services/attention/entity_resolver.py
@@ -11,10 +11,11 @@ Resolution pipeline:
import logging
import re
+from datetime import datetime, timezone
from typing import Optional
import httpx
-from sqlalchemy import select, update
+from sqlalchemy import or_, select, update
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.ext.asyncio import AsyncSession
@@ -26,13 +27,28 @@ logger = logging.getLogger(__name__)
# Legal suffixes to strip for canonical name derivation
_SUFFIX_PATTERN = re.compile(
r",?\s+\b(Inc\.?|Corp\.?|Corporation|Holdings?|Ltd\.?|Limited|LLC|L\.L\.C\.|"
- r"Group|Co\.?|Company|Technologies|Technology|International|Industries|"
+ r"L\.P\.?|LP|" # Limited Partnership (e.g. "Enterprise Products Partners L.P.")
+ r"Group|Co\.?|Compan(?:y|ies)|Technologies|Technology|International|Industries|"
r"Pharmaceuticals?|Therapeutics?|Sciences?|Bancorp|Financial|Holding|"
r"Acquisition|Acquisitions|Capital|Partners|Trust|"
+ r"Nv|N\.V\.?|Plc\.?|p\.l\.c\.?|" # Dutch (N.V.) / British (Plc / p.l.c.)
+ r"S\.A\.?B?(?:\s+de\s+C\.V\.)?|" # Spanish (S.A., S.A.B., S.A.B. de C.V.)
+ r"Aktiengesellschaft|GmbH|AG|SE|" # German/Swiss/European
r"Com)\s*$", # "Com" catches SEC-style ".com" artifacts (e.g. "AMAZON COM")
re.IGNORECASE,
)
+# SEC filing artifact: state-of-incorporation suffix after slash
+# e.g. "Costco Wholesale Corp /New", "Wells Fargo & Company/Mn", "Applied Materials Inc /DE",
+# "Canadian Imperial Bank Of Commerce /Can/"
+_SEC_NEW_PATTERN = re.compile(r"\s*/\s*(?:New|[A-Za-z]{2,4})\s*$", re.IGNORECASE)
+
+# Danish/Norwegian corporate designation: "NOVO NORDISK A/S" → "NOVO NORDISK"
+_AS_PATTERN = re.compile(r"\bA/S\s*$", re.IGNORECASE)
+
+# Belgian SA/NV: "Anheuser-Busch InBev SA/NV" → strip "/NV" first, then "SA" via suffix
+_SA_NV_PATTERN = re.compile(r"\s*SA/NV\s*$", re.IGNORECASE)
+
_COMPANY_KEYWORDS = {
"company", "corporation", "inc", "corp", "ltd", "llc", "holdings",
"stock", "shares", "nasdaq", "nyse", "ticker", "finance", "financial",
@@ -47,7 +63,30 @@ def _normalize_name(raw_name: str) -> tuple[str, list[str]]:
Preserves the original and intermediate forms as aliases.
"""
aliases = []
- current = raw_name.strip()
+ # Normalize path separators: some DB records use backslash (e.g. "US BANCORP \DE\")
+ current = raw_name.strip().replace("\\", "/").rstrip("/").strip()
+
+ # Belgian "SA/NV" corporate designation (e.g. "Anheuser-Busch InBev SA/NV")
+ stripped_sanv = _SA_NV_PATTERN.sub("", current).strip()
+ if stripped_sanv and stripped_sanv != current:
+ aliases.append(current)
+ current = stripped_sanv
+
+ # Strip Danish/Norwegian "A/S" corporate designation (e.g. "NOVO NORDISK A/S")
+ stripped_as = _AS_PATTERN.sub("", current).strip()
+ if stripped_as and stripped_as != current:
+ aliases.append(current)
+ current = stripped_as
+
+ # Strip SEC reincorporation artifact "/New" (and state/province codes like "/DE", "/Can")
+ stripped_new = _SEC_NEW_PATTERN.sub("", current).strip()
+ if stripped_new and stripped_new != current:
+ aliases.append(current)
+ current = stripped_new
+
+ # Normalize SEC dot-com artifact: "Amazon.Com" / "Amazon.com" → "Amazon Com"
+ # so the iterative loop can strip "Com" as a regular suffix.
+ current = re.sub(r"\.com\b", " Com", current, flags=re.IGNORECASE)
for _ in range(5): # max 5 iterations to avoid infinite loops
stripped = _SUFFIX_PATTERN.sub("", current).strip().rstrip(",").strip()
@@ -56,7 +95,9 @@ def _normalize_name(raw_name: str) -> tuple[str, list[str]]:
aliases.append(current)
current = stripped
- canonical = current
+ # Strip trailing punctuation artifacts left by suffix removal (e.g. "&" from
+ # "JPMorgan Chase & Co" → strip "Co" → "JPMorgan Chase &").
+ canonical = current.rstrip(" &/,").strip()
# Also add the fully original name if not already captured
if raw_name.strip() != canonical and raw_name.strip() not in aliases:
aliases.insert(0, raw_name.strip())
@@ -154,7 +195,11 @@ def _score_wiki_result(result: dict, canonical_name: str, aliases: list[str]) ->
"""
raw_title = result.get("title", "")
title = raw_title.lower()
- snippet = result.get("snippet", "").lower()
+ # Wikipedia snippets contain HTML — strip before matching.
+ # Replace each tag with a space then collapse runs so "Costco Wholesale"
+ # becomes "Costco Wholesale" rather than "Costco Wholesale" (breaking string match).
+ raw_snippet = re.sub(r"\s+", " ", re.sub(r"<[^>]+>", " ", result.get("snippet", ""))).strip()
+ snippet = raw_snippet.lower()
combined = title + " " + snippet
# Penalize obvious non-company pages immediately
@@ -165,31 +210,56 @@ def _score_wiki_result(result: dict, canonical_name: str, aliases: list[str]) ->
if any(sig in combined for sig in non_company_signals):
return 0.1
- # Require snippet to have at least one finance keyword for non-exact-match titles
- finance_signals = ["company", "corporation", "stock", "nasdaq", "nyse", "shares",
- "investor", "business", "enterprise", "holdings", "inc."]
- if not any(sig in combined for sig in finance_signals):
- return 0.15
+ # Penalize legal case titles: "X v. Y" format (e.g. "FSF v. Cisco Systems, Inc.")
+ if re.search(r'\w v\. \w', raw_title):
+ return 0.05
canonical_lower = canonical_name.lower()
all_names = [canonical_name] + aliases
all_names_lower = [n.lower() for n in all_names]
- # Check for exact title match (e.g. "Apple Inc." == alias "Apple Inc.")
+ # Exact title match — checked BEFORE the finance-signal filter so that
+ # valid company pages whose snippet focuses on technical/product details
+ # (e.g. TSMC → fabs, JPMorgan → banking operations) still score 0.95.
raw_title_stripped = raw_title.strip()
for name in all_names:
if raw_title_stripped.lower() == name.lower():
- return 0.95 # exact match
+ return 0.95
+
+ # Space-collapsed match: handles merged brand names like "ExxonMobil" vs "Exxon Mobil"
+ title_no_space = raw_title_stripped.lower().replace(" ", "")
+ for name in all_names:
+ name_no_space = name.lower().replace(" ", "")
+ if len(name_no_space) > 4 and title_no_space == name_no_space:
+ return 0.90
+
+ # Require snippet to have at least one finance keyword for non-exact-match titles
+ finance_signals = ["company", "corporation", "stock", "nasdaq", "nyse", "shares",
+ "investor", "business", "enterprise", "holdings", "inc."]
+ if not any(sig in combined for sig in finance_signals):
+ return 0.15
+
+ # Hyphen-normalized forms: "COCA COLA" matches "The Coca-Cola Company"
+ # because "coca cola" is in "the coca cola company" after replacing hyphens with spaces.
+ title_norm = title.replace("-", " ")
+ canonical_norm = canonical_lower.replace("-", " ")
+ all_names_norm = [n.replace("-", " ") for n in all_names_lower]
# Check if title starts with canonical name
- title_starts_with_canonical = title.startswith(canonical_lower)
- title_contains_canonical = canonical_lower in title
+ title_starts_with_canonical = title_norm.startswith(canonical_norm)
+ title_contains_canonical = canonical_norm in title_norm
# Also check aliases
- title_starts_with_alias = any(title.startswith(n) for n in all_names_lower)
- title_contains_alias = any(n in title for n in all_names_lower)
+ title_starts_with_alias = any(title_norm.startswith(n) for n in all_names_norm)
+ title_contains_alias = any(n in title_norm for n in all_names_norm)
if not (title_contains_canonical or title_contains_alias):
+ # Snippet-contains fallback: handles acronym titles (e.g. "TSMC" article whose
+ # snippet reads "Taiwan Semiconductor Manufacturing Company Limited (TSMC)...")
+ # and short-title articles (e.g. "Costco" snippet contains "Costco Wholesale").
+ snippet_norm = snippet.replace("-", " ")
+ if canonical_norm in snippet_norm or any(n in snippet_norm for n in all_names_norm):
+ return 0.6
return 0.0
# Check for pages that are ABOUT the company (vs. lists, histories, etc.)
@@ -207,11 +277,13 @@ def _score_wiki_result(result: dict, canonical_name: str, aliases: list[str]) ->
else:
base = 0.0
- # Reward company/finance keywords in title or snippet
+ # Reward company/finance keywords in title or snippet.
+ # Cap at 0.89 so title-starts-with matches never outrank exact-title (0.95)
+ # or space-collapsed exact matches (0.90), regardless of keyword density.
keyword_hits = sum(1 for kw in _COMPANY_KEYWORDS if kw in combined)
keyword_score = min(keyword_hits / 3, 1.0)
- return min(base + keyword_score * 0.3, 1.0)
+ return min(base + keyword_score * 0.3, 0.89)
async def _resolve_wiki(
@@ -311,41 +383,60 @@ async def resolve_entity(
ticker, wiki_title, confidence,
)
- # Upsert into company_entity_map
- stmt = (
- pg_insert(CompanyEntityMap)
- .values(
- ticker=ticker,
+ # Upsert into company_entity_map.
+ #
+ # No-downgrade guard: a transient Wikipedia failure (429/network) is
+ # swallowed by _search_wikipedia → returns [] → (wiki_title=None,
+ # confidence=0.0). Without this guard, re-running resolution while
+ # rate-limited would overwrite a previously-good mapping with NULL —
+ # which is exactly how the 2026-03-17 bulk run left 1423/1697 rows
+ # broken. The upsert therefore only updates when the new result is
+ # itself good (wiki_title not NULL) OR the existing row was already
+ # unresolved (wiki_title NULL). Manual overrides are never touched.
+ insert_stmt = pg_insert(CompanyEntityMap).values(
+ ticker=ticker,
+ canonical_name=canonical_name,
+ wiki_title=wiki_title,
+ gdelt_query=gdelt_query,
+ aliases_json=aliases,
+ resolver_confidence=confidence,
+ is_manual_override=False,
+ )
+ stmt = insert_stmt.on_conflict_do_update(
+ index_elements=["ticker"],
+ set_=dict(
canonical_name=canonical_name,
wiki_title=wiki_title,
gdelt_query=gdelt_query,
aliases_json=aliases,
resolver_confidence=confidence,
- is_manual_override=False,
- )
- .on_conflict_do_update(
- index_elements=["ticker"],
- set_=dict(
- canonical_name=canonical_name,
- wiki_title=wiki_title,
- gdelt_query=gdelt_query,
- aliases_json=aliases,
- resolver_confidence=confidence,
- ),
- where=CompanyEntityMap.is_manual_override == False, # noqa: E712
- )
- .returning(CompanyEntityMap)
- )
+ # Explicit: ORM `onupdate` does NOT fire for INSERT...ON CONFLICT,
+ # so stamp it here. The on-demand re-resolve recency guard in the
+ # /event endpoint relies on this reflecting the last attempt.
+ updated_at=datetime.now(timezone.utc),
+ ),
+ where=(
+ (CompanyEntityMap.is_manual_override == False) # noqa: E712
+ & or_(
+ insert_stmt.excluded.wiki_title.isnot(None),
+ CompanyEntityMap.wiki_title.is_(None),
+ )
+ ),
+ ).returning(CompanyEntityMap)
- result = await db.execute(stmt)
+ await db.execute(stmt)
await db.commit()
- row = result.scalars().first()
- if row is None:
- # Manual override prevented update — return existing
- existing_result2 = await db.execute(
- select(CompanyEntityMap).where(CompanyEntityMap.ticker == ticker)
- )
- row = existing_result2.scalars().first()
+ # Read back via fresh SELECT with populate_existing=True.
+ # After commit, SQLAlchemy expires identity-map entries but does NOT evict them.
+ # A plain SELECT in the same session can return the expired (stale) cached object
+ # instead of reading the committed DB state. populate_existing forces the ORM to
+ # overwrite the identity-map entry with the fresh DB row.
+ fresh_result = await db.execute(
+ select(CompanyEntityMap)
+ .where(CompanyEntityMap.ticker == ticker)
+ .execution_options(populate_existing=True)
+ )
+ row = fresh_result.scalars().first()
return row
diff --git a/app/services/attention/feature_materializer.py b/app/services/attention/feature_materializer.py
index a0d1fb1..c6e36d4 100644
--- a/app/services/attention/feature_materializer.py
+++ b/app/services/attention/feature_materializer.py
@@ -169,19 +169,20 @@ async def materialize_features(
.returning(AttentionFeaturesDaily)
)
- result = await db.execute(stmt)
+ await db.execute(stmt)
await db.commit()
- row = result.scalars().first()
- if row is None:
- # Fetch after upsert
- row_result = await db.execute(
- select(AttentionFeaturesDaily).where(
- AttentionFeaturesDaily.ticker == ticker,
- AttentionFeaturesDaily.date == event_date,
- )
+ # Always re-fetch with populate_existing=True to avoid SQLAlchemy identity-map
+ # returning stale pre-upsert values (same pattern as resolve_entity).
+ row_result = await db.execute(
+ select(AttentionFeaturesDaily)
+ .where(
+ AttentionFeaturesDaily.ticker == ticker,
+ AttentionFeaturesDaily.date == event_date,
)
- row = row_result.scalars().first()
+ .execution_options(populate_existing=True)
+ )
+ row = row_result.scalars().first()
logger.info(
"Materialized features for %s on %s: wiki=%s gdelt_1d=%d gdelt_3d=%d",