stock-oracle/app/services/company_metadata_service.py

"""
CompanyMetadataService — authoritative source for company metadata.

Lookup order:
  1. Redis cache (24h TTL)
  2. universe_ticker_registry (DB) — if sector NOT NULL, return directly
  3. yfinance .info enrichment (semaphore=5) — upsert results back to registry + companies
  4. If yfinance fails, return whatever registry has (may have NULL sector)
  5. If ticker not in registry AND yfinance fails/invalid → raise ValueError("invalid ticker")
"""

import asyncio
import logging
from datetime import datetime, timezone
from typing import Optional

try:
    import yfinance_plus as yf
except ImportError:
    import yfinance as yf
from sqlalchemy import select
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.ext.asyncio import AsyncSession

from app.models.financial import Company
from app.models.universe_snapshot import UniverseSnapshot, UniverseTickerRegistry
from app.utils.cache import build_cache_key, get_cached_response, set_cached_response

logger = logging.getLogger(__name__)

_YFINANCE_SEMAPHORE = asyncio.Semaphore(5)

_EXCHANGE_MAP = {
    "NYQ": "NYSE", "NMS": "NASDAQ", "NGM": "NASDAQ",
    "NCM": "NASDAQ", "ASE": "AMEX", "PCX": "NYSE_ARCA",
}
_SKIP_QUOTE_TYPES = {"ETF", "MUTUALFUND", "INDEX", "CURRENCY", "FUTURE", "OPTION"}
_CACHE_TTL = 60 * 60 * 24  # 24h


def _canonical_exchange(raw: Optional[str]) -> Optional[str]:
    if not raw:
        return None
    return _EXCHANGE_MAP.get(raw.upper(), raw.upper()) or None


async def _fetch_yfinance_info(ticker: str) -> Optional[dict]:
    """Fetch yfinance .info in a thread pool with semaphore and 20s timeout."""
    loop = asyncio.get_event_loop()

    def _sync_fetch():
        try:
            import yfinance as _base_yf  # Always use base yfinance for auth reliability
            info = _base_yf.Ticker(ticker).info
            # yfinance's first HTTP call may lack a valid auth token and return
            # a partial response (sector/industry = None). Retry once if an
            # equity ticker is missing sector — the second call uses the auth
            # token that was cached by the first call.
            if info and not info.get("sector") and info.get("quoteType") == "EQUITY":
                info = _base_yf.Ticker(ticker).info
            return info
        except Exception as e:
            logger.warning("yfinance .info failed for %s: %s", ticker, e)
            return None

    async with _YFINANCE_SEMAPHORE:
        try:
            return await asyncio.wait_for(
                loop.run_in_executor(None, _sync_fetch),
                timeout=20,
            )
        except asyncio.TimeoutError:
            logger.warning("yfinance .info timed out for %s", ticker)
            return None
        except Exception as e:
            logger.warning("yfinance enrichment error for %s: %s", ticker, e)
            return None


async def _upsert_registry(db: AsyncSession, ticker: str, data: dict) -> None:
    stmt = pg_insert(UniverseTickerRegistry).values(
        ticker=ticker,
        name=data.get("name"),
        cik=data.get("cik"),
        sector=data.get("sector"),
        industry=data.get("industry"),
        exchange=data.get("exchange"),
        is_active=True,
        updated_at=datetime.now(timezone.utc),
    ).on_conflict_do_update(
        constraint="uq_universe_ticker_registry",
        set_={
            "name": data.get("name"),
            "sector": data.get("sector"),
            "industry": data.get("industry"),
            "exchange": data.get("exchange"),
            "updated_at": datetime.now(timezone.utc),
        },
    )
    await db.execute(stmt)


async def _upsert_company(db: AsyncSession, ticker: str, data: dict) -> None:
    result = await db.execute(select(Company).where(Company.ticker == ticker))
    company = result.scalar_one_or_none()
    now = datetime.now(timezone.utc)

    if company:
        if data.get("name") and (not company.name or company.name == f"{ticker} Corporation"):
            company.name = data["name"]
        if data.get("sector"):
            company.sector = data["sector"]
        if data.get("industry"):
            company.industry = data["industry"]
        if data.get("exchange"):
            company.exchange = data["exchange"]
        if data.get("country"):
            company.country = data["country"]
        if data.get("market_cap"):
            company.market_cap = data["market_cap"]
        if data.get("business_description"):
            company.business_description = data["business_description"]
        company.updated_at = now
    else:
        company = Company(
            ticker=ticker,
            name=data.get("name") or f"{ticker} Corporation",
            cik=data.get("cik"),
            exchange=data.get("exchange"),
            sector=data.get("sector"),
            industry=data.get("industry"),
            country=data.get("country"),
            market_cap=data.get("market_cap"),
            business_description=data.get("business_description"),
            created_at=now,
            updated_at=now,
        )
        db.add(company)

    await db.commit()


async def get_metadata(db: AsyncSession, ticker: str) -> dict:
    """
    Return company metadata dict for a ticker.
    Raises ValueError("invalid ticker: {ticker}") if ticker is unknown.

    Returned dict keys: ticker, name, cik, exchange, sector, industry,
                        country, market_cap, business_description
    """
    ticker = ticker.upper()
    cache_key = build_cache_key("company:meta", ticker)

    # --- 1. Redis cache ---
    cached = await get_cached_response(cache_key)
    if cached:
        body, _ = cached
        return body

    # --- 2. Registry DB ---
    reg_result = await db.execute(
        select(UniverseTickerRegistry).where(UniverseTickerRegistry.ticker == ticker)
    )
    reg = reg_result.scalar_one_or_none()

    # --- 3. market_cap from latest snapshot (opportunistic) ---
    snap_market_cap: Optional[float] = None
    snap_result = await db.execute(
        select(UniverseSnapshot)
        .where(UniverseSnapshot.ticker == ticker)
        .order_by(UniverseSnapshot.snapshot_date.desc())
        .limit(1)
    )
    snap = snap_result.scalar_one_or_none()
    if snap:
        snap_market_cap = snap.market_cap

    if reg and reg.sector:
        # Fast path: registry has sector — no yfinance needed
        data = _build_from_registry(ticker, reg, snap_market_cap)
        await set_cached_response(cache_key, data, ttl_seconds=_CACHE_TTL)
        return data

    # --- 4. yfinance enrichment ---
    info = await _fetch_yfinance_info(ticker)

    if info:
        quote_type = info.get("quoteType") or ""
        if quote_type in _SKIP_QUOTE_TYPES:
            # Valid but not an equity — return minimal
            data = _build_minimal(ticker, info, snap_market_cap)
            await set_cached_response(cache_key, data, ttl_seconds=_CACHE_TTL)
            return data

        if not quote_type and not info.get("longName") and not info.get("shortName"):
            # Likely invalid ticker
            if not reg:
                raise ValueError(f"invalid ticker: {ticker}")
            # Fall through to registry-only result

        enriched = {
            "name": info.get("longName") or info.get("shortName"),
            "cik": reg.cik if reg else None,
            "exchange": _canonical_exchange(info.get("exchange")),
            "sector": info.get("sector"),
            "industry": info.get("industry"),
            "country": info.get("country"),
            "market_cap": info.get("marketCap") or snap_market_cap,
            "business_description": info.get("longBusinessSummary"),
        }

        # Persist enrichment
        try:
            await _upsert_registry(db, ticker, enriched)
            await _upsert_company(db, ticker, enriched)
        except Exception as e:
            logger.warning("Failed to persist enrichment for %s: %s", ticker, e)

        data = {
            "ticker": ticker,
            **enriched,
        }
        await set_cached_response(cache_key, data, ttl_seconds=_CACHE_TTL)
        return data

    # --- 5. yfinance failed — use registry if available ---
    if reg:
        data = _build_from_registry(ticker, reg, snap_market_cap)
        # Short TTL so we retry enrichment soon
        await set_cached_response(cache_key, data, ttl_seconds=60 * 15)
        return data

    raise ValueError(f"invalid ticker: {ticker}")


def _build_from_registry(
    ticker: str, reg: UniverseTickerRegistry, market_cap: Optional[float]
) -> dict:
    return {
        "ticker": ticker,
        "name": reg.name,
        "cik": reg.cik,
        "exchange": reg.exchange,
        "sector": reg.sector,
        "industry": reg.industry,
        "country": None,
        "market_cap": market_cap,
        "business_description": None,
    }


def _build_minimal(ticker: str, info: dict, market_cap: Optional[float]) -> dict:
    return {
        "ticker": ticker,
        "name": info.get("longName") or info.get("shortName"),
        "cik": None,
        "exchange": _canonical_exchange(info.get("exchange")),
        "sector": info.get("sector"),
        "industry": info.get("industry"),
        "country": info.get("country"),
        "market_cap": info.get("marketCap") or market_cap,
        "business_description": None,
    }