stock-oracle/app/services/company_metadata_service.py

"""
CompanyMetadataService — authoritative source for company metadata.

Lookup order:
  1. Redis cache (24h TTL)
  2. universe_ticker_registry (DB) — if sector NOT NULL, return directly
  3. yfinance .info enrichment (semaphore=5) — upsert results back to registry + companies
  4. If yfinance fails, return whatever registry has (may have NULL sector)
  5. If ticker not in registry AND yfinance fails/invalid → raise ValueError("invalid ticker")
"""

import asyncio
import logging
from datetime import datetime, timezone
from typing import Optional

import yfinance as yf
from sqlalchemy import select
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.ext.asyncio import AsyncSession

from app.models.financial import Company
from app.models.universe_snapshot import UniverseSnapshot, UniverseTickerRegistry
from app.utils.cache import build_cache_key, get_cached_response, set_cached_response

logger = logging.getLogger(__name__)

_YFINANCE_SEMAPHORE = asyncio.Semaphore(5)

_EXCHANGE_MAP = {
    "NYQ": "NYSE", "NMS": "NASDAQ", "NGM": "NASDAQ",
    "NCM": "NASDAQ", "ASE": "AMEX", "PCX": "NYSE_ARCA",
}
_SKIP_QUOTE_TYPES = {"ETF", "MUTUALFUND", "INDEX", "CURRENCY", "FUTURE", "OPTION"}
_CACHE_TTL = 60 * 60 * 24  # 24h


def _canonical_exchange(raw: Optional[str]) -> Optional[str]:
    if not raw:
        return None
    return _EXCHANGE_MAP.get(raw.upper(), raw.upper()) or None


async def _fetch_yfinance_info(ticker: str) -> Optional[dict]:
    """Fetch yfinance .info in a thread pool with semaphore and 20s timeout."""
    loop = asyncio.get_event_loop()

    def _sync_fetch():
        try:
            t = yf.Ticker(ticker)
            return t.info
        except Exception as e:
            logger.warning("yfinance .info failed for %s: %s", ticker, e)
            return None

    async with _YFINANCE_SEMAPHORE:
        try:
            return await asyncio.wait_for(
                loop.run_in_executor(None, _sync_fetch),
                timeout=20,
            )
        except asyncio.TimeoutError:
            logger.warning("yfinance .info timed out for %s", ticker)
            return None
        except Exception as e:
            logger.warning("yfinance enrichment error for %s: %s", ticker, e)
            return None


async def _upsert_registry(db: AsyncSession, ticker: str, data: dict) -> None:
    stmt = pg_insert(UniverseTickerRegistry).values(
        ticker=ticker,
        name=data.get("name"),
        cik=data.get("cik"),
        sector=data.get("sector"),
        industry=data.get("industry"),
        exchange=data.get("exchange"),
        is_active=True,
        updated_at=datetime.now(timezone.utc),
    ).on_conflict_do_update(
        constraint="uq_universe_ticker_registry",
        set_={
            "name": data.get("name"),
            "sector": data.get("sector"),
            "industry": data.get("industry"),
            "exchange": data.get("exchange"),
            "updated_at": datetime.now(timezone.utc),
        },
    )
    await db.execute(stmt)


async def _upsert_company(db: AsyncSession, ticker: str, data: dict) -> None:
    result = await db.execute(select(Company).where(Company.ticker == ticker))
    company = result.scalar_one_or_none()
    now = datetime.now(timezone.utc)

    if company:
        if data.get("name") and (not company.name or company.name == f"{ticker} Corporation"):
            company.name = data["name"]
        if data.get("sector"):
            company.sector = data["sector"]
        if data.get("industry"):
            company.industry = data["industry"]
        if data.get("exchange"):
            company.exchange = data["exchange"]
        if data.get("country"):
            company.country = data["country"]
        if data.get("market_cap"):
            company.market_cap = data["market_cap"]
        if data.get("business_description"):
            company.business_description = data["business_description"]
        company.updated_at = now
    else:
        company = Company(
            ticker=ticker,
            name=data.get("name") or f"{ticker} Corporation",
            cik=data.get("cik"),
            exchange=data.get("exchange"),
            sector=data.get("sector"),
            industry=data.get("industry"),
            country=data.get("country"),
            market_cap=data.get("market_cap"),
            business_description=data.get("business_description"),
            created_at=now,
            updated_at=now,
        )
        db.add(company)

    await db.commit()


async def get_metadata(db: AsyncSession, ticker: str) -> dict:
    """
    Return company metadata dict for a ticker.
    Raises ValueError("invalid ticker: {ticker}") if ticker is unknown.

    Returned dict keys: ticker, name, cik, exchange, sector, industry,
                        country, market_cap, business_description
    """
    ticker = ticker.upper()
    cache_key = build_cache_key("company:meta", ticker)

    # --- 1. Redis cache ---
    cached = await get_cached_response(cache_key)
    if cached:
        body, _ = cached
        return body

    # --- 2. Registry DB ---
    reg_result = await db.execute(
        select(UniverseTickerRegistry).where(UniverseTickerRegistry.ticker == ticker)
    )
    reg = reg_result.scalar_one_or_none()

    # --- 3. market_cap from latest snapshot (opportunistic) ---
    snap_market_cap: Optional[float] = None
    snap_result = await db.execute(
        select(UniverseSnapshot)
        .where(UniverseSnapshot.ticker == ticker)
        .order_by(UniverseSnapshot.snapshot_date.desc())
        .limit(1)
    )
    snap = snap_result.scalar_one_or_none()
    if snap:
        snap_market_cap = snap.market_cap

    if reg and reg.sector:
        # Fast path: registry has sector — no yfinance needed
        data = _build_from_registry(ticker, reg, snap_market_cap)
        await set_cached_response(cache_key, data, ttl_seconds=_CACHE_TTL)
        return data

    # --- 4. yfinance enrichment ---
    info = await _fetch_yfinance_info(ticker)

    if info:
        quote_type = info.get("quoteType") or ""
        if quote_type in _SKIP_QUOTE_TYPES:
            # Valid but not an equity — return minimal
            data = _build_minimal(ticker, info, snap_market_cap)
            await set_cached_response(cache_key, data, ttl_seconds=_CACHE_TTL)
            return data

        if not quote_type and not info.get("longName") and not info.get("shortName"):
            # Likely invalid ticker
            if not reg:
                raise ValueError(f"invalid ticker: {ticker}")
            # Fall through to registry-only result

        enriched = {
            "name": info.get("longName") or info.get("shortName"),
            "cik": reg.cik if reg else None,
            "exchange": _canonical_exchange(info.get("exchange")),
            "sector": info.get("sector"),
            "industry": info.get("industry"),
            "country": info.get("country"),
            "market_cap": info.get("marketCap") or snap_market_cap,
            "business_description": info.get("longBusinessSummary"),
        }

        # Persist enrichment
        try:
            await _upsert_registry(db, ticker, enriched)
            await _upsert_company(db, ticker, enriched)
        except Exception as e:
            logger.warning("Failed to persist enrichment for %s: %s", ticker, e)

        data = {
            "ticker": ticker,
            **enriched,
        }
        await set_cached_response(cache_key, data, ttl_seconds=_CACHE_TTL)
        return data

    # --- 5. yfinance failed — use registry if available ---
    if reg:
        data = _build_from_registry(ticker, reg, snap_market_cap)
        # Short TTL so we retry enrichment soon
        await set_cached_response(cache_key, data, ttl_seconds=60 * 15)
        return data

    raise ValueError(f"invalid ticker: {ticker}")


def _build_from_registry(
    ticker: str, reg: UniverseTickerRegistry, market_cap: Optional[float]
) -> dict:
    return {
        "ticker": ticker,
        "name": reg.name,
        "cik": reg.cik,
        "exchange": reg.exchange,
        "sector": reg.sector,
        "industry": reg.industry,
        "country": None,
        "market_cap": market_cap,
        "business_description": None,
    }


def _build_minimal(ticker: str, info: dict, market_cap: Optional[float]) -> dict:
    return {
        "ticker": ticker,
        "name": info.get("longName") or info.get("shortName"),
        "cik": None,
        "exchange": _canonical_exchange(info.get("exchange")),
        "sector": info.get("sector"),
        "industry": info.get("industry"),
        "country": info.get("country"),
        "market_cap": info.get("marketCap") or market_cap,
        "business_description": None,
    }