""" CompanyMetadataService — authoritative source for company metadata. Lookup order: 1. Redis cache (24h TTL) 2. universe_ticker_registry (DB) — if sector NOT NULL, return directly 3. yfinance .info enrichment (semaphore=5) — upsert results back to registry + companies 4. If yfinance fails, return whatever registry has (may have NULL sector) 5. If ticker not in registry AND yfinance fails/invalid → raise ValueError("invalid ticker") """ import asyncio import logging from datetime import datetime, timezone from typing import Optional try: import yfinance_plus as yf except ImportError: import yfinance as yf from sqlalchemy import select from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.ext.asyncio import AsyncSession from app.models.financial import Company from app.models.universe_snapshot import UniverseSnapshot, UniverseTickerRegistry from app.utils.cache import build_cache_key, get_cached_response, set_cached_response logger = logging.getLogger(__name__) _YFINANCE_SEMAPHORE = asyncio.Semaphore(5) _EXCHANGE_MAP = { "NYQ": "NYSE", "NMS": "NASDAQ", "NGM": "NASDAQ", "NCM": "NASDAQ", "ASE": "AMEX", "PCX": "NYSE_ARCA", } _SKIP_QUOTE_TYPES = {"ETF", "MUTUALFUND", "INDEX", "CURRENCY", "FUTURE", "OPTION"} _CACHE_TTL = 60 * 60 * 24 # 24h def _canonical_exchange(raw: Optional[str]) -> Optional[str]: if not raw: return None return _EXCHANGE_MAP.get(raw.upper(), raw.upper()) or None async def _fetch_yfinance_info(ticker: str) -> Optional[dict]: """Fetch yfinance .info in a thread pool with semaphore and 20s timeout.""" loop = asyncio.get_event_loop() def _sync_fetch(): try: import yfinance as _base_yf # Always use base yfinance for auth reliability info = _base_yf.Ticker(ticker).info # yfinance's first HTTP call may lack a valid auth token and return # a partial response (sector/industry = None). Retry once if an # equity ticker is missing sector — the second call uses the auth # token that was cached by the first call. if info and not info.get("sector") and info.get("quoteType") == "EQUITY": info = _base_yf.Ticker(ticker).info return info except Exception as e: logger.warning("yfinance .info failed for %s: %s", ticker, e) return None async with _YFINANCE_SEMAPHORE: try: return await asyncio.wait_for( loop.run_in_executor(None, _sync_fetch), timeout=20, ) except asyncio.TimeoutError: logger.warning("yfinance .info timed out for %s", ticker) return None except Exception as e: logger.warning("yfinance enrichment error for %s: %s", ticker, e) return None async def _upsert_registry(db: AsyncSession, ticker: str, data: dict) -> None: stmt = pg_insert(UniverseTickerRegistry).values( ticker=ticker, name=data.get("name"), cik=data.get("cik"), sector=data.get("sector"), industry=data.get("industry"), exchange=data.get("exchange"), is_active=True, updated_at=datetime.now(timezone.utc), ).on_conflict_do_update( constraint="uq_universe_ticker_registry", set_={ "name": data.get("name"), "sector": data.get("sector"), "industry": data.get("industry"), "exchange": data.get("exchange"), "updated_at": datetime.now(timezone.utc), }, ) await db.execute(stmt) async def _upsert_company(db: AsyncSession, ticker: str, data: dict) -> None: result = await db.execute(select(Company).where(Company.ticker == ticker)) company = result.scalar_one_or_none() now = datetime.now(timezone.utc) if company: if data.get("name") and (not company.name or company.name == f"{ticker} Corporation"): company.name = data["name"] if data.get("sector"): company.sector = data["sector"] if data.get("industry"): company.industry = data["industry"] if data.get("exchange"): company.exchange = data["exchange"] if data.get("country"): company.country = data["country"] if data.get("market_cap"): company.market_cap = data["market_cap"] if data.get("business_description"): company.business_description = data["business_description"] company.updated_at = now else: company = Company( ticker=ticker, name=data.get("name") or f"{ticker} Corporation", cik=data.get("cik"), exchange=data.get("exchange"), sector=data.get("sector"), industry=data.get("industry"), country=data.get("country"), market_cap=data.get("market_cap"), business_description=data.get("business_description"), created_at=now, updated_at=now, ) db.add(company) await db.commit() async def get_metadata(db: AsyncSession, ticker: str) -> dict: """ Return company metadata dict for a ticker. Raises ValueError("invalid ticker: {ticker}") if ticker is unknown. Returned dict keys: ticker, name, cik, exchange, sector, industry, country, market_cap, business_description """ ticker = ticker.upper() cache_key = build_cache_key("company:meta", ticker) # --- 1. Redis cache --- cached = await get_cached_response(cache_key) if cached: body, _ = cached return body # --- 2. Registry DB --- reg_result = await db.execute( select(UniverseTickerRegistry).where(UniverseTickerRegistry.ticker == ticker) ) reg = reg_result.scalar_one_or_none() # --- 3. market_cap from latest snapshot (opportunistic) --- snap_market_cap: Optional[float] = None snap_result = await db.execute( select(UniverseSnapshot) .where(UniverseSnapshot.ticker == ticker) .order_by(UniverseSnapshot.snapshot_date.desc()) .limit(1) ) snap = snap_result.scalar_one_or_none() if snap: snap_market_cap = snap.market_cap if reg and reg.sector: # Fast path: registry has sector — no yfinance needed data = _build_from_registry(ticker, reg, snap_market_cap) await set_cached_response(cache_key, data, ttl_seconds=_CACHE_TTL) return data # --- 4. yfinance enrichment --- info = await _fetch_yfinance_info(ticker) if info: quote_type = info.get("quoteType") or "" if quote_type in _SKIP_QUOTE_TYPES: # Valid but not an equity — return minimal data = _build_minimal(ticker, info, snap_market_cap) await set_cached_response(cache_key, data, ttl_seconds=_CACHE_TTL) return data if not quote_type and not info.get("longName") and not info.get("shortName"): # Likely invalid ticker if not reg: raise ValueError(f"invalid ticker: {ticker}") # Fall through to registry-only result enriched = { "name": info.get("longName") or info.get("shortName"), "cik": reg.cik if reg else None, "exchange": _canonical_exchange(info.get("exchange")), "sector": info.get("sector"), "industry": info.get("industry"), "country": info.get("country"), "market_cap": info.get("marketCap") or snap_market_cap, "business_description": info.get("longBusinessSummary"), } # Persist enrichment try: await _upsert_registry(db, ticker, enriched) await _upsert_company(db, ticker, enriched) except Exception as e: logger.warning("Failed to persist enrichment for %s: %s", ticker, e) data = { "ticker": ticker, **enriched, } await set_cached_response(cache_key, data, ttl_seconds=_CACHE_TTL) return data # --- 5. yfinance failed — use registry if available --- if reg: data = _build_from_registry(ticker, reg, snap_market_cap) # Short TTL so we retry enrichment soon await set_cached_response(cache_key, data, ttl_seconds=60 * 15) return data raise ValueError(f"invalid ticker: {ticker}") def _build_from_registry( ticker: str, reg: UniverseTickerRegistry, market_cap: Optional[float] ) -> dict: return { "ticker": ticker, "name": reg.name, "cik": reg.cik, "exchange": reg.exchange, "sector": reg.sector, "industry": reg.industry, "country": None, "market_cap": market_cap, "business_description": None, } def _build_minimal(ticker: str, info: dict, market_cap: Optional[float]) -> dict: return { "ticker": ticker, "name": info.get("longName") or info.get("shortName"), "cik": None, "exchange": _canonical_exchange(info.get("exchange")), "sector": info.get("sector"), "industry": info.get("industry"), "country": info.get("country"), "market_cap": info.get("marketCap") or market_cap, "business_description": None, }