feat: company metadata 안정화 — 실제 sector/industry/exchange 보강
- CompanyMetadataService 신규: Redis → registry → yfinance 순서 조회,
semaphore(5) 보호, 24h cache, companies + registry UPSERT
- GET /company/{ticker} + POST /company/bulk (최대 100개, 세션 격리)
- companies 테이블에 exchange/country/market_cap 컬럼 추가 (alembic)
- CompanyInfo 스키마 exchange/country/market_cap 필드 추가
- _is_placeholder() 체크: 기존 "Technology/Software/XYZ Corporation" 행 재보강
- financial endpoint: financials/price 실패 시 company block은 유지 (500 방지)
- bulk endpoint: 각 ticker마다 독립 AsyncSession으로 동시 세션 충돌 방지
- scripts/backfill_registry_sector.py: 9376개 NULL-sector 일괄 보강 스크립트
검증: AU→Basic Materials, USAS→Basic Materials, CPRX→Healthcare,
HE→Utilities, ACHR→Industrials, ZZZZZZ→404, AAPL 기존 데이터 유지
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
main
I Luk Kim
4 months ago
parent
59f78cd3ee
commit
7f28266bd6
@ -0,0 +1,26 @@
|
||||
"""companies: add exchange, country, market_cap columns
|
||||
|
||||
Revision ID: k2c3d4e5f6g7
|
||||
Revises: j1b2c3d4e5f6
|
||||
Create Date: 2026-04-20
|
||||
|
||||
"""
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
revision = 'k2c3d4e5f6g7'
|
||||
down_revision = 'j1b2c3d4e5f6'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column('companies', sa.Column('exchange', sa.String(20), nullable=True))
|
||||
op.add_column('companies', sa.Column('country', sa.String(100), nullable=True))
|
||||
op.add_column('companies', sa.Column('market_cap', sa.Float(), nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column('companies', 'market_cap')
|
||||
op.drop_column('companies', 'country')
|
||||
op.drop_column('companies', 'exchange')
|
||||
@ -0,0 +1,96 @@
|
||||
"""
|
||||
Company metadata endpoints — sector/industry/exchange/market_cap for any ticker.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import List
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.core.database import AsyncSessionLocal, get_db
|
||||
from app.schemas.company import (
|
||||
BulkCompanyItem,
|
||||
BulkCompanyRequest,
|
||||
BulkCompanyResponse,
|
||||
CompanyMetadataResponse,
|
||||
)
|
||||
from app.services import company_metadata_service as cms
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
_BULK_MAX = 100
|
||||
|
||||
|
||||
@router.get(
|
||||
"/{ticker}",
|
||||
response_model=CompanyMetadataResponse,
|
||||
summary="Get company metadata",
|
||||
description=(
|
||||
"Returns sector, industry, exchange, market_cap, country, and other metadata "
|
||||
"for a ticker. Valid tickers without financial statements still return 200. "
|
||||
"Unknown tickers return 404."
|
||||
),
|
||||
)
|
||||
async def get_company(
|
||||
ticker: str,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
ticker = ticker.upper()
|
||||
try:
|
||||
data = await cms.get_metadata(db, ticker)
|
||||
except ValueError as e:
|
||||
if "invalid ticker" in str(e).lower():
|
||||
raise HTTPException(status_code=404, detail=f"Unknown ticker: {ticker}")
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
except Exception as e:
|
||||
logger.error("company metadata error for %s: %s", ticker, e)
|
||||
raise HTTPException(status_code=500, detail="Failed to retrieve company metadata")
|
||||
|
||||
return CompanyMetadataResponse(**data)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/bulk",
|
||||
response_model=BulkCompanyResponse,
|
||||
summary="Bulk company metadata",
|
||||
description=(
|
||||
f"Fetch metadata for up to {_BULK_MAX} tickers in one request. "
|
||||
"Partial failures are allowed — each item has either `data` or `error`."
|
||||
),
|
||||
)
|
||||
async def bulk_company(
|
||||
request: BulkCompanyRequest,
|
||||
):
|
||||
tickers = [t.upper() for t in request.tickers]
|
||||
if not tickers:
|
||||
raise HTTPException(status_code=400, detail="tickers list is empty")
|
||||
if len(tickers) > _BULK_MAX:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Too many tickers: max {_BULK_MAX}, got {len(tickers)}",
|
||||
)
|
||||
|
||||
async def _fetch_one(ticker: str) -> BulkCompanyItem:
|
||||
async with AsyncSessionLocal() as session:
|
||||
try:
|
||||
data = await cms.get_metadata(session, ticker)
|
||||
return BulkCompanyItem(ticker=ticker, data=CompanyMetadataResponse(**data))
|
||||
except ValueError as e:
|
||||
return BulkCompanyItem(ticker=ticker, error=str(e))
|
||||
except Exception as e:
|
||||
logger.warning("bulk company error for %s: %s", ticker, e)
|
||||
return BulkCompanyItem(ticker=ticker, error="lookup failed")
|
||||
|
||||
results: List[BulkCompanyItem] = await asyncio.gather(*[_fetch_one(t) for t in tickers])
|
||||
|
||||
success_count = sum(1 for r in results if r.data is not None)
|
||||
return BulkCompanyResponse(
|
||||
results=results,
|
||||
total=len(results),
|
||||
success_count=success_count,
|
||||
error_count=len(results) - success_count,
|
||||
)
|
||||
@ -0,0 +1,35 @@
|
||||
"""
|
||||
Company metadata schemas for the dedicated /company endpoints.
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, List, Optional
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class CompanyMetadataResponse(BaseModel):
|
||||
ticker: str
|
||||
name: Optional[str] = None
|
||||
cik: Optional[str] = None
|
||||
exchange: Optional[str] = None
|
||||
sector: Optional[str] = None
|
||||
industry: Optional[str] = None
|
||||
country: Optional[str] = None
|
||||
market_cap: Optional[float] = None
|
||||
business_description: Optional[str] = None
|
||||
|
||||
|
||||
class BulkCompanyRequest(BaseModel):
|
||||
tickers: List[str]
|
||||
|
||||
|
||||
class BulkCompanyItem(BaseModel):
|
||||
ticker: str
|
||||
data: Optional[CompanyMetadataResponse] = None
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
class BulkCompanyResponse(BaseModel):
|
||||
results: List[BulkCompanyItem]
|
||||
total: int
|
||||
success_count: int
|
||||
error_count: int
|
||||
@ -0,0 +1,253 @@
|
||||
"""
|
||||
CompanyMetadataService — authoritative source for company metadata.
|
||||
|
||||
Lookup order:
|
||||
1. Redis cache (24h TTL)
|
||||
2. universe_ticker_registry (DB) — if sector NOT NULL, return directly
|
||||
3. yfinance .info enrichment (semaphore=5) — upsert results back to registry + companies
|
||||
4. If yfinance fails, return whatever registry has (may have NULL sector)
|
||||
5. If ticker not in registry AND yfinance fails/invalid → raise ValueError("invalid ticker")
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
|
||||
import yfinance as yf
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.financial import Company
|
||||
from app.models.universe_snapshot import UniverseSnapshot, UniverseTickerRegistry
|
||||
from app.utils.cache import build_cache_key, get_cached_response, set_cached_response
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_YFINANCE_SEMAPHORE = asyncio.Semaphore(5)
|
||||
|
||||
_EXCHANGE_MAP = {
|
||||
"NYQ": "NYSE", "NMS": "NASDAQ", "NGM": "NASDAQ",
|
||||
"NCM": "NASDAQ", "ASE": "AMEX", "PCX": "NYSE_ARCA",
|
||||
}
|
||||
_SKIP_QUOTE_TYPES = {"ETF", "MUTUALFUND", "INDEX", "CURRENCY", "FUTURE", "OPTION"}
|
||||
_CACHE_TTL = 60 * 60 * 24 # 24h
|
||||
|
||||
|
||||
def _canonical_exchange(raw: Optional[str]) -> Optional[str]:
|
||||
if not raw:
|
||||
return None
|
||||
return _EXCHANGE_MAP.get(raw.upper(), raw.upper()) or None
|
||||
|
||||
|
||||
async def _fetch_yfinance_info(ticker: str) -> Optional[dict]:
|
||||
"""Fetch yfinance .info in a thread pool with semaphore and 20s timeout."""
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
def _sync_fetch():
|
||||
try:
|
||||
t = yf.Ticker(ticker)
|
||||
return t.info
|
||||
except Exception as e:
|
||||
logger.warning("yfinance .info failed for %s: %s", ticker, e)
|
||||
return None
|
||||
|
||||
async with _YFINANCE_SEMAPHORE:
|
||||
try:
|
||||
return await asyncio.wait_for(
|
||||
loop.run_in_executor(None, _sync_fetch),
|
||||
timeout=20,
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
logger.warning("yfinance .info timed out for %s", ticker)
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.warning("yfinance enrichment error for %s: %s", ticker, e)
|
||||
return None
|
||||
|
||||
|
||||
async def _upsert_registry(db: AsyncSession, ticker: str, data: dict) -> None:
|
||||
stmt = pg_insert(UniverseTickerRegistry).values(
|
||||
ticker=ticker,
|
||||
name=data.get("name"),
|
||||
cik=data.get("cik"),
|
||||
sector=data.get("sector"),
|
||||
industry=data.get("industry"),
|
||||
exchange=data.get("exchange"),
|
||||
is_active=True,
|
||||
updated_at=datetime.now(timezone.utc),
|
||||
).on_conflict_do_update(
|
||||
constraint="uq_universe_ticker_registry",
|
||||
set_={
|
||||
"name": data.get("name"),
|
||||
"sector": data.get("sector"),
|
||||
"industry": data.get("industry"),
|
||||
"exchange": data.get("exchange"),
|
||||
"updated_at": datetime.now(timezone.utc),
|
||||
},
|
||||
)
|
||||
await db.execute(stmt)
|
||||
|
||||
|
||||
async def _upsert_company(db: AsyncSession, ticker: str, data: dict) -> None:
|
||||
result = await db.execute(select(Company).where(Company.ticker == ticker))
|
||||
company = result.scalar_one_or_none()
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
if company:
|
||||
if data.get("name") and (not company.name or company.name == f"{ticker} Corporation"):
|
||||
company.name = data["name"]
|
||||
if data.get("sector"):
|
||||
company.sector = data["sector"]
|
||||
if data.get("industry"):
|
||||
company.industry = data["industry"]
|
||||
if data.get("exchange"):
|
||||
company.exchange = data["exchange"]
|
||||
if data.get("country"):
|
||||
company.country = data["country"]
|
||||
if data.get("market_cap"):
|
||||
company.market_cap = data["market_cap"]
|
||||
if data.get("business_description"):
|
||||
company.business_description = data["business_description"]
|
||||
company.updated_at = now
|
||||
else:
|
||||
company = Company(
|
||||
ticker=ticker,
|
||||
name=data.get("name") or f"{ticker} Corporation",
|
||||
cik=data.get("cik"),
|
||||
exchange=data.get("exchange"),
|
||||
sector=data.get("sector"),
|
||||
industry=data.get("industry"),
|
||||
country=data.get("country"),
|
||||
market_cap=data.get("market_cap"),
|
||||
business_description=data.get("business_description"),
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
db.add(company)
|
||||
|
||||
await db.commit()
|
||||
|
||||
|
||||
async def get_metadata(db: AsyncSession, ticker: str) -> dict:
|
||||
"""
|
||||
Return company metadata dict for a ticker.
|
||||
Raises ValueError("invalid ticker: {ticker}") if ticker is unknown.
|
||||
|
||||
Returned dict keys: ticker, name, cik, exchange, sector, industry,
|
||||
country, market_cap, business_description
|
||||
"""
|
||||
ticker = ticker.upper()
|
||||
cache_key = build_cache_key("company:meta", ticker)
|
||||
|
||||
# --- 1. Redis cache ---
|
||||
cached = await get_cached_response(cache_key)
|
||||
if cached:
|
||||
body, _ = cached
|
||||
return body
|
||||
|
||||
# --- 2. Registry DB ---
|
||||
reg_result = await db.execute(
|
||||
select(UniverseTickerRegistry).where(UniverseTickerRegistry.ticker == ticker)
|
||||
)
|
||||
reg = reg_result.scalar_one_or_none()
|
||||
|
||||
# --- 3. market_cap from latest snapshot (opportunistic) ---
|
||||
snap_market_cap: Optional[float] = None
|
||||
snap_result = await db.execute(
|
||||
select(UniverseSnapshot)
|
||||
.where(UniverseSnapshot.ticker == ticker)
|
||||
.order_by(UniverseSnapshot.snapshot_date.desc())
|
||||
.limit(1)
|
||||
)
|
||||
snap = snap_result.scalar_one_or_none()
|
||||
if snap:
|
||||
snap_market_cap = snap.market_cap
|
||||
|
||||
if reg and reg.sector:
|
||||
# Fast path: registry has sector — no yfinance needed
|
||||
data = _build_from_registry(ticker, reg, snap_market_cap)
|
||||
await set_cached_response(cache_key, data, ttl_seconds=_CACHE_TTL)
|
||||
return data
|
||||
|
||||
# --- 4. yfinance enrichment ---
|
||||
info = await _fetch_yfinance_info(ticker)
|
||||
|
||||
if info:
|
||||
quote_type = info.get("quoteType") or ""
|
||||
if quote_type in _SKIP_QUOTE_TYPES:
|
||||
# Valid but not an equity — return minimal
|
||||
data = _build_minimal(ticker, info, snap_market_cap)
|
||||
await set_cached_response(cache_key, data, ttl_seconds=_CACHE_TTL)
|
||||
return data
|
||||
|
||||
if not quote_type and not info.get("longName") and not info.get("shortName"):
|
||||
# Likely invalid ticker
|
||||
if not reg:
|
||||
raise ValueError(f"invalid ticker: {ticker}")
|
||||
# Fall through to registry-only result
|
||||
|
||||
enriched = {
|
||||
"name": info.get("longName") or info.get("shortName"),
|
||||
"cik": reg.cik if reg else None,
|
||||
"exchange": _canonical_exchange(info.get("exchange")),
|
||||
"sector": info.get("sector"),
|
||||
"industry": info.get("industry"),
|
||||
"country": info.get("country"),
|
||||
"market_cap": info.get("marketCap") or snap_market_cap,
|
||||
"business_description": info.get("longBusinessSummary"),
|
||||
}
|
||||
|
||||
# Persist enrichment
|
||||
try:
|
||||
await _upsert_registry(db, ticker, enriched)
|
||||
await _upsert_company(db, ticker, enriched)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to persist enrichment for %s: %s", ticker, e)
|
||||
|
||||
data = {
|
||||
"ticker": ticker,
|
||||
**enriched,
|
||||
}
|
||||
await set_cached_response(cache_key, data, ttl_seconds=_CACHE_TTL)
|
||||
return data
|
||||
|
||||
# --- 5. yfinance failed — use registry if available ---
|
||||
if reg:
|
||||
data = _build_from_registry(ticker, reg, snap_market_cap)
|
||||
# Short TTL so we retry enrichment soon
|
||||
await set_cached_response(cache_key, data, ttl_seconds=60 * 15)
|
||||
return data
|
||||
|
||||
raise ValueError(f"invalid ticker: {ticker}")
|
||||
|
||||
|
||||
def _build_from_registry(
|
||||
ticker: str, reg: UniverseTickerRegistry, market_cap: Optional[float]
|
||||
) -> dict:
|
||||
return {
|
||||
"ticker": ticker,
|
||||
"name": reg.name,
|
||||
"cik": reg.cik,
|
||||
"exchange": reg.exchange,
|
||||
"sector": reg.sector,
|
||||
"industry": reg.industry,
|
||||
"country": None,
|
||||
"market_cap": market_cap,
|
||||
"business_description": None,
|
||||
}
|
||||
|
||||
|
||||
def _build_minimal(ticker: str, info: dict, market_cap: Optional[float]) -> dict:
|
||||
return {
|
||||
"ticker": ticker,
|
||||
"name": info.get("longName") or info.get("shortName"),
|
||||
"cik": None,
|
||||
"exchange": _canonical_exchange(info.get("exchange")),
|
||||
"sector": info.get("sector"),
|
||||
"industry": info.get("industry"),
|
||||
"country": info.get("country"),
|
||||
"market_cap": info.get("marketCap") or market_cap,
|
||||
"business_description": None,
|
||||
}
|
||||
@ -0,0 +1,93 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Backfill sector/industry/exchange/country for all active tickers in universe_ticker_registry
|
||||
that currently have sector=NULL.
|
||||
|
||||
Usage:
|
||||
python scripts/backfill_registry_sector.py [--batch 50] [--dry-run]
|
||||
|
||||
Expected runtime: ~30 min for 9376 tickers at semaphore=5.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import argparse
|
||||
import logging
|
||||
import sys
|
||||
import os
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine, async_sessionmaker
|
||||
|
||||
from app.core.config import settings
|
||||
from app.models.universe_snapshot import UniverseTickerRegistry
|
||||
from app.services import company_metadata_service as cms
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def backfill(batch_size: int, dry_run: bool) -> None:
|
||||
engine = create_async_engine(settings.DATABASE_URL, echo=False)
|
||||
Session = async_sessionmaker(engine, expire_on_commit=False)
|
||||
|
||||
async with Session() as db:
|
||||
result = await db.execute(
|
||||
select(UniverseTickerRegistry.ticker)
|
||||
.where(
|
||||
UniverseTickerRegistry.is_active == True,
|
||||
UniverseTickerRegistry.sector == None,
|
||||
)
|
||||
.order_by(UniverseTickerRegistry.ticker)
|
||||
)
|
||||
tickers = [row[0] for row in result.fetchall()]
|
||||
|
||||
logger.info("Found %d tickers with NULL sector", len(tickers))
|
||||
if dry_run:
|
||||
logger.info("Dry-run mode — no changes will be written.")
|
||||
return
|
||||
|
||||
success = 0
|
||||
failed = 0
|
||||
|
||||
for i in range(0, len(tickers), batch_size):
|
||||
batch = tickers[i:i + batch_size]
|
||||
logger.info("Batch %d/%d — processing %d tickers", i // batch_size + 1,
|
||||
(len(tickers) + batch_size - 1) // batch_size, len(batch))
|
||||
|
||||
async def _enrich(ticker: str) -> bool:
|
||||
async with Session() as db:
|
||||
try:
|
||||
meta = await cms.get_metadata(db, ticker)
|
||||
sector = meta.get("sector")
|
||||
logger.info(" %s → sector=%s", ticker, sector or "NULL")
|
||||
return bool(sector)
|
||||
except ValueError:
|
||||
logger.debug(" %s → invalid ticker, skipping", ticker)
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.warning(" %s → error: %s", ticker, e)
|
||||
return False
|
||||
|
||||
results = await asyncio.gather(*[_enrich(t) for t in batch])
|
||||
success += sum(results)
|
||||
failed += sum(1 for r in results if not r)
|
||||
|
||||
# Brief pause between batches to be kind to yfinance
|
||||
await asyncio.sleep(1)
|
||||
|
||||
logger.info("Done. success=%d failed/skipped=%d total=%d", success, failed, len(tickers))
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--batch", type=int, default=50)
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
args = parser.parse_args()
|
||||
asyncio.run(backfill(args.batch, args.dry_run))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Reference in New Issue