|
|
"""
|
|
|
Historical Stock Universe endpoints — backtesting universe construction
|
|
|
|
|
|
Endpoints:
|
|
|
GET /universe/screen — Screen stocks at a historical date by market_cap, sector, etc.
|
|
|
GET /universe/registry — Browse the registered ticker universe
|
|
|
POST /universe/admin/discover — Discover and register US tickers via yfinance screener
|
|
|
POST /universe/admin/build-snapshots — Build monthly market_cap snapshots (long-running)
|
|
|
"""
|
|
|
|
|
|
import logging
|
|
|
from typing import Optional
|
|
|
|
|
|
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Query
|
|
|
from fastapi.responses import Response
|
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
|
|
from app.core.database import get_db
|
|
|
from app.schemas.universe import (
|
|
|
RegistryResponse,
|
|
|
SnapshotBuildRequest,
|
|
|
TickerRegistryItem,
|
|
|
UniverseScreenResponse,
|
|
|
UniverseSnapshotItem,
|
|
|
)
|
|
|
from app.services.universe_service import UniverseService
|
|
|
from app.utils.cache import with_cache
|
|
|
|
|
|
router = APIRouter()
|
|
|
logger = logging.getLogger("app.api.v1.universe")
|
|
|
|
|
|
|
|
|
@router.get(
|
|
|
"/screen",
|
|
|
response_model=UniverseScreenResponse,
|
|
|
summary="Screen stocks at a historical date",
|
|
|
description=(
|
|
|
"Query monthly market_cap snapshots to find stocks matching criteria at a past date.\\n\\n"
|
|
|
"**용도**: 백테스팅 전략 유니버스 구성 — 특정 시점 시총/섹터 기준 종목 필터링.\\n\\n"
|
|
|
"**데이터 소스**: SEC EDGAR shares_outstanding × yfinance monthly close.\\n"
|
|
|
"**제한**: 현재 상장 종목만 포함 (survivorship bias). 상폐 종목 미포함.\\n\\n"
|
|
|
"**사전 조건**: `/universe/admin/discover` 후 `/universe/admin/build-snapshots` 실행 필요."
|
|
|
),
|
|
|
)
|
|
|
@with_cache(namespace="universe:screen", ttl=3600, key_params=["date", "market_cap_min", "market_cap_max", "sector", "exchange", "page", "page_size", "sort_by", "sort_ascending"])
|
|
|
async def screen_historical(
|
|
|
response: Response,
|
|
|
date: str = Query(..., description="Historical date YYYY-MM-DD (rounded to month start)"),
|
|
|
market_cap_min: Optional[float] = Query(None, description="Min market cap (USD), e.g. 2e9"),
|
|
|
market_cap_max: Optional[float] = Query(None, description="Max market cap (USD), e.g. 20e9"),
|
|
|
sector: Optional[str] = Query(None, description="Sector filter (e.g. Technology, Healthcare)"),
|
|
|
exchange: Optional[str] = Query(None, description="Exchange filter (NYSE, NASDAQ, AMEX)"),
|
|
|
page: int = Query(1, ge=1, description="Page number"),
|
|
|
page_size: int = Query(100, ge=1, le=500, description="Results per page"),
|
|
|
sort_by: str = Query("market_cap", description="Sort field: market_cap or ticker"),
|
|
|
sort_ascending: bool = Query(False, description="Sort direction"),
|
|
|
db: AsyncSession = Depends(get_db),
|
|
|
):
|
|
|
svc = UniverseService()
|
|
|
try:
|
|
|
items, total, snapshot_date = await svc.screen_historical(
|
|
|
db,
|
|
|
date_str=date,
|
|
|
market_cap_min=market_cap_min,
|
|
|
market_cap_max=market_cap_max,
|
|
|
sector=sector,
|
|
|
exchange=exchange,
|
|
|
page=page,
|
|
|
page_size=page_size,
|
|
|
sort_by=sort_by,
|
|
|
sort_ascending=sort_ascending,
|
|
|
)
|
|
|
except ValueError as e:
|
|
|
raise HTTPException(status_code=400, detail=str(e))
|
|
|
except Exception as e:
|
|
|
logger.error(f"Universe screen error: {e}")
|
|
|
raise HTTPException(status_code=502, detail=f"Universe screen failed: {e}")
|
|
|
|
|
|
total_pages = max(1, (total + page_size - 1) // page_size)
|
|
|
filters: dict = {}
|
|
|
if market_cap_min is not None:
|
|
|
filters["market_cap_min"] = market_cap_min
|
|
|
if market_cap_max is not None:
|
|
|
filters["market_cap_max"] = market_cap_max
|
|
|
if sector:
|
|
|
filters["sector"] = sector
|
|
|
if exchange:
|
|
|
filters["exchange"] = exchange
|
|
|
|
|
|
if total == 0:
|
|
|
note = (
|
|
|
"No snapshots found for this date. "
|
|
|
"Run POST /universe/admin/build-snapshots to populate data."
|
|
|
)
|
|
|
else:
|
|
|
note = "Survivorship bias: currently-listed stocks only. Delisted companies excluded."
|
|
|
|
|
|
return UniverseScreenResponse(
|
|
|
stocks=[UniverseSnapshotItem(**item) for item in items],
|
|
|
total_count=total,
|
|
|
page=page,
|
|
|
page_size=page_size,
|
|
|
total_pages=total_pages,
|
|
|
snapshot_date=snapshot_date,
|
|
|
filters_applied=filters,
|
|
|
metadata={
|
|
|
"sort_by": sort_by,
|
|
|
"sort_ascending": sort_ascending,
|
|
|
"note": note,
|
|
|
},
|
|
|
)
|
|
|
|
|
|
|
|
|
@router.get(
|
|
|
"/registry",
|
|
|
response_model=RegistryResponse,
|
|
|
summary="Browse registered ticker universe",
|
|
|
description="List tickers registered in the universe (populated via /admin/discover).",
|
|
|
)
|
|
|
async def get_registry(
|
|
|
sector: Optional[str] = Query(None, description="Filter by sector"),
|
|
|
exchange: Optional[str] = Query(None, description="Filter by exchange"),
|
|
|
is_active: Optional[bool] = Query(None, description="Filter by active status"),
|
|
|
page: int = Query(1, ge=1),
|
|
|
page_size: int = Query(100, ge=1, le=1000),
|
|
|
db: AsyncSession = Depends(get_db),
|
|
|
):
|
|
|
svc = UniverseService()
|
|
|
try:
|
|
|
rows, total = await svc.get_registry(
|
|
|
db,
|
|
|
sector=sector,
|
|
|
exchange=exchange,
|
|
|
is_active=is_active,
|
|
|
page=page,
|
|
|
page_size=page_size,
|
|
|
)
|
|
|
except Exception as e:
|
|
|
logger.error(f"Registry fetch error: {e}")
|
|
|
raise HTTPException(status_code=502, detail=str(e))
|
|
|
|
|
|
total_pages = max(1, (total + page_size - 1) // page_size)
|
|
|
return RegistryResponse(
|
|
|
tickers=[TickerRegistryItem.model_validate(r) for r in rows],
|
|
|
total_count=total,
|
|
|
page=page,
|
|
|
page_size=page_size,
|
|
|
total_pages=total_pages,
|
|
|
)
|
|
|
|
|
|
|
|
|
@router.post(
|
|
|
"/admin/discover",
|
|
|
summary="Discover and register US tickers",
|
|
|
description=(
|
|
|
"Scrapes US-listed stocks via yfinance screener and registers them in the universe.\\n\\n"
|
|
|
"**소요 시간**: 약 1~5분 (시총 기준에 따라 다름).\\n"
|
|
|
"**권장**: `market_cap_min=100000000` ($100M) → ~3000~5000 종목."
|
|
|
),
|
|
|
)
|
|
|
async def discover_tickers(
|
|
|
market_cap_min: float = Query(
|
|
|
1e8,
|
|
|
description="Min market cap for inclusion (USD). Default $100M.",
|
|
|
),
|
|
|
db: AsyncSession = Depends(get_db),
|
|
|
):
|
|
|
svc = UniverseService()
|
|
|
try:
|
|
|
result = await svc.discover_tickers(db, market_cap_min=market_cap_min)
|
|
|
return {
|
|
|
"status": "completed",
|
|
|
**result,
|
|
|
"note": "Run POST /universe/admin/build-snapshots to compute historical market_cap snapshots.",
|
|
|
}
|
|
|
except Exception as e:
|
|
|
logger.error(f"Universe discover error: {e}")
|
|
|
raise HTTPException(status_code=502, detail=f"Discover failed: {e}")
|
|
|
|
|
|
|
|
|
async def _run_build_snapshots(
|
|
|
tickers, start_date: str, end_date: str, force_rebuild: bool
|
|
|
):
|
|
|
"""Background task wrapper for build_snapshots.
|
|
|
|
|
|
Spawns a completely separate thread with its own asyncio event loop so
|
|
|
the long-running build never touches the API's event loop or DB connection
|
|
|
pool, keeping the API responsive throughout.
|
|
|
"""
|
|
|
import asyncio as _asyncio
|
|
|
import threading
|
|
|
from app.core.database import AsyncSessionLocal
|
|
|
|
|
|
def _thread_main():
|
|
|
loop = _asyncio.new_event_loop()
|
|
|
_asyncio.set_event_loop(loop)
|
|
|
|
|
|
async def _build():
|
|
|
# Create a fresh engine + session factory bound to THIS event loop.
|
|
|
# asyncpg connections are loop-bound; reusing the API's AsyncSessionLocal
|
|
|
# in a different loop raises "Future attached to a different loop".
|
|
|
from sqlalchemy.ext.asyncio import (
|
|
|
AsyncSession,
|
|
|
async_sessionmaker,
|
|
|
create_async_engine,
|
|
|
)
|
|
|
from sqlalchemy.orm import sessionmaker as _sessionmaker
|
|
|
from app.core.config import settings
|
|
|
|
|
|
_engine = create_async_engine(
|
|
|
settings.DATABASE_URL,
|
|
|
echo=False,
|
|
|
pool_size=2,
|
|
|
max_overflow=0,
|
|
|
pool_timeout=30,
|
|
|
pool_pre_ping=True,
|
|
|
connect_args={"server_settings": {"jit": "off"}},
|
|
|
)
|
|
|
_SessionLocal = _sessionmaker(
|
|
|
_engine, class_=AsyncSession, expire_on_commit=False
|
|
|
)
|
|
|
svc = UniverseService()
|
|
|
try:
|
|
|
result = await svc.build_snapshots(
|
|
|
_SessionLocal,
|
|
|
tickers=tickers,
|
|
|
start_date=start_date,
|
|
|
end_date=end_date,
|
|
|
force_rebuild=force_rebuild,
|
|
|
)
|
|
|
logger.info(f"Universe background build complete: {result}")
|
|
|
except Exception as e:
|
|
|
logger.error(f"Universe background build failed: {e}")
|
|
|
finally:
|
|
|
await _engine.dispose()
|
|
|
|
|
|
try:
|
|
|
loop.run_until_complete(_build())
|
|
|
finally:
|
|
|
loop.close()
|
|
|
|
|
|
t = threading.Thread(target=_thread_main, daemon=True, name="universe-build")
|
|
|
t.start()
|
|
|
logger.info(f"Universe build started in isolated thread (daemon)")
|
|
|
|
|
|
|
|
|
@router.post(
|
|
|
"/admin/build-snapshots",
|
|
|
summary="Build monthly market_cap snapshots",
|
|
|
description=(
|
|
|
"Computes monthly market_cap snapshots for registered tickers and stores them "
|
|
|
"in `universe_snapshot`.\\n\\n"
|
|
|
"**데이터 소스**: SEC EDGAR companyfacts (shares_outstanding) + yfinance monthly close.\\n\\n"
|
|
|
"**소요 시간**: 전체 유니버스(~4000 종목) × 10년 기준 30~60분. "
|
|
|
"백그라운드에서 실행되므로 응답은 즉시 반환됩니다.\\n\\n"
|
|
|
"**권장 시작점**: `tickers=[AAPL,MSFT,GOOGL]`로 소규모 테스트 후 전체 빌드."
|
|
|
),
|
|
|
)
|
|
|
async def build_snapshots(
|
|
|
body: SnapshotBuildRequest,
|
|
|
background_tasks: BackgroundTasks,
|
|
|
db: AsyncSession = Depends(get_db),
|
|
|
):
|
|
|
tickers = [t.upper() for t in body.tickers] if body.tickers else None
|
|
|
|
|
|
# Validate dates
|
|
|
try:
|
|
|
from datetime import date
|
|
|
date.fromisoformat(body.start_date)
|
|
|
date.fromisoformat(body.end_date)
|
|
|
except ValueError as e:
|
|
|
raise HTTPException(status_code=400, detail=f"Invalid date: {e}")
|
|
|
|
|
|
# For small ticker lists (≤20), run synchronously for immediate feedback
|
|
|
if tickers and len(tickers) <= 20:
|
|
|
svc = UniverseService()
|
|
|
try:
|
|
|
result = await svc.build_snapshots(
|
|
|
db, # existing session is fine for small quick jobs
|
|
|
tickers=tickers,
|
|
|
start_date=body.start_date,
|
|
|
end_date=body.end_date,
|
|
|
force_rebuild=body.force_rebuild,
|
|
|
)
|
|
|
return {"status": "completed", **result}
|
|
|
except Exception as e:
|
|
|
logger.error(f"Snapshot build error: {e}")
|
|
|
raise HTTPException(status_code=502, detail=str(e))
|
|
|
|
|
|
# Large jobs → background
|
|
|
ticker_count = len(tickers) if tickers else "all registry"
|
|
|
background_tasks.add_task(
|
|
|
_run_build_snapshots,
|
|
|
tickers,
|
|
|
body.start_date,
|
|
|
body.end_date,
|
|
|
body.force_rebuild,
|
|
|
)
|
|
|
return {
|
|
|
"status": "started",
|
|
|
"tickers_queued": ticker_count,
|
|
|
"date_range": f"{body.start_date} → {body.end_date}",
|
|
|
"note": (
|
|
|
"Building in background. "
|
|
|
"Query GET /universe/screen after a few minutes to verify data."
|
|
|
),
|
|
|
}
|