You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

551 lines
22 KiB
Python

"""
Alpaca Market Data endpoints — standalone price data via Alpaca API
"""
import asyncio
import gc
import logging
from datetime import date, datetime, timezone, timedelta
from typing import Optional
from zoneinfo import ZoneInfo
from fastapi import APIRouter, BackgroundTasks, HTTPException, Query
from sqlalchemy import text
from app.core.database import AsyncSessionLocal
logger = logging.getLogger(__name__)
_ET = ZoneInfo("America/New_York")
_MARKET_CLOSE_HOUR = 16 # 4:00 PM ET
# Limit concurrent Alpaca intraday processing to prevent event-loop saturation
# under bulk backfill workloads.
_INTRADAY_SEMAPHORE = asyncio.Semaphore(5)
# Max seconds a request waits for a semaphore slot before returning 429.
# Prevents hundreds of requests piling up (each holding a connection slot)
# during A-Z full-ticker scans.
_SEMAPHORE_WAIT_TIMEOUT = 10
def _market_closed_for(d: date) -> bool:
"""Return True if the US equity market session for date d has ended."""
now_et = datetime.now(_ET)
market_close_et = datetime(d.year, d.month, d.day, _MARKET_CLOSE_HOUR, 0, tzinfo=_ET)
return now_et >= market_close_et
from app.schemas.financial import (
AlpacaMultiBarsResponse,
AlpacaMultiSnapshotResponse,
AlpacaSnapshotResponse,
)
from app.services.alpaca_client import AlpacaClient
from app.services.alpaca_price_service import AlpacaPriceService
router = APIRouter()
def _require_alpaca() -> AlpacaPriceService:
svc = AlpacaPriceService()
if not svc.is_available():
raise HTTPException(
status_code=503,
detail="Alpaca API keys not configured. Set ALPACA_API_KEY and ALPACA_SECRET_KEY.",
)
return svc
# ------------------------------------------------------------------
# Status
# ------------------------------------------------------------------
@router.get(
"/status",
summary="Alpaca connection status",
description="Check Alpaca API key validity and connection health.",
)
async def alpaca_status():
client = AlpacaClient()
if not client.is_configured():
return {
"configured": False,
"message": "ALPACA_API_KEY / ALPACA_SECRET_KEY not set",
}
status = await client.check_connection()
await client.close()
return {"configured": True, **status}
# ------------------------------------------------------------------
# Intraday bars (DB-backed)
# ------------------------------------------------------------------
@router.get(
"/intraday",
response_model=AlpacaMultiBarsResponse,
summary="Get historical intraday bars for multiple tickers (SIP feed, DB-backed)",
description=(
"멀티 종목 과거 분봉 데이터를 Alpaca **SIP 피드**로 가져옵니다. "
"DB에 저장되며 재요청 시 Alpaca 미호출.\n\n"
"**⚠️ 장 중 당일 데이터 불가** — 장 마감(오후 4시 ET) 후에는 당일 날짜도 조회 가능\n\n"
"| 항목 | 내용 |\n"
"|------|------|\n"
"| 피드 | **SIP** (전체 미국 거래소 통합) |\n"
"| 거래량 | **100%** 정확 |\n"
"| 조회 범위 | **2016년~오늘(장 마감 후)** |\n"
"| DB 저장 | 있음 (재요청 시 Alpaca 미사용) |\n\n"
"**권장 용도**: 백테스트, 과거 분봉 분석\n\n"
"- `tickers`: comma-separated, e.g. `AAPL,MSFT,BF-B`\n"
"- `interval`: `1m`, `5m`, `15m`, `30m`, `1h`\n"
"- 내부 100개 단위 자동 배치 분할 (500종목 → Alpaca 5회 호출)\n"
"- Requires `ALPACA_API_KEY` / `ALPACA_SECRET_KEY`"
),
)
async def get_alpaca_intraday_multi(
tickers: str = Query(..., description="Comma-separated tickers, e.g. AAPL,MSFT,BF-B"),
interval: str = Query("5m", description="Interval: 1m, 5m, 15m, 30m, 1h"),
start_date: Optional[date] = Query(None, description="Start date (YYYY-MM-DD). Default: yesterday"),
end_date: Optional[date] = Query(None, description="End date (YYYY-MM-DD). Must be before today. Default: yesterday"),
force_refresh: bool = Query(False, description="Re-fetch from Alpaca even if DB has data"),
):
"""Multi-ticker historical intraday bars via Alpaca SIP (up to today after market close)."""
symbols = [s.strip().upper() for s in tickers.split(",") if s.strip()]
if not symbols:
raise HTTPException(status_code=400, detail="No tickers provided.")
if len(symbols) > 1000:
raise HTTPException(status_code=400, detail="Maximum 1000 tickers per request.")
yesterday = date.today() - timedelta(days=1)
_start = start_date or yesterday
_end = end_date or yesterday
today = date.today()
if _end > today:
raise HTTPException(
status_code=400,
detail="미래 날짜는 조회할 수 없습니다.",
)
if _end == today and not _market_closed_for(today):
raise HTTPException(
status_code=400,
detail="장 중에는 당일 SIP 데이터를 조회할 수 없습니다. "
"실시간 데이터는 GET /api/v1/alpaca/intraday/today 를 사용하세요.",
)
svc = _require_alpaca()
start_dt = datetime.combine(_start, datetime.min.time()).replace(tzinfo=timezone.utc)
end_dt = datetime.combine(_end, datetime.max.time()).replace(tzinfo=timezone.utc)
async def _do_fetch():
try:
await asyncio.wait_for(
_INTRADAY_SEMAPHORE.acquire(), timeout=_SEMAPHORE_WAIT_TIMEOUT
)
except asyncio.TimeoutError:
raise HTTPException(
status_code=429,
detail="서버가 바빠서 요청을 처리할 수 없습니다. 잠시 후 다시 시도하세요.",
)
try:
return await svc.get_or_fetch_multi_bars(
symbols, start_dt, end_dt, interval, force_refresh, feed="sip"
)
finally:
_INTRADAY_SEMAPHORE.release()
try:
await svc.client.close()
except Exception:
pass
try:
data = await asyncio.wait_for(_do_fetch(), timeout=120)
except asyncio.TimeoutError:
raise HTTPException(
status_code=504,
detail="요청 시간 초과 (120초). 티커 수를 줄이거나 나중에 다시 시도하세요.",
)
except HTTPException:
raise
except Exception as e:
err = str(e)
detail = f"Alpaca API error: {err}"
if "502" in err or "Bad Gateway" in err:
detail = (
f"Alpaca 502 Bad Gateway — 요청당 심볼 수 초과 가능성. "
f"내부 배치 크기: 100개/요청. 원인: {err}"
)
raise HTTPException(status_code=502, detail=detail)
gc.collect()
bars = {
ticker: [
{
"timestamp": row.date.isoformat(),
"open": row.open,
"high": row.high,
"low": row.low,
"close": row.close,
"volume": row.volume,
}
for row in rows
]
for ticker, rows in data.items()
}
return AlpacaMultiBarsResponse(interval=interval, count=len(symbols), bars=bars)
@router.get(
"/intraday/today",
response_model=AlpacaMultiBarsResponse,
summary="Get today's real-time intraday bars for multiple tickers (IEX feed, DB-backed)",
description=(
"당일(오늘) 실시간 분봉 데이터를 Alpaca **IEX 피드**로 가져옵니다. "
"장 중 재요청 시 항상 Alpaca에서 최신 데이터를 가져옵니다.\n\n"
"**⚠️ 오늘 데이터만 조회 가능** — 과거 데이터는 `/intraday` 사용\n\n"
"| 항목 | 내용 |\n"
"|------|------|\n"
"| 피드 | **IEX** (IEX 거래소 단일) |\n"
"| 지연 | **실시간** (지연 없음) |\n"
"| 거래량 | 실제의 약 **2~5%** (IEX 거래소 거래만 집계) |\n"
"| High/Low range | SIP 대비 좁게 표시될 수 있음 |\n"
"| DB 저장 | 있음 (장 중 항상 재조회) |\n\n"
"**권장 용도**: 당일 ORB 전략, 실시간 장 중 모니터링\n\n"
"- `tickers`: comma-separated, e.g. `AAPL,MSFT,BF-B`\n"
"- `interval`: `1m`, `5m`, `15m`, `30m`, `1h`\n"
"- 내부 100개 단위 자동 배치 분할\n"
"- Requires `ALPACA_API_KEY` / `ALPACA_SECRET_KEY`"
),
)
async def get_alpaca_intraday_today(
tickers: str = Query(..., description="Comma-separated tickers, e.g. AAPL,MSFT,BF-B"),
interval: str = Query("5m", description="Interval: 1m, 5m, 15m, 30m, 1h"),
):
"""Today's real-time intraday bars via Alpaca IEX (always re-fetches latest)."""
symbols = [s.strip().upper() for s in tickers.split(",") if s.strip()]
if not symbols:
raise HTTPException(status_code=400, detail="No tickers provided.")
if len(symbols) > 1000:
raise HTTPException(status_code=400, detail="Maximum 1000 tickers per request.")
svc = _require_alpaca()
today = date.today()
start_dt = datetime.combine(today, datetime.min.time()).replace(tzinfo=timezone.utc)
end_dt = datetime.combine(today, datetime.max.time()).replace(tzinfo=timezone.utc)
async def _do_fetch():
try:
await asyncio.wait_for(
_INTRADAY_SEMAPHORE.acquire(), timeout=_SEMAPHORE_WAIT_TIMEOUT
)
except asyncio.TimeoutError:
raise HTTPException(
status_code=429,
detail="서버가 바빠서 요청을 처리할 수 없습니다. 잠시 후 다시 시도하세요.",
)
try:
return await svc.get_or_fetch_multi_bars(
symbols, start_dt, end_dt, interval, force_refresh=True, feed="iex"
)
finally:
_INTRADAY_SEMAPHORE.release()
try:
await svc.client.close()
except Exception:
pass
try:
data = await asyncio.wait_for(_do_fetch(), timeout=120)
except asyncio.TimeoutError:
raise HTTPException(
status_code=504,
detail="요청 시간 초과 (120초). 티커 수를 줄이거나 나중에 다시 시도하세요.",
)
except HTTPException:
raise
except Exception as e:
err = str(e)
detail = f"Alpaca API error: {err}"
if "502" in err or "Bad Gateway" in err:
detail = (
f"Alpaca 502 Bad Gateway — 요청당 심볼 수 초과 가능성. "
f"내부 배치 크기: 100개/요청. 원인: {err}"
)
raise HTTPException(status_code=502, detail=detail)
gc.collect()
bars = {
ticker: [
{
"timestamp": row.date.isoformat(),
"open": row.open,
"high": row.high,
"low": row.low,
"close": row.close,
"volume": row.volume,
}
for row in rows
]
for ticker, rows in data.items()
}
return AlpacaMultiBarsResponse(interval=interval, count=len(symbols), bars=bars)
# ------------------------------------------------------------------
# Real-time snapshot
# ------------------------------------------------------------------
def _parse_snapshot(ticker: str, raw: dict) -> AlpacaSnapshotResponse:
"""Convert raw Alpaca snapshot dict → AlpacaSnapshotResponse."""
trade = raw.get("latestTrade") or {}
quote = raw.get("latestQuote") or {}
daily = raw.get("dailyBar") or {}
prev = raw.get("prevDailyBar") or {}
price = trade.get("p")
prev_close = prev.get("c")
change = round(price - prev_close, 4) if price is not None and prev_close else None
change_pct = round(change / prev_close * 100, 4) if change is not None and prev_close else None
return AlpacaSnapshotResponse(
ticker=ticker.upper(),
timestamp=trade.get("t"),
price=price,
trade_size=trade.get("s"),
bid=quote.get("bp"),
ask=quote.get("ap"),
bid_size=quote.get("bs"),
ask_size=quote.get("as"),
open=daily.get("o"),
high=daily.get("h"),
low=daily.get("l"),
volume=daily.get("v"),
vwap=daily.get("vw"),
prev_close=prev_close,
change=change,
change_pct=change_pct,
)
# ------------------------------------------------------------------
# Admin: PIT price backfill
# ------------------------------------------------------------------
def _finra_to_alpaca(symbol: str) -> str:
"""FINRA "/" → Alpaca "." (BRK/B → BRK.B, AAC/U → AAC.U)."""
return symbol.replace("/", ".").replace("-", ".")
async def _run_pit_backfill(start_str: str, end_str: str, force: bool) -> None:
"""Background task: backfill Alpaca 1d bars for all FINRA PIT symbols."""
from sqlalchemy.dialects.postgresql import insert as pg_insert
from app.models.alpaca_price import AlpacaPriceData
BATCH_SIZE = 100
CHUNK_SIZE = 2300 # asyncpg 32767 bind-param limit
ADJUSTMENT = "all"
TOLERANCE_DAYS = 5
client = AlpacaClient()
if not client.is_configured():
logger.error("PIT backfill: Alpaca keys not configured")
return
try:
# Step 1: FINRA PIT 심볼 + 활동기간
async with AsyncSessionLocal() as db:
rows = (await db.execute(text("""
SELECT symbol,
MIN(date)::date AS finra_first,
MAX(date)::date AS finra_last,
COUNT(DISTINCT date::date) AS finra_days
FROM finra_short_volume
GROUP BY symbol
"""))).fetchall()
finra_info = {r.symbol: (r.finra_first, r.finra_last, r.finra_days) for r in rows}
total_obs = sum(v[2] for v in finra_info.values())
logger.info(f"PIT backfill: {len(finra_info):,} FINRA symbols")
# Step 2: 기존 Alpaca 1d 커버리지 (정규화 키로 저장)
async with AsyncSessionLocal() as db:
cov_rows = (await db.execute(text("""
SELECT ticker, MAX(date)::date AS alpaca_max
FROM alpaca_price_data WHERE interval = '1d'
GROUP BY ticker
"""))).fetchall()
alpaca_cov: dict = {}
for r in cov_rows:
key = _finra_to_alpaca(r.ticker).upper()
if key not in alpaca_cov or r.alpaca_max > alpaca_cov[key]:
alpaca_cov[key] = r.alpaca_max
# Step 3: 수집 필요 심볼 결정
if force:
need_fetch = list(finra_info.keys())
else:
need_fetch = [
sym for sym, (_, finra_last, _) in finra_info.items()
if (am := alpaca_cov.get(_finra_to_alpaca(sym).upper())) is None
or (finra_last - am).days > TOLERANCE_DAYS
]
logger.info(f"PIT backfill: fetching {len(need_fetch):,} / {len(finra_info):,} symbols")
# Step 4: 배치 수집 + upsert
total_inserted = 0
n_batches = (len(need_fetch) + BATCH_SIZE - 1) // BATCH_SIZE
for batch_idx in range(0, len(need_fetch), BATCH_SIZE):
batch = need_fetch[batch_idx: batch_idx + BATCH_SIZE]
batch_num = batch_idx // BATCH_SIZE + 1
if batch_num == 1 or batch_num % 50 == 0:
logger.info(f"PIT backfill batch {batch_num}/{n_batches} | inserted={total_inserted:,}")
reverse_map = {_finra_to_alpaca(s).upper(): s for s in batch}
try:
raw = await client.get_multi_bars(
symbols=batch, timeframe="1d",
start=start_str, end=end_str,
adjustment=ADJUSTMENT,
)
except Exception as exc:
logger.warning(f"PIT backfill batch {batch_num} error: {exc}")
continue
rows_to_insert = []
for alpaca_sym, bar_list in raw.items():
if not bar_list:
continue
original = reverse_map.get(alpaca_sym.upper(), alpaca_sym)
for bar in bar_list:
dt = datetime.fromisoformat(bar["t"].replace("Z", "+00:00"))
if dt.tzinfo is None:
dt = dt.replace(tzinfo=timezone.utc)
rows_to_insert.append({
"ticker": original,
"date": dt,
"interval": "1d",
"open": float(bar.get("o") or 0),
"high": float(bar.get("h") or 0),
"low": float(bar.get("l") or 0),
"close": float(bar.get("c") or 0),
"volume": float(bar.get("v") or 0),
"vwap": float(bar["vw"]) if bar.get("vw") else None,
"trade_count": int(bar["n"]) if bar.get("n") else None,
"data_source": "ALPACA",
})
if rows_to_insert:
async with AsyncSessionLocal() as db:
for i in range(0, len(rows_to_insert), CHUNK_SIZE):
stmt = pg_insert(AlpacaPriceData).values(
rows_to_insert[i: i + CHUNK_SIZE]
)
stmt = stmt.on_conflict_do_nothing(constraint="uq_alpaca_price_data")
result = await db.execute(stmt)
total_inserted += result.rowcount
await asyncio.sleep(0)
await db.commit()
# Step 5: 완료 리포트
async with AsyncSessionLocal() as db:
new_cov = {
_finra_to_alpaca(r.ticker).upper()
for r in (await db.execute(text(
"SELECT DISTINCT ticker FROM alpaca_price_data WHERE interval='1d'"
))).fetchall()
}
covered_obs = sum(fd for sym, (_, _, fd) in finra_info.items() if _finra_to_alpaca(sym).upper() in new_cov)
missing_obs = total_obs - covered_obs
bias_pct = 100.0 * missing_obs / total_obs if total_obs else 0
logger.info(
f"PIT backfill complete — inserted={total_inserted:,} | "
f"covered={len(new_cov):,} symbols | "
f"residual bias={bias_pct:.2f}% (row-weighted)"
)
finally:
await client.close()
@router.post(
"/admin/backfill-pit",
summary="PIT 가격 백필 — 상폐 종목 포함 전체 FINRA 심볼",
description=(
"FINRA short-volume DB에 등장한 모든 심볼(현 활성 유니버스 + 상폐/합병 과거 심볼)의 "
"Alpaca SIP 일봉(1d)을 백필합니다. 생존편향-0 수익 계산에 필요.\n\n"
"**특성**:\n"
"- `adjustment=all` (분할+배당 조정) — 상폐 종목은 future-proof\n"
"- DB-first, idempotent (`on_conflict_do_nothing`) — 재실행 안전\n"
"- Alpaca SIP 일봉은 무료 플랜에서 2016-01-04부터 제공\n"
"- 기본 시작일: 2018-08-01 (FINRA DB 시작일)\n\n"
"**백그라운드 실행**: 즉시 `started` 응답, 1-3시간 소요.\n"
"진행 상황: `GET /alpaca/status` 또는 DB `SELECT COUNT(DISTINCT ticker) FROM alpaca_price_data WHERE interval='1d';`\n\n"
"**PIT 뷰** (백필 후): `SELECT DISTINCT symbol FROM pit_universe_membership WHERE d='2023-03-09';`"
),
)
async def backfill_pit_prices(
background_tasks: BackgroundTasks,
start_date: date = Query(date(2018, 8, 1), description="백필 시작일 (기본: 2018-08-01)"),
force: bool = Query(False, description="이미 커버된 심볼도 재수집"),
):
svc = _require_alpaca() # API 키 확인
_ = svc # 키 확인용
end_date = datetime.now(timezone.utc).date() - timedelta(days=1)
start_str = start_date.isoformat()
end_str = end_date.isoformat()
background_tasks.add_task(_run_pit_backfill, start_str, end_str, force)
return {
"status": "started",
"start_date": start_str,
"end_date": end_str,
"adjustment": "all",
"note": (
"Backfilling Alpaca 1d bars for all FINRA PIT symbols in background. "
"Typically 1-3 hours for ~22k symbols. Check container logs for progress."
),
}
@router.get(
"/snapshot",
response_model=AlpacaMultiSnapshotResponse,
summary="Real-time snapshots for multiple tickers (IEX feed)",
description=(
"멀티 종목 실시간 스냅샷. 최신 체결가, bid/ask, 당일 OHLCV, 전일 대비 변동률 포함.\n\n"
"단일 종목도 `?tickers=AAPL`로 조회 가능.\n\n"
"| 항목 | 내용 |\n"
"|------|------|\n"
"| 피드 | **IEX** — 무료 플랜에서 snapshot은 SIP 불가 |\n"
"| 지연 | **실시간** (지연 없음) |\n"
"| 거래량 | IEX 기준 (실제의 2~5%) |\n"
"| 캐시 | **없음** — 매 요청마다 Alpaca 직접 호출 |\n\n"
"- Requires `ALPACA_API_KEY` / `ALPACA_SECRET_KEY`"
),
)
async def get_snapshots(
tickers: str = Query(..., description="Comma-separated ticker symbols, e.g. AAPL,MSFT,NVDA"),
):
symbols = [s.strip().upper() for s in tickers.split(",") if s.strip()]
if not symbols:
raise HTTPException(status_code=400, detail="No tickers provided.")
if len(symbols) > 1000:
raise HTTPException(status_code=400, detail="Maximum 1000 tickers per request.")
client = AlpacaClient()
if not client.is_configured():
raise HTTPException(status_code=503, detail="Alpaca API keys not configured.")
try:
raw_map = await client.get_snapshots(symbols)
results = [_parse_snapshot(sym, raw_map.get(sym, {})) for sym in symbols]
return AlpacaMultiSnapshotResponse(count=len(results), snapshots=results)
except Exception as e:
raise HTTPException(status_code=502, detail=f"Alpaca API error: {e}")
finally:
await client.close()