fix: 대량 스캔 시 서버 사망 방지 — 세마포어 fast-fail + limit-concurrency 조정

- Alpaca intraday: 세마포어 대기 10초 제한, 초과 시 429 즉시 반환
  (기존: 120초 대기하며 커넥션 슬롯 점유 → 이벤트 루프 포화)
- Filing search: 세마포어(8) + 10초 fast-fail 추가
  (기존: 동시성 제한 없이 A-Z 스캔 시 수백 개 요청 쌓임)
- limit-concurrency 50→25 (실제 처리량은 세마포어가 제한하므로 여유 슬롯 불필요)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
main
I Luk Kim 4 months ago
parent 4e77043671
commit 0c4446a6ec

@ -14,10 +14,12 @@ _ET = ZoneInfo("America/New_York")
_MARKET_CLOSE_HOUR = 16 # 4:00 PM ET _MARKET_CLOSE_HOUR = 16 # 4:00 PM ET
# Limit concurrent Alpaca intraday processing to prevent event-loop saturation # Limit concurrent Alpaca intraday processing to prevent event-loop saturation
# under bulk backfill workloads. Callers beyond this limit wait on the semaphore # under bulk backfill workloads.
# (cheap asyncio wait) rather than flooding httpx connections and DB sessions.
# Semaphore(5): BaseHTTPMiddleware removed → task count halved → safe to allow 5.
_INTRADAY_SEMAPHORE = asyncio.Semaphore(5) _INTRADAY_SEMAPHORE = asyncio.Semaphore(5)
# Max seconds a request waits for a semaphore slot before returning 429.
# Prevents hundreds of requests piling up (each holding a connection slot)
# during A-Z full-ticker scans.
_SEMAPHORE_WAIT_TIMEOUT = 10
def _market_closed_for(d: date) -> bool: def _market_closed_for(d: date) -> bool:
@ -129,12 +131,21 @@ async def get_alpaca_intraday_multi(
end_dt = datetime.combine(_end, datetime.max.time()).replace(tzinfo=timezone.utc) end_dt = datetime.combine(_end, datetime.max.time()).replace(tzinfo=timezone.utc)
async def _do_fetch(): async def _do_fetch():
async with _INTRADAY_SEMAPHORE: try:
await asyncio.wait_for(
_INTRADAY_SEMAPHORE.acquire(), timeout=_SEMAPHORE_WAIT_TIMEOUT
)
except asyncio.TimeoutError:
raise HTTPException(
status_code=429,
detail="서버가 바빠서 요청을 처리할 수 없습니다. 잠시 후 다시 시도하세요.",
)
try: try:
return await svc.get_or_fetch_multi_bars( return await svc.get_or_fetch_multi_bars(
symbols, start_dt, end_dt, interval, force_refresh, feed="sip" symbols, start_dt, end_dt, interval, force_refresh, feed="sip"
) )
finally: finally:
_INTRADAY_SEMAPHORE.release()
try: try:
await svc.client.close() await svc.client.close()
except Exception: except Exception:
@ -219,12 +230,21 @@ async def get_alpaca_intraday_today(
end_dt = datetime.combine(today, datetime.max.time()).replace(tzinfo=timezone.utc) end_dt = datetime.combine(today, datetime.max.time()).replace(tzinfo=timezone.utc)
async def _do_fetch(): async def _do_fetch():
async with _INTRADAY_SEMAPHORE: try:
await asyncio.wait_for(
_INTRADAY_SEMAPHORE.acquire(), timeout=_SEMAPHORE_WAIT_TIMEOUT
)
except asyncio.TimeoutError:
raise HTTPException(
status_code=429,
detail="서버가 바빠서 요청을 처리할 수 없습니다. 잠시 후 다시 시도하세요.",
)
try: try:
return await svc.get_or_fetch_multi_bars( return await svc.get_or_fetch_multi_bars(
symbols, start_dt, end_dt, interval, force_refresh=True, feed="iex" symbols, start_dt, end_dt, interval, force_refresh=True, feed="iex"
) )
finally: finally:
_INTRADAY_SEMAPHORE.release()
try: try:
await svc.client.close() await svc.client.close()
except Exception: except Exception:

@ -37,6 +37,12 @@ from sqlalchemy import select, and_, func as sql_func
router = APIRouter() router = APIRouter()
logger = logging.getLogger("app.api.v1.filings") logger = logging.getLogger("app.api.v1.filings")
# Limit concurrent filing search requests to prevent event-loop saturation
# during bulk A-Z ticker scans. Requests beyond this wait up to
# _SEARCH_SEMAPHORE_TIMEOUT seconds, then fail fast with 429.
_SEARCH_SEMAPHORE = asyncio.Semaphore(8)
_SEARCH_SEMAPHORE_TIMEOUT = 10
@router.get( @router.get(
"/search/{ticker}", "/search/{ticker}",
@ -89,6 +95,18 @@ async def search_filings(
except ValueError: except ValueError:
raise HTTPException(status_code=400, detail="Invalid date format. Use YYYY-MM-DD") raise HTTPException(status_code=400, detail="Invalid date format. Use YYYY-MM-DD")
# Fast-fail when server is overloaded (bulk A-Z scans)
try:
await asyncio.wait_for(
_SEARCH_SEMAPHORE.acquire(), timeout=_SEARCH_SEMAPHORE_TIMEOUT
)
except asyncio.TimeoutError:
raise HTTPException(
status_code=429,
detail="서버가 바빠서 요청을 처리할 수 없습니다. 잠시 후 다시 시도하세요.",
)
try:
if force_refresh: if force_refresh:
try: try:
await sec_filings_service.index_filings( await sec_filings_service.index_filings(
@ -120,6 +138,8 @@ async def search_filings(
except Exception as e: except Exception as e:
logger.error(f"Filing search failed for {ticker}: {e}") logger.error(f"Filing search failed for {ticker}: {e}")
raise HTTPException(status_code=502, detail=f"Filing search failed: {e}") raise HTTPException(status_code=502, detail=f"Filing search failed: {e}")
finally:
_SEARCH_SEMAPHORE.release()
summaries = [] summaries = []
for f in filings: for f in filings:

@ -68,7 +68,7 @@ services:
mem_limit: 2g mem_limit: 2g
memswap_limit: 2g memswap_limit: 2g
restart: unless-stopped restart: unless-stopped
command: ["python", "-m", "uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "18000", "--reload", "--limit-concurrency", "50"] command: ["python", "-m", "uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "18000", "--reload", "--limit-concurrency", "25"]
# Frontend Application # Frontend Application
frontend: frontend:

Loading…
Cancel
Save