fix: request_logs 18GB 블로트 방지 — 7일 retention + autovacuum 강화

원인: request_logs에 cleanup 정책이 없어 65M행이 쌓였다가 DELETE되면서
VACUUM FULL 없이 18GB dead page bloat 발생.
PostgreSQL 메모리/디스크 압박 → API worker OOM kill의 간접 원인.

수정:
- _request_log_cleanup() 백그라운드 코루틴 추가
  → 1시간마다 실행, 7일 초과 request_logs 자동 DELETE
  → start_request_log_flusher()에서 함께 스케줄
- request_logs autovacuum scale_factor 0.2 → 0.01
  → 1% 변경 시 autovacuum 즉시 실행, dead tuple 빠르게 회수

즉각 조치: TRUNCATE request_logs 실행 (25GB → 56kB)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
main
I Luk Kim 4 months ago
parent abfab0fa57
commit 7e1a4699e9

@ -16,7 +16,7 @@ import json
import time import time
import traceback import traceback
import uuid import uuid
from datetime import datetime, timezone from datetime import datetime, timedelta, timezone
from typing import Callable, Optional from typing import Callable, Optional
from fastapi import Request, Response from fastapi import Request, Response
@ -38,6 +38,8 @@ logger = logging.getLogger(__name__)
_REQUEST_LOG_QUEUE: asyncio.Queue = asyncio.Queue(maxsize=10_000) _REQUEST_LOG_QUEUE: asyncio.Queue = asyncio.Queue(maxsize=10_000)
_FLUSH_INTERVAL_SECONDS: float = 2.0 # flush at most every 2 s _FLUSH_INTERVAL_SECONDS: float = 2.0 # flush at most every 2 s
_FLUSH_BATCH_SIZE: int = 500 # or when 500 entries are queued _FLUSH_BATCH_SIZE: int = 500 # or when 500 entries are queued
_LOG_RETENTION_DAYS: int = 7 # delete request_logs older than this
_CLEANUP_INTERVAL_HOURS: int = 1 # run cleanup every N hours
async def _flush_request_logs(entries: list) -> None: async def _flush_request_logs(entries: list) -> None:
@ -82,9 +84,31 @@ async def _request_log_flusher() -> None:
await _flush_request_logs(entries) await _flush_request_logs(entries)
async def _request_log_cleanup() -> None:
"""Background coroutine: delete request_logs older than _LOG_RETENTION_DAYS."""
while True:
await asyncio.sleep(_CLEANUP_INTERVAL_HOURS * 3600)
try:
cutoff = datetime.now(timezone.utc) - timedelta(days=_LOG_RETENTION_DAYS)
from app.core.database import AsyncSessionLocal
from sqlalchemy import text
async with AsyncSessionLocal() as db:
result = await db.execute(
text("DELETE FROM request_logs WHERE created_at < :cutoff"),
{"cutoff": cutoff},
)
deleted = result.rowcount
await db.commit()
if deleted:
logger.info(f"request_logs cleanup: deleted {deleted} rows older than {_LOG_RETENTION_DAYS}d")
except Exception as e:
logger.error(f"request_logs cleanup failed: {e}")
def start_request_log_flusher() -> None: def start_request_log_flusher() -> None:
"""Schedule the background flusher coroutine. Call once at app startup.""" """Schedule the background flusher coroutine. Call once at app startup."""
asyncio.ensure_future(_request_log_flusher()) asyncio.ensure_future(_request_log_flusher())
asyncio.ensure_future(_request_log_cleanup())
logger.info("Request log flusher background task started") logger.info("Request log flusher background task started")

Loading…
Cancel
Save