fix(sec): 대형 exhibit 파일로 인한 서버 메모리 고갈 방지
- fetch_text에 max_bytes 파라미터 추가: Content-Length 헤더로 다운로드 전 사전 reject, 헤더 없으면 content.read(max_bytes+1)로 제한적 읽기 - ValueError는 즉시 raise (retry 없음 — 크기는 재시도해도 안 줄어듦) - in-memory 캐시(_text_cache) 1MB 가드: 대형 응답은 디스크 캐시에만 저장 - MAX_EXHIBIT_SIZE 1MB → 5MB, fetch_text(max_bytes=...) 호출로 다운로드 전 체크 - exhibit deadline 30초 → 15초 (서비스 + 엔드포인트 + bulk) - 신규 테스트 5개: Content-Length 사전 거부, body 제한 읽기, 메모리 캐시 가드, 소형 캐시 유지, ValueError no-retry Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>main
I Luk Kim
5 months ago
parent
89113d45ab
commit
e2c242c7cc
@ -0,0 +1,413 @@
|
||||
"""
|
||||
Unit tests for SEC filings fixes:
|
||||
1. Backpressure: SECHttpClient._pending counter rejects when queue full
|
||||
2. Negative cache helpers: get_negative_cached / set_negative_cached
|
||||
3. Exhibit endpoint negative caching: 404 cached, skipped on repeat
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import pytest
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fix 1: Backpressure — _pending counter
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestSECHttpClientBackpressure:
|
||||
def _make_client(self):
|
||||
from app.services.sec_http_client import SECHttpClient
|
||||
client = SECHttpClient.__new__(SECHttpClient)
|
||||
client._pending = 0
|
||||
client._MAX_PENDING = 5
|
||||
client._text_cache = {}
|
||||
client._json_cache = {}
|
||||
client._cache_dir = "/tmp/test_sec_cache_bp"
|
||||
client._req_sem = asyncio.Semaphore(2)
|
||||
from app.services.sec_http_client import _TokenBucket
|
||||
client._rate_limiter = _TokenBucket(rate=100.0, capacity=100.0)
|
||||
client.http_timeout = MagicMock()
|
||||
client.http_timeout.total = 12
|
||||
client._session = None
|
||||
return client
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pending_counter_increments_and_decrements(self):
|
||||
"""_pending goes up during fetch and back to 0 after."""
|
||||
client = self._make_client()
|
||||
|
||||
# Patch _get_session to return a mock that yields a 200 response
|
||||
mock_resp = AsyncMock()
|
||||
mock_resp.status = 200
|
||||
mock_resp.text = AsyncMock(return_value="hello world")
|
||||
mock_resp.__aenter__ = AsyncMock(return_value=mock_resp)
|
||||
mock_resp.__aexit__ = AsyncMock(return_value=False)
|
||||
|
||||
mock_session = MagicMock()
|
||||
mock_session.get = MagicMock(return_value=mock_resp)
|
||||
|
||||
with patch.object(client, "_get_session", AsyncMock(return_value=mock_session)):
|
||||
with patch("app.services.sec_http_client._is_sec_block_page", return_value=False):
|
||||
with patch("os.path.exists", return_value=False):
|
||||
result = await client.fetch_text("http://example.com/test")
|
||||
|
||||
assert result == "hello world"
|
||||
assert client._pending == 0, "pending must be 0 after completion"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_queue_full_raises_immediately(self):
|
||||
"""When _pending >= _MAX_PENDING, fetch_text raises without touching SEC."""
|
||||
client = self._make_client()
|
||||
client._pending = client._MAX_PENDING # Already at limit
|
||||
|
||||
with patch("os.path.exists", return_value=False):
|
||||
with pytest.raises(RuntimeError, match="SEC request queue full"):
|
||||
await client.fetch_text("http://example.com/overload")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_queue_full_json_raises_immediately(self):
|
||||
"""Same check for fetch_json."""
|
||||
client = self._make_client()
|
||||
client._pending = client._MAX_PENDING
|
||||
|
||||
with patch("os.path.exists", return_value=False):
|
||||
with pytest.raises(RuntimeError, match="SEC request queue full"):
|
||||
await client.fetch_json("http://example.com/overload.json")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pending_decrements_on_exception(self):
|
||||
"""_pending must be decremented even when an exception is raised."""
|
||||
client = self._make_client()
|
||||
assert client._pending == 0
|
||||
|
||||
with patch("os.path.exists", return_value=False):
|
||||
with patch.object(client, "_get_session", AsyncMock(side_effect=RuntimeError("boom"))):
|
||||
with pytest.raises(Exception):
|
||||
await client.fetch_text("http://example.com/fail")
|
||||
|
||||
assert client._pending == 0, "pending must return to 0 after exception"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rate_reduced_to_8(self):
|
||||
"""Rate limiter must be 8.0 req/sec, not 10.0."""
|
||||
from app.services.sec_http_client import SECHttpClient
|
||||
client = SECHttpClient("Test")
|
||||
assert client._rate_limiter._rate == 8.0
|
||||
assert client._rate_limiter._capacity == 8.0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fix 2: Negative cache helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestNegativeCacheHelpers:
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_negative_cached_miss(self):
|
||||
"""Returns False when key not in Redis."""
|
||||
from app.utils.cache import get_negative_cached
|
||||
mock_redis = AsyncMock()
|
||||
mock_redis.exists = AsyncMock(return_value=0)
|
||||
|
||||
with patch("app.utils.cache.get_redis", AsyncMock(return_value=mock_redis)):
|
||||
result = await get_negative_cached("filings:exhibit:404:ACC:EX-99.1")
|
||||
|
||||
assert result is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_negative_cached_hit(self):
|
||||
"""Returns True when key exists in Redis."""
|
||||
from app.utils.cache import get_negative_cached
|
||||
mock_redis = AsyncMock()
|
||||
mock_redis.exists = AsyncMock(return_value=1)
|
||||
|
||||
with patch("app.utils.cache.get_redis", AsyncMock(return_value=mock_redis)):
|
||||
result = await get_negative_cached("filings:exhibit:404:ACC:EX-99.1")
|
||||
|
||||
assert result is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_set_negative_cached_calls_redis(self):
|
||||
"""set_negative_cached calls redis.set with b'1' and the TTL."""
|
||||
from app.utils.cache import set_negative_cached
|
||||
mock_redis = AsyncMock()
|
||||
mock_redis.set = AsyncMock()
|
||||
|
||||
with patch("app.utils.cache.get_redis", AsyncMock(return_value=mock_redis)):
|
||||
await set_negative_cached("filings:exhibit:404:ACC:EX-99.2", ttl=3600)
|
||||
|
||||
mock_redis.set.assert_called_once_with(
|
||||
"filings:exhibit:404:ACC:EX-99.2", b"1", ex=3600
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_negative_cache_graceful_no_redis(self):
|
||||
"""Both helpers return gracefully when Redis is unavailable."""
|
||||
from app.utils.cache import get_negative_cached, set_negative_cached
|
||||
|
||||
with patch("app.utils.cache.get_redis", AsyncMock(return_value=None)):
|
||||
assert await get_negative_cached("any:key") is False
|
||||
await set_negative_cached("any:key") # must not raise
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_negative_cache_graceful_redis_error(self):
|
||||
"""Both helpers swallow Redis errors."""
|
||||
from app.utils.cache import get_negative_cached, set_negative_cached
|
||||
mock_redis = AsyncMock()
|
||||
mock_redis.exists = AsyncMock(side_effect=Exception("connection reset"))
|
||||
mock_redis.set = AsyncMock(side_effect=Exception("connection reset"))
|
||||
|
||||
with patch("app.utils.cache.get_redis", AsyncMock(return_value=mock_redis)):
|
||||
assert await get_negative_cached("any:key") is False
|
||||
await set_negative_cached("any:key") # must not raise
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fix 3: Exhibit endpoint negative caching
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestExhibitEndpointNegativeCaching:
|
||||
"""Test that the GET /exhibit endpoint checks & sets the negative cache."""
|
||||
|
||||
def _make_request(self, accession_number="0001234567-26-000001", exhibit_type="EX-99.1"):
|
||||
return {"accession_number": accession_number, "exhibit_type": exhibit_type}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_negative_cache_hit_returns_404_immediately(self):
|
||||
"""When negative cache is set, endpoint returns 404 without calling service."""
|
||||
from fastapi import HTTPException
|
||||
from app.api.v1.endpoints.filings import get_exhibit_content
|
||||
|
||||
mock_db = AsyncMock()
|
||||
|
||||
with patch("app.api.v1.endpoints.filings.get_negative_cached", AsyncMock(return_value=True)):
|
||||
with patch("app.api.v1.endpoints.filings.sec_filings_service") as mock_svc:
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await get_exhibit_content(
|
||||
accession_number="0001234567-26-000001",
|
||||
response=MagicMock(),
|
||||
exhibit_type="EX-99.1",
|
||||
db=mock_db,
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 404
|
||||
mock_svc.get_exhibit_content.assert_not_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_value_error_sets_negative_cache(self):
|
||||
"""When service raises ValueError (exhibit not found), negative cache is set."""
|
||||
from fastapi import HTTPException
|
||||
from app.api.v1.endpoints.filings import get_exhibit_content
|
||||
|
||||
mock_db = AsyncMock()
|
||||
set_neg = AsyncMock()
|
||||
|
||||
with patch("app.api.v1.endpoints.filings.get_negative_cached", AsyncMock(return_value=False)):
|
||||
with patch("app.api.v1.endpoints.filings.set_negative_cached", set_neg):
|
||||
with patch("app.api.v1.endpoints.filings.sec_filings_service") as mock_svc:
|
||||
mock_svc.get_exhibit_content = AsyncMock(
|
||||
side_effect=ValueError("Exhibit EX-99.1 not found in filing ...")
|
||||
)
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await get_exhibit_content(
|
||||
accession_number="0001234567-26-000001",
|
||||
response=MagicMock(),
|
||||
exhibit_type="EX-99.1",
|
||||
db=mock_db,
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 404
|
||||
set_neg.assert_called_once()
|
||||
# Verify the key and TTL
|
||||
call_args = set_neg.call_args
|
||||
assert "EX-99.1" in call_args[0][0]
|
||||
assert call_args[1].get("ttl") == 3600 or (len(call_args[0]) > 1 and call_args[0][1] == 3600)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_value_error_does_not_set_negative_cache(self):
|
||||
"""Network/SEC errors (non-ValueError) must NOT populate negative cache."""
|
||||
from fastapi import HTTPException
|
||||
from app.api.v1.endpoints.filings import get_exhibit_content
|
||||
|
||||
mock_db = AsyncMock()
|
||||
set_neg = AsyncMock()
|
||||
|
||||
with patch("app.api.v1.endpoints.filings.get_negative_cached", AsyncMock(return_value=False)):
|
||||
with patch("app.api.v1.endpoints.filings.set_negative_cached", set_neg):
|
||||
with patch("app.api.v1.endpoints.filings.sec_filings_service") as mock_svc:
|
||||
mock_svc.get_exhibit_content = AsyncMock(
|
||||
side_effect=RuntimeError("SEC request queue full")
|
||||
)
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await get_exhibit_content(
|
||||
accession_number="0001234567-26-000001",
|
||||
response=MagicMock(),
|
||||
exhibit_type="EX-99.1",
|
||||
db=mock_db,
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 502
|
||||
set_neg.assert_not_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_successful_fetch_does_not_set_negative_cache(self):
|
||||
"""Successful exhibit fetch must not touch the negative cache."""
|
||||
from app.api.v1.endpoints.filings import get_exhibit_content
|
||||
|
||||
mock_db = AsyncMock()
|
||||
set_neg = AsyncMock()
|
||||
|
||||
with patch("app.api.v1.endpoints.filings.get_negative_cached", AsyncMock(return_value=False)):
|
||||
with patch("app.api.v1.endpoints.filings.set_negative_cached", set_neg):
|
||||
with patch("app.api.v1.endpoints.filings.sec_filings_service") as mock_svc:
|
||||
mock_svc.get_exhibit_content = AsyncMock(return_value={
|
||||
"content": "<html>Press Release</html>",
|
||||
"content_type": "text/html",
|
||||
"filename": "exhibit99-1.htm",
|
||||
"url": "https://www.sec.gov/Archives/edgar/data/123/000123/exhibit99-1.htm",
|
||||
})
|
||||
# Bypass the @with_cache decorator by patching set_cached_response
|
||||
with patch("app.utils.cache.set_cached_response", AsyncMock(return_value="etag")):
|
||||
with patch("app.utils.cache.get_cached_response", AsyncMock(return_value=None)):
|
||||
result = await get_exhibit_content(
|
||||
accession_number="0001234567-26-000001",
|
||||
response=MagicMock(),
|
||||
exhibit_type="EX-99.1",
|
||||
db=mock_db,
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
set_neg.assert_not_called()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fix 4 (large exhibit): max_bytes pre-check and memory cache guard
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestFetchTextMaxBytes:
|
||||
def _make_client(self):
|
||||
from app.services.sec_http_client import SECHttpClient, _TokenBucket
|
||||
client = SECHttpClient.__new__(SECHttpClient)
|
||||
client._pending = 0
|
||||
client._MAX_PENDING = 50
|
||||
client._text_cache = {}
|
||||
client._json_cache = {}
|
||||
client._cache_dir = "/tmp/test_sec_cache_mb"
|
||||
client._req_sem = asyncio.Semaphore(2)
|
||||
client._rate_limiter = _TokenBucket(rate=100.0, capacity=100.0)
|
||||
client.http_timeout = MagicMock()
|
||||
client.http_timeout.total = 12
|
||||
client._session = None
|
||||
return client
|
||||
|
||||
def _make_resp(self, content_length=None, body=b"x" * 200):
|
||||
mock_resp = AsyncMock()
|
||||
mock_resp.status = 200
|
||||
mock_resp.content_length = content_length
|
||||
mock_resp.get_encoding = MagicMock(return_value="utf-8")
|
||||
# content.read returns the body bytes
|
||||
mock_resp.content = AsyncMock()
|
||||
mock_resp.content.read = AsyncMock(return_value=body)
|
||||
mock_resp.__aenter__ = AsyncMock(return_value=mock_resp)
|
||||
mock_resp.__aexit__ = AsyncMock(return_value=False)
|
||||
return mock_resp
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_content_length_too_large_raises_immediately(self):
|
||||
"""fetch_text with max_bytes rejects via Content-Length header before reading body."""
|
||||
client = self._make_client()
|
||||
mock_resp = self._make_resp(content_length=5000, body=b"x" * 5000)
|
||||
mock_session = MagicMock()
|
||||
mock_session.get = MagicMock(return_value=mock_resp)
|
||||
|
||||
with patch.object(client, "_get_session", AsyncMock(return_value=mock_session)):
|
||||
with patch("os.path.exists", return_value=False):
|
||||
with pytest.raises(ValueError, match="too large"):
|
||||
await client.fetch_text("http://example.com/big", max_bytes=100)
|
||||
|
||||
# Body should not have been read when Content-Length already exceeded
|
||||
mock_resp.content.read.assert_not_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_content_length_body_too_large_raises(self):
|
||||
"""fetch_text with max_bytes rejects via body read when Content-Length is absent."""
|
||||
client = self._make_client()
|
||||
# No Content-Length header, but body exceeds limit
|
||||
mock_resp = self._make_resp(content_length=None, body=b"x" * 200)
|
||||
mock_session = MagicMock()
|
||||
mock_session.get = MagicMock(return_value=mock_resp)
|
||||
|
||||
with patch.object(client, "_get_session", AsyncMock(return_value=mock_session)):
|
||||
with patch("os.path.exists", return_value=False):
|
||||
with pytest.raises(ValueError, match="exceeded"):
|
||||
await client.fetch_text("http://example.com/big2", max_bytes=100)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_large_text_not_stored_in_memory_cache(self):
|
||||
"""Responses > 1MB must NOT be stored in _text_cache (only disk cache)."""
|
||||
client = self._make_client()
|
||||
large_body = b"A" * (2 * 1024 * 1024) # 2MB
|
||||
mock_resp = AsyncMock()
|
||||
mock_resp.status = 200
|
||||
mock_resp.content_length = None
|
||||
mock_resp.text = AsyncMock(return_value=large_body.decode("utf-8"))
|
||||
mock_resp.__aenter__ = AsyncMock(return_value=mock_resp)
|
||||
mock_resp.__aexit__ = AsyncMock(return_value=False)
|
||||
mock_session = MagicMock()
|
||||
mock_session.get = MagicMock(return_value=mock_resp)
|
||||
|
||||
with patch.object(client, "_get_session", AsyncMock(return_value=mock_session)):
|
||||
with patch("app.services.sec_http_client._is_sec_block_page", return_value=False):
|
||||
with patch("os.path.exists", return_value=False):
|
||||
with patch("builtins.open", MagicMock()):
|
||||
result = await client.fetch_text("http://example.com/large")
|
||||
|
||||
assert "http://example.com/large" not in client._text_cache
|
||||
assert len(result) == len(large_body)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_small_text_stored_in_memory_cache(self):
|
||||
"""Responses <= 1MB should still be stored in _text_cache."""
|
||||
client = self._make_client()
|
||||
small_body = "hello world"
|
||||
mock_resp = AsyncMock()
|
||||
mock_resp.status = 200
|
||||
mock_resp.content_length = None
|
||||
mock_resp.text = AsyncMock(return_value=small_body)
|
||||
mock_resp.__aenter__ = AsyncMock(return_value=mock_resp)
|
||||
mock_resp.__aexit__ = AsyncMock(return_value=False)
|
||||
mock_session = MagicMock()
|
||||
mock_session.get = MagicMock(return_value=mock_resp)
|
||||
|
||||
with patch.object(client, "_get_session", AsyncMock(return_value=mock_session)):
|
||||
with patch("app.services.sec_http_client._is_sec_block_page", return_value=False):
|
||||
with patch("os.path.exists", return_value=False):
|
||||
with patch("builtins.open", MagicMock()):
|
||||
result = await client.fetch_text("http://example.com/small")
|
||||
|
||||
assert client._text_cache.get("http://example.com/small") == small_body
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_size_error_does_not_retry(self):
|
||||
"""ValueError from max_bytes must propagate immediately without retrying."""
|
||||
client = self._make_client()
|
||||
mock_resp = self._make_resp(content_length=None, body=b"x" * 200)
|
||||
mock_session = MagicMock()
|
||||
mock_session.get = MagicMock(return_value=mock_resp)
|
||||
call_count = 0
|
||||
|
||||
original_get = mock_session.get
|
||||
|
||||
def counting_get(*args, **kwargs):
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
return original_get(*args, **kwargs)
|
||||
|
||||
mock_session.get = counting_get
|
||||
|
||||
with patch.object(client, "_get_session", AsyncMock(return_value=mock_session)):
|
||||
with patch("os.path.exists", return_value=False):
|
||||
with pytest.raises(ValueError):
|
||||
await client.fetch_text("http://example.com/nretry", max_bytes=100)
|
||||
|
||||
# Should have tried exactly once (no retry on ValueError)
|
||||
assert call_count == 1, f"Expected 1 attempt, got {call_count}"
|
||||
Loading…
Reference in New Issue