You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
414 lines
18 KiB
Python
414 lines
18 KiB
Python
"""
|
|
Unit tests for SEC filings fixes:
|
|
1. Backpressure: SECHttpClient._pending counter rejects when queue full
|
|
2. Negative cache helpers: get_negative_cached / set_negative_cached
|
|
3. Exhibit endpoint negative caching: 404 cached, skipped on repeat
|
|
"""
|
|
|
|
import asyncio
|
|
import pytest
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fix 1: Backpressure — _pending counter
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestSECHttpClientBackpressure:
|
|
def _make_client(self):
|
|
from app.services.sec_http_client import SECHttpClient
|
|
client = SECHttpClient.__new__(SECHttpClient)
|
|
client._pending = 0
|
|
client._MAX_PENDING = 5
|
|
client._text_cache = {}
|
|
client._json_cache = {}
|
|
client._cache_dir = "/tmp/test_sec_cache_bp"
|
|
client._req_sem = asyncio.Semaphore(2)
|
|
from app.services.sec_http_client import _TokenBucket
|
|
client._rate_limiter = _TokenBucket(rate=100.0, capacity=100.0)
|
|
client.http_timeout = MagicMock()
|
|
client.http_timeout.total = 12
|
|
client._session = None
|
|
return client
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_pending_counter_increments_and_decrements(self):
|
|
"""_pending goes up during fetch and back to 0 after."""
|
|
client = self._make_client()
|
|
|
|
# Patch _get_session to return a mock that yields a 200 response
|
|
mock_resp = AsyncMock()
|
|
mock_resp.status = 200
|
|
mock_resp.text = AsyncMock(return_value="hello world")
|
|
mock_resp.__aenter__ = AsyncMock(return_value=mock_resp)
|
|
mock_resp.__aexit__ = AsyncMock(return_value=False)
|
|
|
|
mock_session = MagicMock()
|
|
mock_session.get = MagicMock(return_value=mock_resp)
|
|
|
|
with patch.object(client, "_get_session", AsyncMock(return_value=mock_session)):
|
|
with patch("app.services.sec_http_client._is_sec_block_page", return_value=False):
|
|
with patch("os.path.exists", return_value=False):
|
|
result = await client.fetch_text("http://example.com/test")
|
|
|
|
assert result == "hello world"
|
|
assert client._pending == 0, "pending must be 0 after completion"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_queue_full_raises_immediately(self):
|
|
"""When _pending >= _MAX_PENDING, fetch_text raises without touching SEC."""
|
|
client = self._make_client()
|
|
client._pending = client._MAX_PENDING # Already at limit
|
|
|
|
with patch("os.path.exists", return_value=False):
|
|
with pytest.raises(RuntimeError, match="SEC request queue full"):
|
|
await client.fetch_text("http://example.com/overload")
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_queue_full_json_raises_immediately(self):
|
|
"""Same check for fetch_json."""
|
|
client = self._make_client()
|
|
client._pending = client._MAX_PENDING
|
|
|
|
with patch("os.path.exists", return_value=False):
|
|
with pytest.raises(RuntimeError, match="SEC request queue full"):
|
|
await client.fetch_json("http://example.com/overload.json")
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_pending_decrements_on_exception(self):
|
|
"""_pending must be decremented even when an exception is raised."""
|
|
client = self._make_client()
|
|
assert client._pending == 0
|
|
|
|
with patch("os.path.exists", return_value=False):
|
|
with patch.object(client, "_get_session", AsyncMock(side_effect=RuntimeError("boom"))):
|
|
with pytest.raises(Exception):
|
|
await client.fetch_text("http://example.com/fail")
|
|
|
|
assert client._pending == 0, "pending must return to 0 after exception"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_rate_reduced_to_8(self):
|
|
"""Rate limiter must be 8.0 req/sec, not 10.0."""
|
|
from app.services.sec_http_client import SECHttpClient
|
|
client = SECHttpClient("Test")
|
|
assert client._rate_limiter._rate == 8.0
|
|
assert client._rate_limiter._capacity == 8.0
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fix 2: Negative cache helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestNegativeCacheHelpers:
|
|
@pytest.mark.asyncio
|
|
async def test_get_negative_cached_miss(self):
|
|
"""Returns False when key not in Redis."""
|
|
from app.utils.cache import get_negative_cached
|
|
mock_redis = AsyncMock()
|
|
mock_redis.exists = AsyncMock(return_value=0)
|
|
|
|
with patch("app.utils.cache.get_redis", AsyncMock(return_value=mock_redis)):
|
|
result = await get_negative_cached("filings:exhibit:404:ACC:EX-99.1")
|
|
|
|
assert result is False
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_negative_cached_hit(self):
|
|
"""Returns True when key exists in Redis."""
|
|
from app.utils.cache import get_negative_cached
|
|
mock_redis = AsyncMock()
|
|
mock_redis.exists = AsyncMock(return_value=1)
|
|
|
|
with patch("app.utils.cache.get_redis", AsyncMock(return_value=mock_redis)):
|
|
result = await get_negative_cached("filings:exhibit:404:ACC:EX-99.1")
|
|
|
|
assert result is True
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_set_negative_cached_calls_redis(self):
|
|
"""set_negative_cached calls redis.set with b'1' and the TTL."""
|
|
from app.utils.cache import set_negative_cached
|
|
mock_redis = AsyncMock()
|
|
mock_redis.set = AsyncMock()
|
|
|
|
with patch("app.utils.cache.get_redis", AsyncMock(return_value=mock_redis)):
|
|
await set_negative_cached("filings:exhibit:404:ACC:EX-99.2", ttl=3600)
|
|
|
|
mock_redis.set.assert_called_once_with(
|
|
"filings:exhibit:404:ACC:EX-99.2", b"1", ex=3600
|
|
)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_negative_cache_graceful_no_redis(self):
|
|
"""Both helpers return gracefully when Redis is unavailable."""
|
|
from app.utils.cache import get_negative_cached, set_negative_cached
|
|
|
|
with patch("app.utils.cache.get_redis", AsyncMock(return_value=None)):
|
|
assert await get_negative_cached("any:key") is False
|
|
await set_negative_cached("any:key") # must not raise
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_negative_cache_graceful_redis_error(self):
|
|
"""Both helpers swallow Redis errors."""
|
|
from app.utils.cache import get_negative_cached, set_negative_cached
|
|
mock_redis = AsyncMock()
|
|
mock_redis.exists = AsyncMock(side_effect=Exception("connection reset"))
|
|
mock_redis.set = AsyncMock(side_effect=Exception("connection reset"))
|
|
|
|
with patch("app.utils.cache.get_redis", AsyncMock(return_value=mock_redis)):
|
|
assert await get_negative_cached("any:key") is False
|
|
await set_negative_cached("any:key") # must not raise
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fix 3: Exhibit endpoint negative caching
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestExhibitEndpointNegativeCaching:
|
|
"""Test that the GET /exhibit endpoint checks & sets the negative cache."""
|
|
|
|
def _make_request(self, accession_number="0001234567-26-000001", exhibit_type="EX-99.1"):
|
|
return {"accession_number": accession_number, "exhibit_type": exhibit_type}
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_negative_cache_hit_returns_404_immediately(self):
|
|
"""When negative cache is set, endpoint returns 404 without calling service."""
|
|
from fastapi import HTTPException
|
|
from app.api.v1.endpoints.filings import get_exhibit_content
|
|
|
|
mock_db = AsyncMock()
|
|
|
|
with patch("app.api.v1.endpoints.filings.get_negative_cached", AsyncMock(return_value=True)):
|
|
with patch("app.api.v1.endpoints.filings.sec_filings_service") as mock_svc:
|
|
with pytest.raises(HTTPException) as exc_info:
|
|
await get_exhibit_content(
|
|
accession_number="0001234567-26-000001",
|
|
response=MagicMock(),
|
|
exhibit_type="EX-99.1",
|
|
db=mock_db,
|
|
)
|
|
|
|
assert exc_info.value.status_code == 404
|
|
mock_svc.get_exhibit_content.assert_not_called()
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_value_error_sets_negative_cache(self):
|
|
"""When service raises ValueError (exhibit not found), negative cache is set."""
|
|
from fastapi import HTTPException
|
|
from app.api.v1.endpoints.filings import get_exhibit_content
|
|
|
|
mock_db = AsyncMock()
|
|
set_neg = AsyncMock()
|
|
|
|
with patch("app.api.v1.endpoints.filings.get_negative_cached", AsyncMock(return_value=False)):
|
|
with patch("app.api.v1.endpoints.filings.set_negative_cached", set_neg):
|
|
with patch("app.api.v1.endpoints.filings.sec_filings_service") as mock_svc:
|
|
mock_svc.get_exhibit_content = AsyncMock(
|
|
side_effect=ValueError("Exhibit EX-99.1 not found in filing ...")
|
|
)
|
|
with pytest.raises(HTTPException) as exc_info:
|
|
await get_exhibit_content(
|
|
accession_number="0001234567-26-000001",
|
|
response=MagicMock(),
|
|
exhibit_type="EX-99.1",
|
|
db=mock_db,
|
|
)
|
|
|
|
assert exc_info.value.status_code == 404
|
|
set_neg.assert_called_once()
|
|
# Verify the key and TTL
|
|
call_args = set_neg.call_args
|
|
assert "EX-99.1" in call_args[0][0]
|
|
assert call_args[1].get("ttl") == 3600 or (len(call_args[0]) > 1 and call_args[0][1] == 3600)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_non_value_error_does_not_set_negative_cache(self):
|
|
"""Network/SEC errors (non-ValueError) must NOT populate negative cache."""
|
|
from fastapi import HTTPException
|
|
from app.api.v1.endpoints.filings import get_exhibit_content
|
|
|
|
mock_db = AsyncMock()
|
|
set_neg = AsyncMock()
|
|
|
|
with patch("app.api.v1.endpoints.filings.get_negative_cached", AsyncMock(return_value=False)):
|
|
with patch("app.api.v1.endpoints.filings.set_negative_cached", set_neg):
|
|
with patch("app.api.v1.endpoints.filings.sec_filings_service") as mock_svc:
|
|
mock_svc.get_exhibit_content = AsyncMock(
|
|
side_effect=RuntimeError("SEC request queue full")
|
|
)
|
|
with pytest.raises(HTTPException) as exc_info:
|
|
await get_exhibit_content(
|
|
accession_number="0001234567-26-000001",
|
|
response=MagicMock(),
|
|
exhibit_type="EX-99.1",
|
|
db=mock_db,
|
|
)
|
|
|
|
assert exc_info.value.status_code == 502
|
|
set_neg.assert_not_called()
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_successful_fetch_does_not_set_negative_cache(self):
|
|
"""Successful exhibit fetch must not touch the negative cache."""
|
|
from app.api.v1.endpoints.filings import get_exhibit_content
|
|
|
|
mock_db = AsyncMock()
|
|
set_neg = AsyncMock()
|
|
|
|
with patch("app.api.v1.endpoints.filings.get_negative_cached", AsyncMock(return_value=False)):
|
|
with patch("app.api.v1.endpoints.filings.set_negative_cached", set_neg):
|
|
with patch("app.api.v1.endpoints.filings.sec_filings_service") as mock_svc:
|
|
mock_svc.get_exhibit_content = AsyncMock(return_value={
|
|
"content": "<html>Press Release</html>",
|
|
"content_type": "text/html",
|
|
"filename": "exhibit99-1.htm",
|
|
"url": "https://www.sec.gov/Archives/edgar/data/123/000123/exhibit99-1.htm",
|
|
})
|
|
# Bypass the @with_cache decorator by patching set_cached_response
|
|
with patch("app.utils.cache.set_cached_response", AsyncMock(return_value="etag")):
|
|
with patch("app.utils.cache.get_cached_response", AsyncMock(return_value=None)):
|
|
result = await get_exhibit_content(
|
|
accession_number="0001234567-26-000001",
|
|
response=MagicMock(),
|
|
exhibit_type="EX-99.1",
|
|
db=mock_db,
|
|
)
|
|
|
|
assert result is not None
|
|
set_neg.assert_not_called()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fix 4 (large exhibit): max_bytes pre-check and memory cache guard
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestFetchTextMaxBytes:
|
|
def _make_client(self):
|
|
from app.services.sec_http_client import SECHttpClient, _TokenBucket
|
|
client = SECHttpClient.__new__(SECHttpClient)
|
|
client._pending = 0
|
|
client._MAX_PENDING = 50
|
|
client._text_cache = {}
|
|
client._json_cache = {}
|
|
client._cache_dir = "/tmp/test_sec_cache_mb"
|
|
client._req_sem = asyncio.Semaphore(2)
|
|
client._rate_limiter = _TokenBucket(rate=100.0, capacity=100.0)
|
|
client.http_timeout = MagicMock()
|
|
client.http_timeout.total = 12
|
|
client._session = None
|
|
return client
|
|
|
|
def _make_resp(self, content_length=None, body=b"x" * 200):
|
|
mock_resp = AsyncMock()
|
|
mock_resp.status = 200
|
|
mock_resp.content_length = content_length
|
|
mock_resp.get_encoding = MagicMock(return_value="utf-8")
|
|
# content.read returns the body bytes
|
|
mock_resp.content = AsyncMock()
|
|
mock_resp.content.read = AsyncMock(return_value=body)
|
|
mock_resp.__aenter__ = AsyncMock(return_value=mock_resp)
|
|
mock_resp.__aexit__ = AsyncMock(return_value=False)
|
|
return mock_resp
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_content_length_too_large_raises_immediately(self):
|
|
"""fetch_text with max_bytes rejects via Content-Length header before reading body."""
|
|
client = self._make_client()
|
|
mock_resp = self._make_resp(content_length=5000, body=b"x" * 5000)
|
|
mock_session = MagicMock()
|
|
mock_session.get = MagicMock(return_value=mock_resp)
|
|
|
|
with patch.object(client, "_get_session", AsyncMock(return_value=mock_session)):
|
|
with patch("os.path.exists", return_value=False):
|
|
with pytest.raises(ValueError, match="too large"):
|
|
await client.fetch_text("http://example.com/big", max_bytes=100)
|
|
|
|
# Body should not have been read when Content-Length already exceeded
|
|
mock_resp.content.read.assert_not_called()
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_no_content_length_body_too_large_raises(self):
|
|
"""fetch_text with max_bytes rejects via body read when Content-Length is absent."""
|
|
client = self._make_client()
|
|
# No Content-Length header, but body exceeds limit
|
|
mock_resp = self._make_resp(content_length=None, body=b"x" * 200)
|
|
mock_session = MagicMock()
|
|
mock_session.get = MagicMock(return_value=mock_resp)
|
|
|
|
with patch.object(client, "_get_session", AsyncMock(return_value=mock_session)):
|
|
with patch("os.path.exists", return_value=False):
|
|
with pytest.raises(ValueError, match="exceeded"):
|
|
await client.fetch_text("http://example.com/big2", max_bytes=100)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_large_text_not_stored_in_memory_cache(self):
|
|
"""Responses > 1MB must NOT be stored in _text_cache (only disk cache)."""
|
|
client = self._make_client()
|
|
large_body = b"A" * (2 * 1024 * 1024) # 2MB
|
|
mock_resp = AsyncMock()
|
|
mock_resp.status = 200
|
|
mock_resp.content_length = None
|
|
mock_resp.text = AsyncMock(return_value=large_body.decode("utf-8"))
|
|
mock_resp.__aenter__ = AsyncMock(return_value=mock_resp)
|
|
mock_resp.__aexit__ = AsyncMock(return_value=False)
|
|
mock_session = MagicMock()
|
|
mock_session.get = MagicMock(return_value=mock_resp)
|
|
|
|
with patch.object(client, "_get_session", AsyncMock(return_value=mock_session)):
|
|
with patch("app.services.sec_http_client._is_sec_block_page", return_value=False):
|
|
with patch("os.path.exists", return_value=False):
|
|
with patch("builtins.open", MagicMock()):
|
|
result = await client.fetch_text("http://example.com/large")
|
|
|
|
assert "http://example.com/large" not in client._text_cache
|
|
assert len(result) == len(large_body)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_small_text_stored_in_memory_cache(self):
|
|
"""Responses <= 1MB should still be stored in _text_cache."""
|
|
client = self._make_client()
|
|
small_body = "hello world"
|
|
mock_resp = AsyncMock()
|
|
mock_resp.status = 200
|
|
mock_resp.content_length = None
|
|
mock_resp.text = AsyncMock(return_value=small_body)
|
|
mock_resp.__aenter__ = AsyncMock(return_value=mock_resp)
|
|
mock_resp.__aexit__ = AsyncMock(return_value=False)
|
|
mock_session = MagicMock()
|
|
mock_session.get = MagicMock(return_value=mock_resp)
|
|
|
|
with patch.object(client, "_get_session", AsyncMock(return_value=mock_session)):
|
|
with patch("app.services.sec_http_client._is_sec_block_page", return_value=False):
|
|
with patch("os.path.exists", return_value=False):
|
|
with patch("builtins.open", MagicMock()):
|
|
result = await client.fetch_text("http://example.com/small")
|
|
|
|
assert client._text_cache.get("http://example.com/small") == small_body
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_size_error_does_not_retry(self):
|
|
"""ValueError from max_bytes must propagate immediately without retrying."""
|
|
client = self._make_client()
|
|
mock_resp = self._make_resp(content_length=None, body=b"x" * 200)
|
|
mock_session = MagicMock()
|
|
mock_session.get = MagicMock(return_value=mock_resp)
|
|
call_count = 0
|
|
|
|
original_get = mock_session.get
|
|
|
|
def counting_get(*args, **kwargs):
|
|
nonlocal call_count
|
|
call_count += 1
|
|
return original_get(*args, **kwargs)
|
|
|
|
mock_session.get = counting_get
|
|
|
|
with patch.object(client, "_get_session", AsyncMock(return_value=mock_session)):
|
|
with patch("os.path.exists", return_value=False):
|
|
with pytest.raises(ValueError):
|
|
await client.fetch_text("http://example.com/nretry", max_bytes=100)
|
|
|
|
# Should have tried exactly once (no retry on ValueError)
|
|
assert call_count == 1, f"Expected 1 attempt, got {call_count}"
|