You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
646 lines
28 KiB
Python
646 lines
28 KiB
Python
"""
|
|
Unit tests for SEC filings fixes:
|
|
1. Backpressure: SECHttpClient._pending counter rejects when queue full
|
|
2. Negative cache helpers: get_negative_cached / set_negative_cached
|
|
3. Exhibit endpoint negative caching: 404 cached, skipped on repeat
|
|
"""
|
|
|
|
import asyncio
|
|
import pytest
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fix 1: Backpressure — _pending counter
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestSECHttpClientBackpressure:
|
|
def _make_client(self):
|
|
from app.services.sec_http_client import SECHttpClient
|
|
client = SECHttpClient.__new__(SECHttpClient)
|
|
client._pending = 0
|
|
client._MAX_PENDING = 5
|
|
client._text_cache = {}
|
|
client._json_cache = {}
|
|
client._cache_dir = "/tmp/test_sec_cache_bp"
|
|
client._req_sem = asyncio.Semaphore(2)
|
|
from app.services.sec_http_client import _TokenBucket
|
|
client._rate_limiter = _TokenBucket(rate=100.0, capacity=100.0)
|
|
client.http_timeout = MagicMock()
|
|
client.http_timeout.total = 12
|
|
client._session = None
|
|
return client
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_pending_counter_increments_and_decrements(self):
|
|
"""_pending goes up during fetch and back to 0 after."""
|
|
client = self._make_client()
|
|
|
|
# Patch _get_session to return a mock that yields a 200 response
|
|
mock_resp = AsyncMock()
|
|
mock_resp.status = 200
|
|
mock_resp.text = AsyncMock(return_value="hello world")
|
|
mock_resp.__aenter__ = AsyncMock(return_value=mock_resp)
|
|
mock_resp.__aexit__ = AsyncMock(return_value=False)
|
|
|
|
mock_session = MagicMock()
|
|
mock_session.get = MagicMock(return_value=mock_resp)
|
|
|
|
with patch.object(client, "_get_session", AsyncMock(return_value=mock_session)):
|
|
with patch("app.services.sec_http_client._is_sec_block_page", return_value=False):
|
|
with patch("os.path.exists", return_value=False):
|
|
result = await client.fetch_text("http://example.com/test")
|
|
|
|
assert result == "hello world"
|
|
assert client._pending == 0, "pending must be 0 after completion"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_queue_full_raises_immediately(self):
|
|
"""When _pending >= _MAX_PENDING, fetch_text raises without touching SEC."""
|
|
client = self._make_client()
|
|
client._pending = client._MAX_PENDING # Already at limit
|
|
|
|
with patch("os.path.exists", return_value=False):
|
|
with pytest.raises(RuntimeError, match="SEC request queue full"):
|
|
await client.fetch_text("http://example.com/overload")
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_queue_full_json_raises_immediately(self):
|
|
"""Same check for fetch_json."""
|
|
client = self._make_client()
|
|
client._pending = client._MAX_PENDING
|
|
|
|
with patch("os.path.exists", return_value=False):
|
|
with pytest.raises(RuntimeError, match="SEC request queue full"):
|
|
await client.fetch_json("http://example.com/overload.json")
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_pending_decrements_on_exception(self):
|
|
"""_pending must be decremented even when an exception is raised."""
|
|
client = self._make_client()
|
|
assert client._pending == 0
|
|
|
|
with patch("os.path.exists", return_value=False):
|
|
with patch.object(client, "_get_session", AsyncMock(side_effect=RuntimeError("boom"))):
|
|
with pytest.raises(Exception):
|
|
await client.fetch_text("http://example.com/fail")
|
|
|
|
assert client._pending == 0, "pending must return to 0 after exception"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_rate_reduced_to_8(self):
|
|
"""Rate limiter must be 8.0 req/sec, not 10.0."""
|
|
from app.services.sec_http_client import SECHttpClient
|
|
client = SECHttpClient("Test")
|
|
assert client._rate_limiter._rate == 8.0
|
|
assert client._rate_limiter._capacity == 8.0
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fix 2: Negative cache helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestNegativeCacheHelpers:
|
|
@pytest.mark.asyncio
|
|
async def test_get_negative_cached_miss(self):
|
|
"""Returns False when key not in Redis."""
|
|
from app.utils.cache import get_negative_cached
|
|
mock_redis = AsyncMock()
|
|
mock_redis.exists = AsyncMock(return_value=0)
|
|
|
|
with patch("app.utils.cache.get_redis", AsyncMock(return_value=mock_redis)):
|
|
result = await get_negative_cached("filings:exhibit:404:ACC:EX-99.1")
|
|
|
|
assert result is False
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_get_negative_cached_hit(self):
|
|
"""Returns True when key exists in Redis."""
|
|
from app.utils.cache import get_negative_cached
|
|
mock_redis = AsyncMock()
|
|
mock_redis.exists = AsyncMock(return_value=1)
|
|
|
|
with patch("app.utils.cache.get_redis", AsyncMock(return_value=mock_redis)):
|
|
result = await get_negative_cached("filings:exhibit:404:ACC:EX-99.1")
|
|
|
|
assert result is True
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_set_negative_cached_calls_redis(self):
|
|
"""set_negative_cached calls redis.set with b'1' and the TTL."""
|
|
from app.utils.cache import set_negative_cached
|
|
mock_redis = AsyncMock()
|
|
mock_redis.set = AsyncMock()
|
|
|
|
with patch("app.utils.cache.get_redis", AsyncMock(return_value=mock_redis)):
|
|
await set_negative_cached("filings:exhibit:404:ACC:EX-99.2", ttl=3600)
|
|
|
|
mock_redis.set.assert_called_once_with(
|
|
"filings:exhibit:404:ACC:EX-99.2", b"1", ex=3600
|
|
)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_negative_cache_graceful_no_redis(self):
|
|
"""Both helpers return gracefully when Redis is unavailable."""
|
|
from app.utils.cache import get_negative_cached, set_negative_cached
|
|
|
|
with patch("app.utils.cache.get_redis", AsyncMock(return_value=None)):
|
|
assert await get_negative_cached("any:key") is False
|
|
await set_negative_cached("any:key") # must not raise
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_negative_cache_graceful_redis_error(self):
|
|
"""Both helpers swallow Redis errors."""
|
|
from app.utils.cache import get_negative_cached, set_negative_cached
|
|
mock_redis = AsyncMock()
|
|
mock_redis.exists = AsyncMock(side_effect=Exception("connection reset"))
|
|
mock_redis.set = AsyncMock(side_effect=Exception("connection reset"))
|
|
|
|
with patch("app.utils.cache.get_redis", AsyncMock(return_value=mock_redis)):
|
|
assert await get_negative_cached("any:key") is False
|
|
await set_negative_cached("any:key") # must not raise
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fix 3: Exhibit endpoint negative caching
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestExhibitEndpointNegativeCaching:
|
|
"""Test that the GET /exhibit endpoint checks & sets the negative cache."""
|
|
|
|
def _make_request(self, accession_number="0001234567-26-000001", exhibit_type="EX-99.1"):
|
|
return {"accession_number": accession_number, "exhibit_type": exhibit_type}
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_negative_cache_hit_returns_404_immediately(self):
|
|
"""When negative cache is set, endpoint returns 404 without calling service."""
|
|
from fastapi import HTTPException
|
|
from app.api.v1.endpoints.filings import get_exhibit_content
|
|
|
|
mock_db = AsyncMock()
|
|
|
|
with patch("app.api.v1.endpoints.filings.get_negative_cached", AsyncMock(return_value=True)):
|
|
with patch("app.api.v1.endpoints.filings.sec_filings_service") as mock_svc:
|
|
with pytest.raises(HTTPException) as exc_info:
|
|
await get_exhibit_content(
|
|
accession_number="0001234567-26-000001",
|
|
response=MagicMock(),
|
|
exhibit_type="EX-99.1",
|
|
db=mock_db,
|
|
)
|
|
|
|
assert exc_info.value.status_code == 404
|
|
mock_svc.get_exhibit_content.assert_not_called()
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_value_error_sets_negative_cache(self):
|
|
"""When service raises ValueError (exhibit not found), negative cache is set."""
|
|
from fastapi import HTTPException
|
|
from app.api.v1.endpoints.filings import get_exhibit_content
|
|
|
|
mock_db = AsyncMock()
|
|
set_neg = AsyncMock()
|
|
|
|
with patch("app.api.v1.endpoints.filings.get_negative_cached", AsyncMock(return_value=False)):
|
|
with patch("app.api.v1.endpoints.filings.set_negative_cached", set_neg):
|
|
with patch("app.api.v1.endpoints.filings.sec_filings_service") as mock_svc:
|
|
mock_svc.get_exhibit_content = AsyncMock(
|
|
side_effect=ValueError("Exhibit EX-99.1 not found in filing ...")
|
|
)
|
|
with pytest.raises(HTTPException) as exc_info:
|
|
await get_exhibit_content(
|
|
accession_number="0001234567-26-000001",
|
|
response=MagicMock(),
|
|
exhibit_type="EX-99.1",
|
|
db=mock_db,
|
|
)
|
|
|
|
assert exc_info.value.status_code == 404
|
|
set_neg.assert_called_once()
|
|
# Verify the key and TTL
|
|
call_args = set_neg.call_args
|
|
assert "EX-99.1" in call_args[0][0]
|
|
assert call_args[1].get("ttl") == 3600 or (len(call_args[0]) > 1 and call_args[0][1] == 3600)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_non_value_error_does_not_set_negative_cache(self):
|
|
"""Network/SEC errors (non-ValueError) must NOT populate negative cache."""
|
|
from fastapi import HTTPException
|
|
from app.api.v1.endpoints.filings import get_exhibit_content
|
|
|
|
mock_db = AsyncMock()
|
|
set_neg = AsyncMock()
|
|
|
|
with patch("app.api.v1.endpoints.filings.get_negative_cached", AsyncMock(return_value=False)):
|
|
with patch("app.api.v1.endpoints.filings.set_negative_cached", set_neg):
|
|
with patch("app.api.v1.endpoints.filings.sec_filings_service") as mock_svc:
|
|
mock_svc.get_exhibit_content = AsyncMock(
|
|
side_effect=RuntimeError("SEC request queue full")
|
|
)
|
|
with pytest.raises(HTTPException) as exc_info:
|
|
await get_exhibit_content(
|
|
accession_number="0001234567-26-000001",
|
|
response=MagicMock(),
|
|
exhibit_type="EX-99.1",
|
|
db=mock_db,
|
|
)
|
|
|
|
assert exc_info.value.status_code == 502
|
|
set_neg.assert_not_called()
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_successful_fetch_does_not_set_negative_cache(self):
|
|
"""Successful exhibit fetch must not touch the negative cache."""
|
|
from app.api.v1.endpoints.filings import get_exhibit_content
|
|
|
|
mock_db = AsyncMock()
|
|
set_neg = AsyncMock()
|
|
|
|
with patch("app.api.v1.endpoints.filings.get_negative_cached", AsyncMock(return_value=False)):
|
|
with patch("app.api.v1.endpoints.filings.set_negative_cached", set_neg):
|
|
with patch("app.api.v1.endpoints.filings.sec_filings_service") as mock_svc:
|
|
mock_svc.get_exhibit_content = AsyncMock(return_value={
|
|
"content": "<html>Press Release</html>",
|
|
"content_type": "text/html",
|
|
"filename": "exhibit99-1.htm",
|
|
"url": "https://www.sec.gov/Archives/edgar/data/123/000123/exhibit99-1.htm",
|
|
})
|
|
# Bypass the @with_cache decorator by patching set_cached_response
|
|
with patch("app.utils.cache.set_cached_response", AsyncMock(return_value="etag")):
|
|
with patch("app.utils.cache.get_cached_response", AsyncMock(return_value=None)):
|
|
result = await get_exhibit_content(
|
|
accession_number="0001234567-26-000001",
|
|
response=MagicMock(),
|
|
exhibit_type="EX-99.1",
|
|
db=mock_db,
|
|
)
|
|
|
|
assert result is not None
|
|
set_neg.assert_not_called()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fix 4 (large exhibit): max_bytes pre-check and memory cache guard
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestFetchTextMaxBytes:
|
|
def _make_client(self):
|
|
from app.services.sec_http_client import SECHttpClient, _TokenBucket
|
|
client = SECHttpClient.__new__(SECHttpClient)
|
|
client._pending = 0
|
|
client._MAX_PENDING = 50
|
|
client._text_cache = {}
|
|
client._json_cache = {}
|
|
client._cache_dir = "/tmp/test_sec_cache_mb"
|
|
client._req_sem = asyncio.Semaphore(2)
|
|
client._rate_limiter = _TokenBucket(rate=100.0, capacity=100.0)
|
|
client.http_timeout = MagicMock()
|
|
client.http_timeout.total = 12
|
|
client._session = None
|
|
return client
|
|
|
|
def _make_resp(self, content_length=None, body=b"x" * 200):
|
|
mock_resp = AsyncMock()
|
|
mock_resp.status = 200
|
|
mock_resp.content_length = content_length
|
|
mock_resp.get_encoding = MagicMock(return_value="utf-8")
|
|
# content.read returns the body bytes
|
|
mock_resp.content = AsyncMock()
|
|
mock_resp.content.read = AsyncMock(return_value=body)
|
|
mock_resp.__aenter__ = AsyncMock(return_value=mock_resp)
|
|
mock_resp.__aexit__ = AsyncMock(return_value=False)
|
|
return mock_resp
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_content_length_too_large_raises_immediately(self):
|
|
"""fetch_text with max_bytes rejects via Content-Length header before reading body."""
|
|
client = self._make_client()
|
|
mock_resp = self._make_resp(content_length=5000, body=b"x" * 5000)
|
|
mock_session = MagicMock()
|
|
mock_session.get = MagicMock(return_value=mock_resp)
|
|
|
|
with patch.object(client, "_get_session", AsyncMock(return_value=mock_session)):
|
|
with patch("os.path.exists", return_value=False):
|
|
with pytest.raises(ValueError, match="too large"):
|
|
await client.fetch_text("http://example.com/big", max_bytes=100)
|
|
|
|
# Body should not have been read when Content-Length already exceeded
|
|
mock_resp.content.read.assert_not_called()
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_no_content_length_body_too_large_raises(self):
|
|
"""fetch_text with max_bytes rejects via body read when Content-Length is absent."""
|
|
client = self._make_client()
|
|
# No Content-Length header, but body exceeds limit
|
|
mock_resp = self._make_resp(content_length=None, body=b"x" * 200)
|
|
mock_session = MagicMock()
|
|
mock_session.get = MagicMock(return_value=mock_resp)
|
|
|
|
with patch.object(client, "_get_session", AsyncMock(return_value=mock_session)):
|
|
with patch("os.path.exists", return_value=False):
|
|
with pytest.raises(ValueError, match="exceeded"):
|
|
await client.fetch_text("http://example.com/big2", max_bytes=100)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_large_text_not_stored_in_memory_cache(self):
|
|
"""Responses > 1MB must NOT be stored in _text_cache (only disk cache)."""
|
|
client = self._make_client()
|
|
large_body = b"A" * (2 * 1024 * 1024) # 2MB
|
|
mock_resp = AsyncMock()
|
|
mock_resp.status = 200
|
|
mock_resp.content_length = None
|
|
mock_resp.text = AsyncMock(return_value=large_body.decode("utf-8"))
|
|
mock_resp.__aenter__ = AsyncMock(return_value=mock_resp)
|
|
mock_resp.__aexit__ = AsyncMock(return_value=False)
|
|
mock_session = MagicMock()
|
|
mock_session.get = MagicMock(return_value=mock_resp)
|
|
|
|
with patch.object(client, "_get_session", AsyncMock(return_value=mock_session)):
|
|
with patch("app.services.sec_http_client._is_sec_block_page", return_value=False):
|
|
with patch("os.path.exists", return_value=False):
|
|
with patch("builtins.open", MagicMock()):
|
|
result = await client.fetch_text("http://example.com/large")
|
|
|
|
assert "http://example.com/large" not in client._text_cache
|
|
assert len(result) == len(large_body)
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_small_text_stored_in_memory_cache(self):
|
|
"""Responses <= 1MB should still be stored in _text_cache."""
|
|
client = self._make_client()
|
|
small_body = "hello world"
|
|
mock_resp = AsyncMock()
|
|
mock_resp.status = 200
|
|
mock_resp.content_length = None
|
|
mock_resp.text = AsyncMock(return_value=small_body)
|
|
mock_resp.__aenter__ = AsyncMock(return_value=mock_resp)
|
|
mock_resp.__aexit__ = AsyncMock(return_value=False)
|
|
mock_session = MagicMock()
|
|
mock_session.get = MagicMock(return_value=mock_resp)
|
|
|
|
with patch.object(client, "_get_session", AsyncMock(return_value=mock_session)):
|
|
with patch("app.services.sec_http_client._is_sec_block_page", return_value=False):
|
|
with patch("os.path.exists", return_value=False):
|
|
with patch("builtins.open", MagicMock()):
|
|
result = await client.fetch_text("http://example.com/small")
|
|
|
|
assert client._text_cache.get("http://example.com/small") == small_body
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_size_error_does_not_retry(self):
|
|
"""ValueError from max_bytes must propagate immediately without retrying."""
|
|
client = self._make_client()
|
|
mock_resp = self._make_resp(content_length=None, body=b"x" * 200)
|
|
mock_session = MagicMock()
|
|
mock_session.get = MagicMock(return_value=mock_resp)
|
|
call_count = 0
|
|
|
|
original_get = mock_session.get
|
|
|
|
def counting_get(*args, **kwargs):
|
|
nonlocal call_count
|
|
call_count += 1
|
|
return original_get(*args, **kwargs)
|
|
|
|
mock_session.get = counting_get
|
|
|
|
with patch.object(client, "_get_session", AsyncMock(return_value=mock_session)):
|
|
with patch("os.path.exists", return_value=False):
|
|
with pytest.raises(ValueError):
|
|
await client.fetch_text("http://example.com/nretry", max_bytes=100)
|
|
|
|
# Should have tried exactly once (no retry on ValueError)
|
|
assert call_count == 1, f"Expected 1 attempt, got {call_count}"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fix 4: skip_cache in fetch_json
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestFetchJsonSkipCache:
|
|
"""Tests for the skip_cache parameter added to fetch_json."""
|
|
|
|
def _make_client(self):
|
|
from app.services.sec_http_client import SECHttpClient, _TokenBucket
|
|
client = SECHttpClient.__new__(SECHttpClient)
|
|
client._pending = 0
|
|
client._MAX_PENDING = 50
|
|
client._text_cache = {}
|
|
client._json_cache = {}
|
|
client._cache_dir = "/tmp/test_sec_cache_skip"
|
|
client._req_sem = asyncio.Semaphore(2)
|
|
client._rate_limiter = _TokenBucket(rate=100.0, capacity=100.0)
|
|
client.http_timeout = MagicMock()
|
|
client.http_timeout.total = 12
|
|
client._session = None
|
|
return client
|
|
|
|
def _make_200_json_session(self, payload: dict):
|
|
mock_resp = AsyncMock()
|
|
mock_resp.status = 200
|
|
mock_resp.json = AsyncMock(return_value=payload)
|
|
mock_resp.__aenter__ = AsyncMock(return_value=mock_resp)
|
|
mock_resp.__aexit__ = AsyncMock(return_value=False)
|
|
mock_session = MagicMock()
|
|
mock_session.get = MagicMock(return_value=mock_resp)
|
|
return mock_session
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_skip_cache_false_uses_memory_cache(self):
|
|
"""skip_cache=False (default) returns in-memory cached value without hitting SEC."""
|
|
client = self._make_client()
|
|
url = "https://data.sec.gov/submissions/CIK0001234567.json"
|
|
client._json_cache[url] = {"cached": True}
|
|
|
|
call_count = 0
|
|
async def fake_get_session():
|
|
nonlocal call_count
|
|
call_count += 1
|
|
return MagicMock()
|
|
|
|
with patch.object(client, "_get_session", fake_get_session):
|
|
result = await client.fetch_json(url, skip_cache=False)
|
|
|
|
assert result == {"cached": True}
|
|
assert call_count == 0, "Should serve from memory cache without calling SEC"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_skip_cache_true_bypasses_memory_cache(self):
|
|
"""skip_cache=True ignores the in-memory cache and fetches fresh from SEC."""
|
|
client = self._make_client()
|
|
url = "https://data.sec.gov/submissions/CIK0001234567.json"
|
|
client._json_cache[url] = {"stale": True}
|
|
|
|
fresh_payload = {"fresh": True}
|
|
mock_session = self._make_200_json_session(fresh_payload)
|
|
|
|
with patch.object(client, "_get_session", AsyncMock(return_value=mock_session)):
|
|
with patch("builtins.open", MagicMock()):
|
|
with patch("os.path.exists", return_value=False):
|
|
result = await client.fetch_json(url, skip_cache=True)
|
|
|
|
assert result == fresh_payload, "Should return fresh data ignoring stale cache"
|
|
# In-memory cache should now be updated with fresh data
|
|
assert client._json_cache[url] == fresh_payload
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_skip_cache_true_bypasses_disk_cache(self):
|
|
"""skip_cache=True does not read from disk even when disk cache file exists."""
|
|
client = self._make_client()
|
|
url = "https://data.sec.gov/submissions/CIK0001234567.json"
|
|
# Empty in-memory cache to force disk check
|
|
client._json_cache = {}
|
|
|
|
fresh_payload = {"from_sec": True}
|
|
mock_session = self._make_200_json_session(fresh_payload)
|
|
|
|
read_disk_called = False
|
|
def fake_exists(path):
|
|
return True # Pretend disk cache file exists
|
|
|
|
original_open = open
|
|
def fake_open(path, *a, **kw):
|
|
nonlocal read_disk_called
|
|
# If this is a READ attempt (mode 'r'), flag it
|
|
mode = a[0] if a else kw.get('mode', 'r')
|
|
if 'r' in mode and '.json' in str(path):
|
|
read_disk_called = True
|
|
return MagicMock().__enter__.return_value
|
|
|
|
with patch.object(client, "_get_session", AsyncMock(return_value=mock_session)):
|
|
with patch("os.path.exists", fake_exists):
|
|
with patch("builtins.open", MagicMock()):
|
|
# skip_cache=True should skip the disk read block entirely
|
|
result = await client.fetch_json(url, skip_cache=True)
|
|
|
|
# Disk cache should NOT have been read (we got fresh payload from SEC mock)
|
|
assert result == fresh_payload
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_skip_cache_false_reads_disk_cache(self):
|
|
"""skip_cache=False (default) reads from disk when in-memory cache is empty."""
|
|
import json as _json
|
|
client = self._make_client()
|
|
url = "https://data.sec.gov/submissions/CIK0001234567.json"
|
|
client._json_cache = {} # empty memory cache
|
|
|
|
disk_payload = {"from_disk": True}
|
|
|
|
import time as _time
|
|
# Disk cache exists and is fresh
|
|
with patch("os.path.exists", return_value=True):
|
|
with patch("os.path.getmtime", return_value=_time.time()):
|
|
with patch("builtins.open", MagicMock(
|
|
return_value=MagicMock(
|
|
__enter__=MagicMock(return_value=MagicMock(
|
|
read=MagicMock(return_value=_json.dumps(disk_payload))
|
|
)),
|
|
__exit__=MagicMock(return_value=False)
|
|
)
|
|
)):
|
|
with patch("json.load", return_value=disk_payload):
|
|
result = await client.fetch_json(url, skip_cache=False)
|
|
|
|
assert result == disk_payload
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Fix 5: index_filings skip_cache / force_refresh -> skip_cache
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestIndexFilingsSkipCache:
|
|
"""Tests that index_filings passes skip_cache correctly to the HTTP client."""
|
|
|
|
def _make_service(self):
|
|
from app.services.sec_filings_service import SECFilingsService
|
|
svc = SECFilingsService.__new__(SECFilingsService)
|
|
from app.services.sec_http_client import SECHttpClient
|
|
svc._http = MagicMock(spec=SECHttpClient)
|
|
svc._http.sec_base_data = "https://data.sec.gov"
|
|
svc._http.sec_base_www = "https://www.sec.gov"
|
|
svc._http.set_deadline = MagicMock()
|
|
svc._http.clear_deadline = MagicMock()
|
|
svc._http.remaining_time = MagicMock(return_value=None)
|
|
svc._doc_fetch_locks = {}
|
|
svc._reindex_locks = {}
|
|
svc.SUPPORTED_FORM_TYPES = {"8-K", "10-K"}
|
|
svc.CHUNK_SIZE = 500
|
|
return svc
|
|
|
|
def _make_empty_submissions(self):
|
|
return {
|
|
"filings": {
|
|
"recent": {
|
|
"form": [],
|
|
"filingDate": [],
|
|
"accessionNumber": [],
|
|
"primaryDocument": [],
|
|
"primaryDocDescription": [],
|
|
"acceptanceDateTime": [],
|
|
},
|
|
"files": [],
|
|
}
|
|
}
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_force_refresh_implies_skip_cache(self):
|
|
"""force_refresh=True must call fetch_json with skip_cache=True."""
|
|
svc = self._make_service()
|
|
svc._http.get_company_cik = AsyncMock(return_value="0001730168")
|
|
svc._http.fetch_json = AsyncMock(return_value=self._make_empty_submissions())
|
|
|
|
mock_db = AsyncMock()
|
|
mock_db.execute = AsyncMock(return_value=MagicMock(scalars=MagicMock(return_value=MagicMock(all=MagicMock(return_value=[])))))
|
|
|
|
await svc.index_filings(mock_db, "AVGO", force_refresh=True)
|
|
|
|
# fetch_json must have been called with skip_cache=True
|
|
calls = svc._http.fetch_json.call_args_list
|
|
assert len(calls) >= 1
|
|
for call in calls:
|
|
_, kwargs = call
|
|
assert kwargs.get("skip_cache") is True, \
|
|
f"Expected skip_cache=True when force_refresh=True, got: {call}"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_skip_cache_false_by_default(self):
|
|
"""Normal index_filings call must NOT bypass cache."""
|
|
svc = self._make_service()
|
|
svc._http.get_company_cik = AsyncMock(return_value="0001730168")
|
|
svc._http.fetch_json = AsyncMock(return_value=self._make_empty_submissions())
|
|
|
|
mock_db = AsyncMock()
|
|
mock_db.execute = AsyncMock(return_value=MagicMock(
|
|
fetchall=MagicMock(return_value=[])
|
|
))
|
|
|
|
await svc.index_filings(mock_db, "AVGO")
|
|
|
|
calls = svc._http.fetch_json.call_args_list
|
|
assert len(calls) >= 1
|
|
for call in calls:
|
|
_, kwargs = call
|
|
assert kwargs.get("skip_cache") is False, \
|
|
f"Expected skip_cache=False by default, got: {call}"
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_explicit_skip_cache_true(self):
|
|
"""Explicit skip_cache=True (without force_refresh) must bypass cache."""
|
|
svc = self._make_service()
|
|
svc._http.get_company_cik = AsyncMock(return_value="0001730168")
|
|
svc._http.fetch_json = AsyncMock(return_value=self._make_empty_submissions())
|
|
|
|
mock_db = AsyncMock()
|
|
mock_db.execute = AsyncMock(return_value=MagicMock(
|
|
fetchall=MagicMock(return_value=[])
|
|
))
|
|
|
|
await svc.index_filings(mock_db, "AVGO", skip_cache=True)
|
|
|
|
calls = svc._http.fetch_json.call_args_list
|
|
assert len(calls) >= 1
|
|
for call in calls:
|
|
_, kwargs = call
|
|
assert kwargs.get("skip_cache") is True, \
|
|
f"Expected skip_cache=True when explicit, got: {call}"
|