""" Unit tests for SEC filings fixes: 1. Backpressure: SECHttpClient._pending counter rejects when queue full 2. Negative cache helpers: get_negative_cached / set_negative_cached 3. Exhibit endpoint negative caching: 404 cached, skipped on repeat """ import asyncio import pytest from unittest.mock import AsyncMock, MagicMock, patch # --------------------------------------------------------------------------- # Fix 1: Backpressure — _pending counter # --------------------------------------------------------------------------- class TestSECHttpClientBackpressure: def _make_client(self): from app.services.sec_http_client import SECHttpClient client = SECHttpClient.__new__(SECHttpClient) client._pending = 0 client._MAX_PENDING = 5 client._text_cache = {} client._json_cache = {} client._cache_dir = "/tmp/test_sec_cache_bp" client._req_sem = asyncio.Semaphore(2) from app.services.sec_http_client import _TokenBucket client._rate_limiter = _TokenBucket(rate=100.0, capacity=100.0) client.http_timeout = MagicMock() client.http_timeout.total = 12 client._session = None return client @pytest.mark.asyncio async def test_pending_counter_increments_and_decrements(self): """_pending goes up during fetch and back to 0 after.""" client = self._make_client() # Patch _get_session to return a mock that yields a 200 response mock_resp = AsyncMock() mock_resp.status = 200 mock_resp.text = AsyncMock(return_value="hello world") mock_resp.__aenter__ = AsyncMock(return_value=mock_resp) mock_resp.__aexit__ = AsyncMock(return_value=False) mock_session = MagicMock() mock_session.get = MagicMock(return_value=mock_resp) with patch.object(client, "_get_session", AsyncMock(return_value=mock_session)): with patch("app.services.sec_http_client._is_sec_block_page", return_value=False): with patch("os.path.exists", return_value=False): result = await client.fetch_text("http://example.com/test") assert result == "hello world" assert client._pending == 0, "pending must be 0 after completion" @pytest.mark.asyncio async def test_queue_full_raises_immediately(self): """When _pending >= _MAX_PENDING, fetch_text raises without touching SEC.""" client = self._make_client() client._pending = client._MAX_PENDING # Already at limit with patch("os.path.exists", return_value=False): with pytest.raises(RuntimeError, match="SEC request queue full"): await client.fetch_text("http://example.com/overload") @pytest.mark.asyncio async def test_queue_full_json_raises_immediately(self): """Same check for fetch_json.""" client = self._make_client() client._pending = client._MAX_PENDING with patch("os.path.exists", return_value=False): with pytest.raises(RuntimeError, match="SEC request queue full"): await client.fetch_json("http://example.com/overload.json") @pytest.mark.asyncio async def test_pending_decrements_on_exception(self): """_pending must be decremented even when an exception is raised.""" client = self._make_client() assert client._pending == 0 with patch("os.path.exists", return_value=False): with patch.object(client, "_get_session", AsyncMock(side_effect=RuntimeError("boom"))): with pytest.raises(Exception): await client.fetch_text("http://example.com/fail") assert client._pending == 0, "pending must return to 0 after exception" @pytest.mark.asyncio async def test_rate_reduced_to_8(self): """Rate limiter must be 8.0 req/sec, not 10.0.""" from app.services.sec_http_client import SECHttpClient client = SECHttpClient("Test") assert client._rate_limiter._rate == 8.0 assert client._rate_limiter._capacity == 8.0 # --------------------------------------------------------------------------- # Fix 2: Negative cache helpers # --------------------------------------------------------------------------- class TestNegativeCacheHelpers: @pytest.mark.asyncio async def test_get_negative_cached_miss(self): """Returns False when key not in Redis.""" from app.utils.cache import get_negative_cached mock_redis = AsyncMock() mock_redis.exists = AsyncMock(return_value=0) with patch("app.utils.cache.get_redis", AsyncMock(return_value=mock_redis)): result = await get_negative_cached("filings:exhibit:404:ACC:EX-99.1") assert result is False @pytest.mark.asyncio async def test_get_negative_cached_hit(self): """Returns True when key exists in Redis.""" from app.utils.cache import get_negative_cached mock_redis = AsyncMock() mock_redis.exists = AsyncMock(return_value=1) with patch("app.utils.cache.get_redis", AsyncMock(return_value=mock_redis)): result = await get_negative_cached("filings:exhibit:404:ACC:EX-99.1") assert result is True @pytest.mark.asyncio async def test_set_negative_cached_calls_redis(self): """set_negative_cached calls redis.set with b'1' and the TTL.""" from app.utils.cache import set_negative_cached mock_redis = AsyncMock() mock_redis.set = AsyncMock() with patch("app.utils.cache.get_redis", AsyncMock(return_value=mock_redis)): await set_negative_cached("filings:exhibit:404:ACC:EX-99.2", ttl=3600) mock_redis.set.assert_called_once_with( "filings:exhibit:404:ACC:EX-99.2", b"1", ex=3600 ) @pytest.mark.asyncio async def test_negative_cache_graceful_no_redis(self): """Both helpers return gracefully when Redis is unavailable.""" from app.utils.cache import get_negative_cached, set_negative_cached with patch("app.utils.cache.get_redis", AsyncMock(return_value=None)): assert await get_negative_cached("any:key") is False await set_negative_cached("any:key") # must not raise @pytest.mark.asyncio async def test_negative_cache_graceful_redis_error(self): """Both helpers swallow Redis errors.""" from app.utils.cache import get_negative_cached, set_negative_cached mock_redis = AsyncMock() mock_redis.exists = AsyncMock(side_effect=Exception("connection reset")) mock_redis.set = AsyncMock(side_effect=Exception("connection reset")) with patch("app.utils.cache.get_redis", AsyncMock(return_value=mock_redis)): assert await get_negative_cached("any:key") is False await set_negative_cached("any:key") # must not raise # --------------------------------------------------------------------------- # Fix 3: Exhibit endpoint negative caching # --------------------------------------------------------------------------- class TestExhibitEndpointNegativeCaching: """Test that the GET /exhibit endpoint checks & sets the negative cache.""" def _make_request(self, accession_number="0001234567-26-000001", exhibit_type="EX-99.1"): return {"accession_number": accession_number, "exhibit_type": exhibit_type} @pytest.mark.asyncio async def test_negative_cache_hit_returns_404_immediately(self): """When negative cache is set, endpoint returns 404 without calling service.""" from fastapi import HTTPException from app.api.v1.endpoints.filings import get_exhibit_content mock_db = AsyncMock() with patch("app.api.v1.endpoints.filings.get_negative_cached", AsyncMock(return_value=True)): with patch("app.api.v1.endpoints.filings.sec_filings_service") as mock_svc: with pytest.raises(HTTPException) as exc_info: await get_exhibit_content( accession_number="0001234567-26-000001", response=MagicMock(), exhibit_type="EX-99.1", db=mock_db, ) assert exc_info.value.status_code == 404 mock_svc.get_exhibit_content.assert_not_called() @pytest.mark.asyncio async def test_value_error_sets_negative_cache(self): """When service raises ValueError (exhibit not found), negative cache is set.""" from fastapi import HTTPException from app.api.v1.endpoints.filings import get_exhibit_content mock_db = AsyncMock() set_neg = AsyncMock() with patch("app.api.v1.endpoints.filings.get_negative_cached", AsyncMock(return_value=False)): with patch("app.api.v1.endpoints.filings.set_negative_cached", set_neg): with patch("app.api.v1.endpoints.filings.sec_filings_service") as mock_svc: mock_svc.get_exhibit_content = AsyncMock( side_effect=ValueError("Exhibit EX-99.1 not found in filing ...") ) with pytest.raises(HTTPException) as exc_info: await get_exhibit_content( accession_number="0001234567-26-000001", response=MagicMock(), exhibit_type="EX-99.1", db=mock_db, ) assert exc_info.value.status_code == 404 set_neg.assert_called_once() # Verify the key and TTL call_args = set_neg.call_args assert "EX-99.1" in call_args[0][0] assert call_args[1].get("ttl") == 3600 or (len(call_args[0]) > 1 and call_args[0][1] == 3600) @pytest.mark.asyncio async def test_non_value_error_does_not_set_negative_cache(self): """Network/SEC errors (non-ValueError) must NOT populate negative cache.""" from fastapi import HTTPException from app.api.v1.endpoints.filings import get_exhibit_content mock_db = AsyncMock() set_neg = AsyncMock() with patch("app.api.v1.endpoints.filings.get_negative_cached", AsyncMock(return_value=False)): with patch("app.api.v1.endpoints.filings.set_negative_cached", set_neg): with patch("app.api.v1.endpoints.filings.sec_filings_service") as mock_svc: mock_svc.get_exhibit_content = AsyncMock( side_effect=RuntimeError("SEC request queue full") ) with pytest.raises(HTTPException) as exc_info: await get_exhibit_content( accession_number="0001234567-26-000001", response=MagicMock(), exhibit_type="EX-99.1", db=mock_db, ) assert exc_info.value.status_code == 502 set_neg.assert_not_called() @pytest.mark.asyncio async def test_successful_fetch_does_not_set_negative_cache(self): """Successful exhibit fetch must not touch the negative cache.""" from app.api.v1.endpoints.filings import get_exhibit_content mock_db = AsyncMock() set_neg = AsyncMock() with patch("app.api.v1.endpoints.filings.get_negative_cached", AsyncMock(return_value=False)): with patch("app.api.v1.endpoints.filings.set_negative_cached", set_neg): with patch("app.api.v1.endpoints.filings.sec_filings_service") as mock_svc: mock_svc.get_exhibit_content = AsyncMock(return_value={ "content": "Press Release", "content_type": "text/html", "filename": "exhibit99-1.htm", "url": "https://www.sec.gov/Archives/edgar/data/123/000123/exhibit99-1.htm", }) # Bypass the @with_cache decorator by patching set_cached_response with patch("app.utils.cache.set_cached_response", AsyncMock(return_value="etag")): with patch("app.utils.cache.get_cached_response", AsyncMock(return_value=None)): result = await get_exhibit_content( accession_number="0001234567-26-000001", response=MagicMock(), exhibit_type="EX-99.1", db=mock_db, ) assert result is not None set_neg.assert_not_called() # --------------------------------------------------------------------------- # Fix 4 (large exhibit): max_bytes pre-check and memory cache guard # --------------------------------------------------------------------------- class TestFetchTextMaxBytes: def _make_client(self): from app.services.sec_http_client import SECHttpClient, _TokenBucket client = SECHttpClient.__new__(SECHttpClient) client._pending = 0 client._MAX_PENDING = 50 client._text_cache = {} client._json_cache = {} client._cache_dir = "/tmp/test_sec_cache_mb" client._req_sem = asyncio.Semaphore(2) client._rate_limiter = _TokenBucket(rate=100.0, capacity=100.0) client.http_timeout = MagicMock() client.http_timeout.total = 12 client._session = None return client def _make_resp(self, content_length=None, body=b"x" * 200): mock_resp = AsyncMock() mock_resp.status = 200 mock_resp.content_length = content_length mock_resp.get_encoding = MagicMock(return_value="utf-8") # content.read returns the body bytes mock_resp.content = AsyncMock() mock_resp.content.read = AsyncMock(return_value=body) mock_resp.__aenter__ = AsyncMock(return_value=mock_resp) mock_resp.__aexit__ = AsyncMock(return_value=False) return mock_resp @pytest.mark.asyncio async def test_content_length_too_large_raises_immediately(self): """fetch_text with max_bytes rejects via Content-Length header before reading body.""" client = self._make_client() mock_resp = self._make_resp(content_length=5000, body=b"x" * 5000) mock_session = MagicMock() mock_session.get = MagicMock(return_value=mock_resp) with patch.object(client, "_get_session", AsyncMock(return_value=mock_session)): with patch("os.path.exists", return_value=False): with pytest.raises(ValueError, match="too large"): await client.fetch_text("http://example.com/big", max_bytes=100) # Body should not have been read when Content-Length already exceeded mock_resp.content.read.assert_not_called() @pytest.mark.asyncio async def test_no_content_length_body_too_large_raises(self): """fetch_text with max_bytes rejects via body read when Content-Length is absent.""" client = self._make_client() # No Content-Length header, but body exceeds limit mock_resp = self._make_resp(content_length=None, body=b"x" * 200) mock_session = MagicMock() mock_session.get = MagicMock(return_value=mock_resp) with patch.object(client, "_get_session", AsyncMock(return_value=mock_session)): with patch("os.path.exists", return_value=False): with pytest.raises(ValueError, match="exceeded"): await client.fetch_text("http://example.com/big2", max_bytes=100) @pytest.mark.asyncio async def test_large_text_not_stored_in_memory_cache(self): """Responses > 1MB must NOT be stored in _text_cache (only disk cache).""" client = self._make_client() large_body = b"A" * (2 * 1024 * 1024) # 2MB mock_resp = AsyncMock() mock_resp.status = 200 mock_resp.content_length = None mock_resp.text = AsyncMock(return_value=large_body.decode("utf-8")) mock_resp.__aenter__ = AsyncMock(return_value=mock_resp) mock_resp.__aexit__ = AsyncMock(return_value=False) mock_session = MagicMock() mock_session.get = MagicMock(return_value=mock_resp) with patch.object(client, "_get_session", AsyncMock(return_value=mock_session)): with patch("app.services.sec_http_client._is_sec_block_page", return_value=False): with patch("os.path.exists", return_value=False): with patch("builtins.open", MagicMock()): result = await client.fetch_text("http://example.com/large") assert "http://example.com/large" not in client._text_cache assert len(result) == len(large_body) @pytest.mark.asyncio async def test_small_text_stored_in_memory_cache(self): """Responses <= 1MB should still be stored in _text_cache.""" client = self._make_client() small_body = "hello world" mock_resp = AsyncMock() mock_resp.status = 200 mock_resp.content_length = None mock_resp.text = AsyncMock(return_value=small_body) mock_resp.__aenter__ = AsyncMock(return_value=mock_resp) mock_resp.__aexit__ = AsyncMock(return_value=False) mock_session = MagicMock() mock_session.get = MagicMock(return_value=mock_resp) with patch.object(client, "_get_session", AsyncMock(return_value=mock_session)): with patch("app.services.sec_http_client._is_sec_block_page", return_value=False): with patch("os.path.exists", return_value=False): with patch("builtins.open", MagicMock()): result = await client.fetch_text("http://example.com/small") assert client._text_cache.get("http://example.com/small") == small_body @pytest.mark.asyncio async def test_size_error_does_not_retry(self): """ValueError from max_bytes must propagate immediately without retrying.""" client = self._make_client() mock_resp = self._make_resp(content_length=None, body=b"x" * 200) mock_session = MagicMock() mock_session.get = MagicMock(return_value=mock_resp) call_count = 0 original_get = mock_session.get def counting_get(*args, **kwargs): nonlocal call_count call_count += 1 return original_get(*args, **kwargs) mock_session.get = counting_get with patch.object(client, "_get_session", AsyncMock(return_value=mock_session)): with patch("os.path.exists", return_value=False): with pytest.raises(ValueError): await client.fetch_text("http://example.com/nretry", max_bytes=100) # Should have tried exactly once (no retry on ValueError) assert call_count == 1, f"Expected 1 attempt, got {call_count}" # --------------------------------------------------------------------------- # Fix 4: skip_cache in fetch_json # --------------------------------------------------------------------------- class TestFetchJsonSkipCache: """Tests for the skip_cache parameter added to fetch_json.""" def _make_client(self): from app.services.sec_http_client import SECHttpClient, _TokenBucket client = SECHttpClient.__new__(SECHttpClient) client._pending = 0 client._MAX_PENDING = 50 client._text_cache = {} client._json_cache = {} client._cache_dir = "/tmp/test_sec_cache_skip" client._req_sem = asyncio.Semaphore(2) client._rate_limiter = _TokenBucket(rate=100.0, capacity=100.0) client.http_timeout = MagicMock() client.http_timeout.total = 12 client._session = None return client def _make_200_json_session(self, payload: dict): mock_resp = AsyncMock() mock_resp.status = 200 mock_resp.json = AsyncMock(return_value=payload) mock_resp.__aenter__ = AsyncMock(return_value=mock_resp) mock_resp.__aexit__ = AsyncMock(return_value=False) mock_session = MagicMock() mock_session.get = MagicMock(return_value=mock_resp) return mock_session @pytest.mark.asyncio async def test_skip_cache_false_uses_memory_cache(self): """skip_cache=False (default) returns in-memory cached value without hitting SEC.""" client = self._make_client() url = "https://data.sec.gov/submissions/CIK0001234567.json" client._json_cache[url] = {"cached": True} call_count = 0 async def fake_get_session(): nonlocal call_count call_count += 1 return MagicMock() with patch.object(client, "_get_session", fake_get_session): result = await client.fetch_json(url, skip_cache=False) assert result == {"cached": True} assert call_count == 0, "Should serve from memory cache without calling SEC" @pytest.mark.asyncio async def test_skip_cache_true_bypasses_memory_cache(self): """skip_cache=True ignores the in-memory cache and fetches fresh from SEC.""" client = self._make_client() url = "https://data.sec.gov/submissions/CIK0001234567.json" client._json_cache[url] = {"stale": True} fresh_payload = {"fresh": True} mock_session = self._make_200_json_session(fresh_payload) with patch.object(client, "_get_session", AsyncMock(return_value=mock_session)): with patch("builtins.open", MagicMock()): with patch("os.path.exists", return_value=False): result = await client.fetch_json(url, skip_cache=True) assert result == fresh_payload, "Should return fresh data ignoring stale cache" # In-memory cache should now be updated with fresh data assert client._json_cache[url] == fresh_payload @pytest.mark.asyncio async def test_skip_cache_true_bypasses_disk_cache(self): """skip_cache=True does not read from disk even when disk cache file exists.""" client = self._make_client() url = "https://data.sec.gov/submissions/CIK0001234567.json" # Empty in-memory cache to force disk check client._json_cache = {} fresh_payload = {"from_sec": True} mock_session = self._make_200_json_session(fresh_payload) read_disk_called = False def fake_exists(path): return True # Pretend disk cache file exists original_open = open def fake_open(path, *a, **kw): nonlocal read_disk_called # If this is a READ attempt (mode 'r'), flag it mode = a[0] if a else kw.get('mode', 'r') if 'r' in mode and '.json' in str(path): read_disk_called = True return MagicMock().__enter__.return_value with patch.object(client, "_get_session", AsyncMock(return_value=mock_session)): with patch("os.path.exists", fake_exists): with patch("builtins.open", MagicMock()): # skip_cache=True should skip the disk read block entirely result = await client.fetch_json(url, skip_cache=True) # Disk cache should NOT have been read (we got fresh payload from SEC mock) assert result == fresh_payload @pytest.mark.asyncio async def test_skip_cache_false_reads_disk_cache(self): """skip_cache=False (default) reads from disk when in-memory cache is empty.""" import json as _json client = self._make_client() url = "https://data.sec.gov/submissions/CIK0001234567.json" client._json_cache = {} # empty memory cache disk_payload = {"from_disk": True} import time as _time # Disk cache exists and is fresh with patch("os.path.exists", return_value=True): with patch("os.path.getmtime", return_value=_time.time()): with patch("builtins.open", MagicMock( return_value=MagicMock( __enter__=MagicMock(return_value=MagicMock( read=MagicMock(return_value=_json.dumps(disk_payload)) )), __exit__=MagicMock(return_value=False) ) )): with patch("json.load", return_value=disk_payload): result = await client.fetch_json(url, skip_cache=False) assert result == disk_payload # --------------------------------------------------------------------------- # Fix 5: index_filings skip_cache / force_refresh -> skip_cache # --------------------------------------------------------------------------- class TestIndexFilingsSkipCache: """Tests that index_filings passes skip_cache correctly to the HTTP client.""" def _make_service(self): from app.services.sec_filings_service import SECFilingsService svc = SECFilingsService.__new__(SECFilingsService) from app.services.sec_http_client import SECHttpClient svc._http = MagicMock(spec=SECHttpClient) svc._http.sec_base_data = "https://data.sec.gov" svc._http.sec_base_www = "https://www.sec.gov" svc._http.set_deadline = MagicMock() svc._http.clear_deadline = MagicMock() svc._http.remaining_time = MagicMock(return_value=None) svc._doc_fetch_locks = {} svc._reindex_locks = {} svc.SUPPORTED_FORM_TYPES = {"8-K", "10-K"} svc.CHUNK_SIZE = 500 return svc def _make_empty_submissions(self): return { "filings": { "recent": { "form": [], "filingDate": [], "accessionNumber": [], "primaryDocument": [], "primaryDocDescription": [], "acceptanceDateTime": [], }, "files": [], } } @pytest.mark.asyncio async def test_force_refresh_implies_skip_cache(self): """force_refresh=True must call fetch_json with skip_cache=True.""" svc = self._make_service() svc._http.get_company_cik = AsyncMock(return_value="0001730168") svc._http.fetch_json = AsyncMock(return_value=self._make_empty_submissions()) mock_db = AsyncMock() mock_db.execute = AsyncMock(return_value=MagicMock(scalars=MagicMock(return_value=MagicMock(all=MagicMock(return_value=[]))))) await svc.index_filings(mock_db, "AVGO", force_refresh=True) # fetch_json must have been called with skip_cache=True calls = svc._http.fetch_json.call_args_list assert len(calls) >= 1 for call in calls: _, kwargs = call assert kwargs.get("skip_cache") is True, \ f"Expected skip_cache=True when force_refresh=True, got: {call}" @pytest.mark.asyncio async def test_skip_cache_false_by_default(self): """Normal index_filings call must NOT bypass cache.""" svc = self._make_service() svc._http.get_company_cik = AsyncMock(return_value="0001730168") svc._http.fetch_json = AsyncMock(return_value=self._make_empty_submissions()) mock_db = AsyncMock() mock_db.execute = AsyncMock(return_value=MagicMock( fetchall=MagicMock(return_value=[]) )) await svc.index_filings(mock_db, "AVGO") calls = svc._http.fetch_json.call_args_list assert len(calls) >= 1 for call in calls: _, kwargs = call assert kwargs.get("skip_cache") is False, \ f"Expected skip_cache=False by default, got: {call}" @pytest.mark.asyncio async def test_explicit_skip_cache_true(self): """Explicit skip_cache=True (without force_refresh) must bypass cache.""" svc = self._make_service() svc._http.get_company_cik = AsyncMock(return_value="0001730168") svc._http.fetch_json = AsyncMock(return_value=self._make_empty_submissions()) mock_db = AsyncMock() mock_db.execute = AsyncMock(return_value=MagicMock( fetchall=MagicMock(return_value=[]) )) await svc.index_filings(mock_db, "AVGO", skip_cache=True) calls = svc._http.fetch_json.call_args_list assert len(calls) >= 1 for call in calls: _, kwargs = call assert kwargs.get("skip_cache") is True, \ f"Expected skip_cache=True when explicit, got: {call}"