diff --git a/app/services/sec_http_client.py b/app/services/sec_http_client.py index 967472c..494160c 100644 --- a/app/services/sec_http_client.py +++ b/app/services/sec_http_client.py @@ -180,23 +180,29 @@ class SECHttpClient: # HTTP fetch methods # ------------------------------------------------------------------ - async def fetch_json(self, url: str) -> dict: - """Fetch JSON with retry, backoff, and caching.""" + async def fetch_json(self, url: str, skip_cache: bool = False) -> dict: + """Fetch JSON with retry, backoff, and caching. + + Args: + skip_cache: If True, bypass in-memory and disk cache reads (still writes + to cache after a successful fetch so subsequent calls benefit). + """ # In-memory cache - if url in self._json_cache: + if not skip_cache and url in self._json_cache: return self._json_cache[url] # Disk cache - try: - cp = self._cache_path(url) + ".json" - if os.path.exists(cp): - ttl_sec = max(3600, settings.SEC_DATA_REFRESH_HOURS * 3600) - if _time.time() - os.path.getmtime(cp) <= ttl_sec: - with open(cp, "r", encoding="utf-8") as f: - data = json.load(f) - self._json_cache[url] = data - return data - except Exception: - pass + if not skip_cache: + try: + cp = self._cache_path(url) + ".json" + if os.path.exists(cp): + ttl_sec = max(3600, settings.SEC_DATA_REFRESH_HOURS * 3600) + if _time.time() - os.path.getmtime(cp) <= ttl_sec: + with open(cp, "r", encoding="utf-8") as f: + data = json.load(f) + self._json_cache[url] = data + return data + except Exception: + pass # Backpressure: reject immediately if too many SEC requests are already pending. # This prevents thousands of coroutines from stacking up in the event loop, diff --git a/tests/test_sec_filings_fixes.py b/tests/test_sec_filings_fixes.py index 02a6754..9cfd05e 100644 --- a/tests/test_sec_filings_fixes.py +++ b/tests/test_sec_filings_fixes.py @@ -411,3 +411,235 @@ class TestFetchTextMaxBytes: # Should have tried exactly once (no retry on ValueError) assert call_count == 1, f"Expected 1 attempt, got {call_count}" + + +# --------------------------------------------------------------------------- +# Fix 4: skip_cache in fetch_json +# --------------------------------------------------------------------------- + +class TestFetchJsonSkipCache: + """Tests for the skip_cache parameter added to fetch_json.""" + + def _make_client(self): + from app.services.sec_http_client import SECHttpClient, _TokenBucket + client = SECHttpClient.__new__(SECHttpClient) + client._pending = 0 + client._MAX_PENDING = 50 + client._text_cache = {} + client._json_cache = {} + client._cache_dir = "/tmp/test_sec_cache_skip" + client._req_sem = asyncio.Semaphore(2) + client._rate_limiter = _TokenBucket(rate=100.0, capacity=100.0) + client.http_timeout = MagicMock() + client.http_timeout.total = 12 + client._session = None + return client + + def _make_200_json_session(self, payload: dict): + mock_resp = AsyncMock() + mock_resp.status = 200 + mock_resp.json = AsyncMock(return_value=payload) + mock_resp.__aenter__ = AsyncMock(return_value=mock_resp) + mock_resp.__aexit__ = AsyncMock(return_value=False) + mock_session = MagicMock() + mock_session.get = MagicMock(return_value=mock_resp) + return mock_session + + @pytest.mark.asyncio + async def test_skip_cache_false_uses_memory_cache(self): + """skip_cache=False (default) returns in-memory cached value without hitting SEC.""" + client = self._make_client() + url = "https://data.sec.gov/submissions/CIK0001234567.json" + client._json_cache[url] = {"cached": True} + + call_count = 0 + async def fake_get_session(): + nonlocal call_count + call_count += 1 + return MagicMock() + + with patch.object(client, "_get_session", fake_get_session): + result = await client.fetch_json(url, skip_cache=False) + + assert result == {"cached": True} + assert call_count == 0, "Should serve from memory cache without calling SEC" + + @pytest.mark.asyncio + async def test_skip_cache_true_bypasses_memory_cache(self): + """skip_cache=True ignores the in-memory cache and fetches fresh from SEC.""" + client = self._make_client() + url = "https://data.sec.gov/submissions/CIK0001234567.json" + client._json_cache[url] = {"stale": True} + + fresh_payload = {"fresh": True} + mock_session = self._make_200_json_session(fresh_payload) + + with patch.object(client, "_get_session", AsyncMock(return_value=mock_session)): + with patch("builtins.open", MagicMock()): + with patch("os.path.exists", return_value=False): + result = await client.fetch_json(url, skip_cache=True) + + assert result == fresh_payload, "Should return fresh data ignoring stale cache" + # In-memory cache should now be updated with fresh data + assert client._json_cache[url] == fresh_payload + + @pytest.mark.asyncio + async def test_skip_cache_true_bypasses_disk_cache(self): + """skip_cache=True does not read from disk even when disk cache file exists.""" + client = self._make_client() + url = "https://data.sec.gov/submissions/CIK0001234567.json" + # Empty in-memory cache to force disk check + client._json_cache = {} + + fresh_payload = {"from_sec": True} + mock_session = self._make_200_json_session(fresh_payload) + + read_disk_called = False + def fake_exists(path): + return True # Pretend disk cache file exists + + original_open = open + def fake_open(path, *a, **kw): + nonlocal read_disk_called + # If this is a READ attempt (mode 'r'), flag it + mode = a[0] if a else kw.get('mode', 'r') + if 'r' in mode and '.json' in str(path): + read_disk_called = True + return MagicMock().__enter__.return_value + + with patch.object(client, "_get_session", AsyncMock(return_value=mock_session)): + with patch("os.path.exists", fake_exists): + with patch("builtins.open", MagicMock()): + # skip_cache=True should skip the disk read block entirely + result = await client.fetch_json(url, skip_cache=True) + + # Disk cache should NOT have been read (we got fresh payload from SEC mock) + assert result == fresh_payload + + @pytest.mark.asyncio + async def test_skip_cache_false_reads_disk_cache(self): + """skip_cache=False (default) reads from disk when in-memory cache is empty.""" + import json as _json + client = self._make_client() + url = "https://data.sec.gov/submissions/CIK0001234567.json" + client._json_cache = {} # empty memory cache + + disk_payload = {"from_disk": True} + + import time as _time + # Disk cache exists and is fresh + with patch("os.path.exists", return_value=True): + with patch("os.path.getmtime", return_value=_time.time()): + with patch("builtins.open", MagicMock( + return_value=MagicMock( + __enter__=MagicMock(return_value=MagicMock( + read=MagicMock(return_value=_json.dumps(disk_payload)) + )), + __exit__=MagicMock(return_value=False) + ) + )): + with patch("json.load", return_value=disk_payload): + result = await client.fetch_json(url, skip_cache=False) + + assert result == disk_payload + + +# --------------------------------------------------------------------------- +# Fix 5: index_filings skip_cache / force_refresh -> skip_cache +# --------------------------------------------------------------------------- + +class TestIndexFilingsSkipCache: + """Tests that index_filings passes skip_cache correctly to the HTTP client.""" + + def _make_service(self): + from app.services.sec_filings_service import SECFilingsService + svc = SECFilingsService.__new__(SECFilingsService) + from app.services.sec_http_client import SECHttpClient + svc._http = MagicMock(spec=SECHttpClient) + svc._http.sec_base_data = "https://data.sec.gov" + svc._http.sec_base_www = "https://www.sec.gov" + svc._http.set_deadline = MagicMock() + svc._http.clear_deadline = MagicMock() + svc._http.remaining_time = MagicMock(return_value=None) + svc._doc_fetch_locks = {} + svc._reindex_locks = {} + svc.SUPPORTED_FORM_TYPES = {"8-K", "10-K"} + svc.CHUNK_SIZE = 500 + return svc + + def _make_empty_submissions(self): + return { + "filings": { + "recent": { + "form": [], + "filingDate": [], + "accessionNumber": [], + "primaryDocument": [], + "primaryDocDescription": [], + "acceptanceDateTime": [], + }, + "files": [], + } + } + + @pytest.mark.asyncio + async def test_force_refresh_implies_skip_cache(self): + """force_refresh=True must call fetch_json with skip_cache=True.""" + svc = self._make_service() + svc._http.get_company_cik = AsyncMock(return_value="0001730168") + svc._http.fetch_json = AsyncMock(return_value=self._make_empty_submissions()) + + mock_db = AsyncMock() + mock_db.execute = AsyncMock(return_value=MagicMock(scalars=MagicMock(return_value=MagicMock(all=MagicMock(return_value=[]))))) + + await svc.index_filings(mock_db, "AVGO", force_refresh=True) + + # fetch_json must have been called with skip_cache=True + calls = svc._http.fetch_json.call_args_list + assert len(calls) >= 1 + for call in calls: + _, kwargs = call + assert kwargs.get("skip_cache") is True, \ + f"Expected skip_cache=True when force_refresh=True, got: {call}" + + @pytest.mark.asyncio + async def test_skip_cache_false_by_default(self): + """Normal index_filings call must NOT bypass cache.""" + svc = self._make_service() + svc._http.get_company_cik = AsyncMock(return_value="0001730168") + svc._http.fetch_json = AsyncMock(return_value=self._make_empty_submissions()) + + mock_db = AsyncMock() + mock_db.execute = AsyncMock(return_value=MagicMock( + fetchall=MagicMock(return_value=[]) + )) + + await svc.index_filings(mock_db, "AVGO") + + calls = svc._http.fetch_json.call_args_list + assert len(calls) >= 1 + for call in calls: + _, kwargs = call + assert kwargs.get("skip_cache") is False, \ + f"Expected skip_cache=False by default, got: {call}" + + @pytest.mark.asyncio + async def test_explicit_skip_cache_true(self): + """Explicit skip_cache=True (without force_refresh) must bypass cache.""" + svc = self._make_service() + svc._http.get_company_cik = AsyncMock(return_value="0001730168") + svc._http.fetch_json = AsyncMock(return_value=self._make_empty_submissions()) + + mock_db = AsyncMock() + mock_db.execute = AsyncMock(return_value=MagicMock( + fetchall=MagicMock(return_value=[]) + )) + + await svc.index_filings(mock_db, "AVGO", skip_cache=True) + + calls = svc._http.fetch_json.call_args_list + assert len(calls) >= 1 + for call in calls: + _, kwargs = call + assert kwargs.get("skip_cache") is True, \ + f"Expected skip_cache=True when explicit, got: {call}"